同步完整源码 - 2026-05-24
This commit is contained in:
xiaoxue
commit
2b4682de8e
|
|
@ -0,0 +1,15 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
end_of_line = lf
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
|
||||
[{*.yaml, *.yml}]
|
||||
indent_size = 2
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# Documentation codeowner
|
||||
|
||||
/docs/*.md @TC-MO
|
||||
/docs/*.mdx @TC-MO
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
### Description
|
||||
|
||||
<!-- The purpose of the PR, list of the changes, ... -->
|
||||
|
||||
- TODO
|
||||
|
||||
### Issues
|
||||
|
||||
<!-- If applicable, reference any related GitHub issues -->
|
||||
|
||||
- Closes: #TODO
|
||||
|
||||
### Testing
|
||||
|
||||
<!-- Describe the testing process for these changes -->
|
||||
|
||||
- TODO
|
||||
|
||||
### Checklist
|
||||
|
||||
- [ ] CI passed
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
name: Code checks
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
actions_lint_check:
|
||||
name: Actions lint check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Run actionlint
|
||||
uses: rhysd/actionlint@v1.7.12
|
||||
|
||||
spell_check:
|
||||
name: Spell check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Check spelling with typos
|
||||
uses: crate-ci/typos@v1
|
||||
|
||||
lint_check:
|
||||
name: Lint check
|
||||
uses: apify/workflows/.github/workflows/python_lint_check.yaml@main
|
||||
with:
|
||||
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
|
||||
|
||||
type_check:
|
||||
name: Type check
|
||||
uses: apify/workflows/.github/workflows/python_type_check.yaml@main
|
||||
with:
|
||||
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
name: Doc checks
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
doc_checks:
|
||||
name: Doc checks
|
||||
uses: apify/workflows/.github/workflows/python_docs_check.yaml@main
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
name: Package check
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
package_check:
|
||||
name: Package check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up uv package manager
|
||||
uses: astral-sh/setup-uv@v8.1.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Build sdist and wheel
|
||||
run: uv run poe build
|
||||
|
||||
- name: Verify built package
|
||||
uses: apify/actions/python-package-check@v1.1.0
|
||||
with:
|
||||
package_name: crawlee
|
||||
dist_dir: dist
|
||||
python_version: "3.14"
|
||||
extras: all
|
||||
smoke_code: |
|
||||
from crawlee.crawlers import (
|
||||
HttpCrawler, BeautifulSoupCrawler, ParselCrawler,
|
||||
PlaywrightCrawler, AdaptivePlaywrightCrawler,
|
||||
)
|
||||
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
||||
from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient
|
||||
from crawlee import Request
|
||||
HttpCrawler()
|
||||
BeautifulSoupCrawler()
|
||||
ParselCrawler()
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
name: Tests
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
unit_tests:
|
||||
name: Unit tests
|
||||
uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main
|
||||
secrets: inherit
|
||||
with:
|
||||
python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]'
|
||||
operating_systems: '["ubuntu-latest", "windows-latest", "macos-latest"]'
|
||||
python_version_for_codecov: "3.14"
|
||||
operating_system_for_codecov: ubuntu-latest
|
||||
tests_concurrency: "8"
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
name: Beta release
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI, or dispatched from `on_master.yaml`
|
||||
# via the `apify/actions/execute-workflow` action for the automatic beta release on push to master.
|
||||
# Note: This workflow is intentionally NOT a reusable workflow (no `workflow_call`) because PyPI's
|
||||
# Trusted Publishing does not currently support reusable workflows.
|
||||
# See: https://docs.pypi.org/trusted-publishers/troubleshooting/#reusable-workflows-on-github
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: release
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
release_prepare:
|
||||
name: Release prepare
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version_number: ${{ steps.release_prepare.outputs.version_number }}
|
||||
changelog: ${{ steps.release_prepare.outputs.changelog }}
|
||||
steps:
|
||||
- uses: apify/actions/git-cliff-release@v1.1.2
|
||||
id: release_prepare
|
||||
name: Release prepare
|
||||
with:
|
||||
release_type: prerelease
|
||||
existing_changelog_path: CHANGELOG.md
|
||||
|
||||
changelog_update:
|
||||
name: Changelog update
|
||||
needs: [release_prepare]
|
||||
permissions:
|
||||
contents: write
|
||||
uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
|
||||
with:
|
||||
version_number: ${{ needs.release_prepare.outputs.version_number }}
|
||||
changelog: ${{ needs.release_prepare.outputs.changelog }}
|
||||
secrets: inherit
|
||||
|
||||
pypi_publish:
|
||||
name: PyPI publish
|
||||
needs: [release_prepare, changelog_update]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write # Required for OIDC authentication.
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/crawlee
|
||||
steps:
|
||||
- name: Prepare distribution
|
||||
uses: apify/actions/prepare-pypi-distribution@v1.1.2
|
||||
with:
|
||||
package_name: crawlee
|
||||
is_prerelease: "yes"
|
||||
version_number: ${{ needs.release_prepare.outputs.version_number }}
|
||||
ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
|
||||
|
||||
- name: Verify built package
|
||||
uses: apify/actions/python-package-check@v1.1.0
|
||||
with:
|
||||
package_name: crawlee
|
||||
dist_dir: dist
|
||||
python_version: "3.14"
|
||||
extras: all
|
||||
smoke_code: |
|
||||
from crawlee.crawlers import (
|
||||
HttpCrawler, BeautifulSoupCrawler, ParselCrawler,
|
||||
PlaywrightCrawler, AdaptivePlaywrightCrawler,
|
||||
)
|
||||
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
||||
from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient
|
||||
from crawlee import Request
|
||||
HttpCrawler()
|
||||
BeautifulSoupCrawler()
|
||||
ParselCrawler()
|
||||
|
||||
# Publish the package to PyPI using PyPA official GitHub action with OIDC authentication.
|
||||
- name: Publish package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
doc_release_post_publish:
|
||||
name: Doc release post publish
|
||||
needs: [changelog_update, pypi_publish]
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/manual_release_docs.yaml
|
||||
secrets: inherit
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
name: Release docs
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
NODE_VERSION: 22
|
||||
PYTHON_VERSION: 3.14
|
||||
|
||||
jobs:
|
||||
release_docs:
|
||||
name: Release docs
|
||||
environment:
|
||||
name: github-pages
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Set up uv package manager
|
||||
uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: uv run poe install-dev
|
||||
|
||||
- name: Install pnpm and website dependencies
|
||||
uses: apify/actions/pnpm-install@v1.1.2
|
||||
with:
|
||||
working-directory: website
|
||||
|
||||
- name: Build Docusaurus docs
|
||||
run: uv run poe build-docs
|
||||
env:
|
||||
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
|
||||
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
|
||||
|
||||
- name: Set up GitHub Pages
|
||||
uses: actions/configure-pages@v6
|
||||
|
||||
- name: Upload GitHub Pages artifact
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
path: ./website/build
|
||||
|
||||
- name: Deploy artifact to GitHub Pages
|
||||
uses: actions/deploy-pages@v5
|
||||
|
||||
- name: Invalidate CloudFront cache
|
||||
run: |
|
||||
gh workflow run invalidate-cloudfront.yml \
|
||||
--repo apify/apify-docs-private \
|
||||
--field deployment=crawlee-web
|
||||
echo "✅ CloudFront cache invalidation workflow triggered successfully"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
name: Stable release
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI, with options to specify the type of release.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_type:
|
||||
description: Release type
|
||||
required: true
|
||||
type: choice
|
||||
default: auto
|
||||
options:
|
||||
- auto
|
||||
- custom
|
||||
- patch
|
||||
- minor
|
||||
- major
|
||||
custom_version:
|
||||
description: The custom version to bump to (only for "custom" type)
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
concurrency:
|
||||
group: release
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
code_checks:
|
||||
name: Code checks
|
||||
uses: ./.github/workflows/_check_code.yaml
|
||||
|
||||
release_prepare:
|
||||
name: Release prepare
|
||||
needs: [code_checks]
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version_number: ${{ steps.release_prepare.outputs.version_number }}
|
||||
tag_name: ${{ steps.release_prepare.outputs.tag_name }}
|
||||
changelog: ${{ steps.release_prepare.outputs.changelog }}
|
||||
release_notes: ${{ steps.release_prepare.outputs.release_notes }}
|
||||
steps:
|
||||
- uses: apify/actions/git-cliff-release@v1.1.2
|
||||
name: Release prepare
|
||||
id: release_prepare
|
||||
with:
|
||||
release_type: ${{ inputs.release_type }}
|
||||
custom_version: ${{ inputs.custom_version }}
|
||||
existing_changelog_path: CHANGELOG.md
|
||||
|
||||
changelog_update:
|
||||
name: Changelog update
|
||||
needs: [release_prepare]
|
||||
permissions:
|
||||
contents: write
|
||||
uses: apify/workflows/.github/workflows/python_bump_and_update_changelog.yaml@main
|
||||
with:
|
||||
version_number: ${{ needs.release_prepare.outputs.version_number }}
|
||||
changelog: ${{ needs.release_prepare.outputs.changelog }}
|
||||
secrets: inherit
|
||||
|
||||
github_release:
|
||||
name: GitHub release
|
||||
needs: [release_prepare, changelog_update]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- name: GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ needs.release_prepare.outputs.tag_name }}
|
||||
name: ${{ needs.release_prepare.outputs.version_number }}
|
||||
target_commitish: ${{ needs.changelog_update.outputs.changelog_commitish }}
|
||||
body: ${{ needs.release_prepare.outputs.release_notes }}
|
||||
|
||||
pypi_publish:
|
||||
name: PyPI publish
|
||||
needs: [release_prepare, changelog_update]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write # Required for OIDC authentication.
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/project/crawlee
|
||||
steps:
|
||||
- name: Prepare distribution
|
||||
uses: apify/actions/prepare-pypi-distribution@v1.1.2
|
||||
with:
|
||||
package_name: crawlee
|
||||
is_prerelease: ""
|
||||
version_number: ${{ needs.release_prepare.outputs.version_number }}
|
||||
ref: ${{ needs.changelog_update.outputs.changelog_commitish }}
|
||||
|
||||
- name: Verify built package
|
||||
uses: apify/actions/python-package-check@v1.1.0
|
||||
with:
|
||||
package_name: crawlee
|
||||
dist_dir: dist
|
||||
python_version: "3.14"
|
||||
extras: all
|
||||
smoke_code: |
|
||||
from crawlee.crawlers import (
|
||||
HttpCrawler, BeautifulSoupCrawler, ParselCrawler,
|
||||
PlaywrightCrawler, AdaptivePlaywrightCrawler,
|
||||
)
|
||||
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
|
||||
from crawlee.http_clients import HttpxHttpClient, ImpitHttpClient
|
||||
from crawlee import Request
|
||||
HttpCrawler()
|
||||
BeautifulSoupCrawler()
|
||||
ParselCrawler()
|
||||
|
||||
# Publish the package to PyPI using PyPA official GitHub action with OIDC authentication.
|
||||
- name: Publish package to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
|
||||
# TODO: add job for publish package to Conda
|
||||
# https://github.com/apify/crawlee-python/issues/104
|
||||
|
||||
version_docs:
|
||||
name: Version docs
|
||||
needs: [release_prepare, changelog_update, pypi_publish]
|
||||
permissions:
|
||||
contents: write
|
||||
uses: ./.github/workflows/manual_version_docs.yaml
|
||||
with:
|
||||
# Pass the bumped version explicitly — the job's checkout uses the dispatch ref (pre-bump),
|
||||
# so `uv version --short` from pyproject.toml would return the old version.
|
||||
version_number: ${{ needs.release_prepare.outputs.version_number }}
|
||||
secrets: inherit
|
||||
|
||||
doc_release:
|
||||
name: Doc release
|
||||
needs: [changelog_update, pypi_publish, version_docs]
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/manual_release_docs.yaml
|
||||
secrets: inherit
|
||||
|
|
@ -0,0 +1,121 @@
|
|||
name: Version docs
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version_number:
|
||||
description: Version to snapshot (e.g. "1.0.0"). If empty, the current version in pyproject.toml is used.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
# Runs when invoked by another workflow.
|
||||
workflow_call:
|
||||
inputs:
|
||||
version_number:
|
||||
description: Version to snapshot (e.g. "1.0.0"). If empty, the current version in pyproject.toml is used.
|
||||
required: false
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
concurrency:
|
||||
group: version-docs
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
NODE_VERSION: "22"
|
||||
PYTHON_VERSION: "3.14"
|
||||
|
||||
jobs:
|
||||
version_docs:
|
||||
name: Version docs
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Set up uv package manager
|
||||
uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: uv run poe install-dev
|
||||
|
||||
- name: Install pnpm and website dependencies
|
||||
uses: apify/actions/pnpm-install@v1.1.2
|
||||
with:
|
||||
working-directory: website
|
||||
|
||||
- name: Snapshot the current version
|
||||
id: snapshot
|
||||
env:
|
||||
INPUT_VERSION: ${{ inputs.version_number }}
|
||||
run: |
|
||||
cd website
|
||||
|
||||
# Prefer the explicit input (passed by the release workflow after the version bump).
|
||||
# Fall back to pyproject.toml only when run manually without an input — this avoids
|
||||
# the stale-checkout pitfall where the bumped version isn't visible to this job.
|
||||
if [[ -n "$INPUT_VERSION" ]]; then
|
||||
FULL_VERSION="$INPUT_VERSION"
|
||||
else
|
||||
FULL_VERSION="$(uv version --short)"
|
||||
fi
|
||||
MAJOR_MINOR_VERSION="$(echo "$FULL_VERSION" | cut -d. -f1-2)"
|
||||
MAJOR_VERSION="$(echo "$FULL_VERSION" | cut -d. -f1)"
|
||||
echo "version=$FULL_VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Version: $FULL_VERSION, Major.Minor: $MAJOR_MINOR_VERSION, Major: $MAJOR_VERSION"
|
||||
|
||||
# Find the existing versions for this major in versions.json (if any).
|
||||
if [[ -f versions.json ]]; then
|
||||
OLD_VERSIONS="$(jq -r --arg major "$MAJOR_VERSION" '.[] | select(startswith($major + "."))' versions.json)"
|
||||
else
|
||||
OLD_VERSIONS=""
|
||||
echo "[]" > versions.json
|
||||
fi
|
||||
|
||||
# Remove all old versions for this major (if found).
|
||||
if [[ -n "$OLD_VERSIONS" ]]; then
|
||||
while IFS= read -r OLD_VERSION; do
|
||||
[[ -z "$OLD_VERSION" ]] && continue
|
||||
echo "Removing old version $OLD_VERSION for major $MAJOR_VERSION"
|
||||
rm -rf "versioned_docs/version-${OLD_VERSION}"
|
||||
rm -f "versioned_sidebars/version-${OLD_VERSION}-sidebars.json"
|
||||
done <<< "$OLD_VERSIONS"
|
||||
jq --arg major "$MAJOR_VERSION" 'map(select(startswith($major + ".") | not))' versions.json > tmp.json && mv tmp.json versions.json
|
||||
else
|
||||
echo "No existing versions found for major $MAJOR_VERSION, nothing to remove"
|
||||
fi
|
||||
|
||||
# Build API reference and create Docusaurus version snapshots.
|
||||
bash build_api_reference.sh
|
||||
uv run pnpm exec docusaurus docs:version "$MAJOR_MINOR_VERSION"
|
||||
uv run pnpm exec docusaurus api:version "$MAJOR_MINOR_VERSION"
|
||||
|
||||
- name: Commit and push versioned docs
|
||||
uses: apify/actions/signed-commit@v1.1.2
|
||||
with:
|
||||
message: "docs: Version docs for v${{ steps.snapshot.outputs.version }} [skip ci]"
|
||||
add: 'website/versioned_docs website/versioned_sidebars website/versions.json'
|
||||
pull: '--rebase --autostash'
|
||||
github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
name: CI (issue)
|
||||
|
||||
on:
|
||||
# Runs when a new issue is opened.
|
||||
issues:
|
||||
types:
|
||||
- opened
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
label_issues:
|
||||
name: Add labels
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
# Add the "t-tooling" label to all new issues
|
||||
- uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.addLabels({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
labels: ["t-tooling"]
|
||||
})
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
name: CI (master)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags-ignore:
|
||||
- "**" # Ignore all tags to avoid duplicate executions triggered by tag pushes.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
doc_checks:
|
||||
name: Doc checks
|
||||
uses: ./.github/workflows/_check_docs.yaml
|
||||
|
||||
doc_release:
|
||||
# Skip this for non-"docs" commits.
|
||||
if: startsWith(github.event.head_commit.message, 'docs')
|
||||
name: Doc release
|
||||
needs: [doc_checks]
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
uses: ./.github/workflows/manual_release_docs.yaml
|
||||
secrets: inherit
|
||||
|
||||
code_checks:
|
||||
name: Code checks
|
||||
uses: ./.github/workflows/_check_code.yaml
|
||||
|
||||
tests:
|
||||
# Skip this for "docs" commits.
|
||||
if: "!startsWith(github.event.head_commit.message, 'docs')"
|
||||
name: Tests
|
||||
uses: ./.github/workflows/_tests.yaml
|
||||
secrets: inherit
|
||||
|
||||
# The beta release is dispatched as a separate workflow run (instead of calling `manual_release_beta.yaml` via `uses:`)
|
||||
# because PyPI's Trusted Publishing does not currently support reusable workflows.
|
||||
# See: https://docs.pypi.org/trusted-publishers/troubleshooting/#reusable-workflows-on-github
|
||||
beta_release:
|
||||
# Run this only for "feat", "fix", "perf", "refactor" and "style" commits.
|
||||
if: >-
|
||||
startsWith(github.event.head_commit.message, 'feat') ||
|
||||
startsWith(github.event.head_commit.message, 'fix') ||
|
||||
startsWith(github.event.head_commit.message, 'perf') ||
|
||||
startsWith(github.event.head_commit.message, 'refactor') ||
|
||||
startsWith(github.event.head_commit.message, 'style')
|
||||
name: Beta release
|
||||
needs: [code_checks, tests]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: write # Required by execute-workflow.
|
||||
steps:
|
||||
- name: Dispatch beta release workflow
|
||||
uses: apify/actions/execute-workflow@v1.1.2
|
||||
with:
|
||||
workflow: manual_release_beta.yaml
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
name: CI (PR)
|
||||
|
||||
on:
|
||||
# Runs whenever a pull request is opened or updated.
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
pr_title_check:
|
||||
name: PR title check
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: amannn/action-semantic-pull-request@v6.1.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
doc_checks:
|
||||
name: Doc checks
|
||||
uses: ./.github/workflows/_check_docs.yaml
|
||||
|
||||
code_checks:
|
||||
name: Code checks
|
||||
uses: ./.github/workflows/_check_code.yaml
|
||||
|
||||
package_check:
|
||||
name: Package check
|
||||
uses: ./.github/workflows/_check_package.yaml
|
||||
|
||||
tests:
|
||||
name: Tests
|
||||
uses: ./.github/workflows/_tests.yaml
|
||||
secrets: inherit
|
||||
|
|
@ -0,0 +1,144 @@
|
|||
name: Scheduled tests
|
||||
|
||||
on:
|
||||
# Runs when manually triggered from the GitHub UI.
|
||||
workflow_dispatch:
|
||||
|
||||
# Runs on weekdays at 01:00 UTC.
|
||||
schedule:
|
||||
- cron: '0 1 * * 1-5'
|
||||
|
||||
concurrency:
|
||||
group: scheduled-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
NODE_VERSION: 22
|
||||
PYTHON_VERSION: 3.14
|
||||
TESTS_CONCURRENCY: 1
|
||||
|
||||
jobs:
|
||||
end_to_end_tests:
|
||||
name: End-to-end tests
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 12
|
||||
matrix:
|
||||
crawler-type: ["playwright_camoufox", "playwright_chrome", "playwright_firefox", "playwright_webkit", "playwright", "parsel", "beautifulsoup", "adaptive_beautifulsoup", "adaptive_parsel", "stagehand"]
|
||||
http-client: ["httpx", "curl_impersonate", "impit"]
|
||||
package-manager: ["pip", "uv", "poetry"]
|
||||
|
||||
runs-on: "ubuntu-latest"
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
|
||||
- name: Install Apify CLI
|
||||
uses: apify/setup-apify-cli-action@v1.0.0
|
||||
with:
|
||||
token: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}
|
||||
|
||||
- name: Set up Python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
# installed to be able to patch crawlee in the poetry.lock with custom wheel file for poetry based templates
|
||||
- name: Install poetry
|
||||
run: pipx install poetry
|
||||
|
||||
- name: Set up uv package manager
|
||||
uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
# Sync the project, but no need to install the browsers into the test runner environment.
|
||||
- name: Install Python dependencies
|
||||
run: uv run poe install-sync
|
||||
|
||||
- name: Run templates end-to-end tests
|
||||
run: uv run poe e2e-templates-tests -m "${{ matrix.http-client }} and ${{ matrix.crawler-type }} and ${{ matrix.package-manager }}"
|
||||
env:
|
||||
APIFY_TEST_USER_API_TOKEN: ${{ secrets.APIFY_TEST_USER_API_TOKEN }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
|
||||
# Send a Slack notification to the team alerting channel when scheduled e2e tests fail.
|
||||
# Skipped on workflow_dispatch (manual runs) so that ad-hoc triggers don't spam the channel.
|
||||
notify_on_failure:
|
||||
name: Notify Slack on failure
|
||||
needs: end_to_end_tests
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
steps:
|
||||
- name: Build Slack payload
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
RUN_ATTEMPT: ${{ github.run_attempt }}
|
||||
WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
HEADING: ':red_circle: Scheduled e2e tests failed'
|
||||
run: |
|
||||
# Retry the API call to tolerate transient 5xx from GitHub.
|
||||
max_attempts=5
|
||||
fetched=0
|
||||
for attempt in $(seq 1 "${max_attempts}"); do
|
||||
if failed_jobs=$(gh api \
|
||||
"repos/${REPO}/actions/runs/${RUN_ID}/attempts/${RUN_ATTEMPT}/jobs?per_page=100" \
|
||||
--jq '[.jobs[] | select(.conclusion == "failure") | "• \(.name)"] | join("\n")'); then
|
||||
fetched=1
|
||||
break
|
||||
fi
|
||||
if [[ "${attempt}" -lt "${max_attempts}" ]]; then
|
||||
sleep "$((attempt * 5))"
|
||||
fi
|
||||
done
|
||||
if [[ "${fetched}" -eq 0 ]]; then
|
||||
echo "Failed to fetch job list after ${max_attempts} attempts; sending notification without it." >&2
|
||||
failed_jobs="(unable to fetch job list — see workflow run)"
|
||||
fi
|
||||
jq -n \
|
||||
--arg repo "${REPO}" \
|
||||
--arg url "${WORKFLOW_URL}" \
|
||||
--arg heading "${HEADING}" \
|
||||
--arg failed "${failed_jobs}" \
|
||||
'{
|
||||
text: "\($heading) in \($repo)",
|
||||
blocks: [
|
||||
{
|
||||
type: "header",
|
||||
text: { type: "plain_text", text: $heading, emoji: true }
|
||||
},
|
||||
{
|
||||
type: "section",
|
||||
fields: [
|
||||
{ type: "mrkdwn", text: "*Repository:*\n\($repo)" },
|
||||
{ type: "mrkdwn", text: "*Workflow run:*\n<\($url)|View on GitHub>" }
|
||||
]
|
||||
},
|
||||
{
|
||||
type: "section",
|
||||
text: { type: "mrkdwn", text: "*Failed jobs:*\n\($failed)" }
|
||||
}
|
||||
]
|
||||
}' > slack-payload.json
|
||||
|
||||
- name: Send Slack notification
|
||||
uses: slackapi/slack-github-action@v3.0.2
|
||||
with:
|
||||
webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload-file-path: slack-payload.json
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
# AI assistant files
|
||||
.agent
|
||||
.agents
|
||||
.ai
|
||||
.aider
|
||||
.claude
|
||||
.codeium
|
||||
.continue
|
||||
.copilot
|
||||
.cursor
|
||||
.gemini
|
||||
.llm
|
||||
.llms
|
||||
.openai
|
||||
.serena
|
||||
.windsurf
|
||||
.zed-ai
|
||||
AGENTS.local.md
|
||||
CLAUDE.local.md
|
||||
GEMINI.local.md
|
||||
|
||||
# Cache
|
||||
__pycache__
|
||||
.pytest_cache
|
||||
.pytest-tmp
|
||||
.ruff_cache
|
||||
.ty_cache
|
||||
.uv-cache
|
||||
|
||||
# Virtual envs
|
||||
.direnv
|
||||
.env
|
||||
.envrc
|
||||
.python-version
|
||||
.venv
|
||||
|
||||
# Other Python tools
|
||||
.ropeproject
|
||||
|
||||
# Mise
|
||||
mise.toml
|
||||
.mise.toml
|
||||
|
||||
# Egg and build artifacts
|
||||
*.egg-info/
|
||||
*.egg
|
||||
dist/
|
||||
build/
|
||||
|
||||
# Coverage reports
|
||||
.coverage*
|
||||
htmlcov
|
||||
coverage-unit.xml
|
||||
coverage-integration.xml
|
||||
|
||||
# IDE, editors
|
||||
*~
|
||||
.DS_Store
|
||||
.idea
|
||||
.nvim.lua
|
||||
.vscode
|
||||
.zed
|
||||
Session.vim
|
||||
|
||||
# Docs
|
||||
docs/changelog.md
|
||||
website/versioned_docs/*/changelog.md
|
||||
website/versioned_docs/*/pyproject.toml
|
||||
|
||||
# Website build artifacts, node dependencies
|
||||
website/build
|
||||
website/node_modules
|
||||
website/yarn.lock
|
||||
website/.yarn
|
||||
website/.docusaurus
|
||||
website/api-typedoc-generated.json
|
||||
website/apify-shared-docspec-dump.jsonl
|
||||
website/docspec-dump.jsonl
|
||||
website/module_shortcuts.json
|
||||
website/typedoc-types*
|
||||
# npm lockfile (we use pnpm)
|
||||
website/package-lock.json
|
||||
|
||||
# Default directory for memory storage
|
||||
storage/
|
||||
|
||||
# Tmp dir
|
||||
tmp/
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
default: true
|
||||
line-length:
|
||||
line_length: 120
|
||||
MD007:
|
||||
indent: 4
|
||||
MD004:
|
||||
style: dash
|
||||
no-inline-html: false
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: lint-check
|
||||
name: Lint check
|
||||
entry: uv run poe lint
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
||||
- id: type-check
|
||||
name: Type check
|
||||
entry: uv run poe type-check
|
||||
language: system
|
||||
pass_filenames: false
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
# Coding guidelines
|
||||
|
||||
This file provides guidance to programming agents when working with code in this repository.
|
||||
|
||||
## Development Commands
|
||||
|
||||
All commands use `uv` (package manager) and `poe` (task runner):
|
||||
|
||||
```bash
|
||||
# Install all dependencies (dev + extras + pre-commit + playwright)
|
||||
uv run poe install-dev
|
||||
|
||||
# Run full check suite (lint + type-check + unit tests)
|
||||
uv run poe check-code
|
||||
|
||||
# Linting (ruff format check + ruff check)
|
||||
uv run poe lint
|
||||
|
||||
# Auto-fix formatting
|
||||
uv run poe format
|
||||
|
||||
# Type checking (ty)
|
||||
uv run poe type-check
|
||||
|
||||
# Run all unit tests
|
||||
uv run poe unit-tests
|
||||
|
||||
# Run a single test file
|
||||
uv run pytest tests/unit/path/to/test_file.py
|
||||
|
||||
# Run a single test by name
|
||||
uv run pytest tests/unit/path/to/test_file.py::test_name -v
|
||||
|
||||
# Run tests with coverage XML report
|
||||
uv run poe unit-tests-cov
|
||||
|
||||
# Build package
|
||||
uv run poe build
|
||||
|
||||
# Clean build artifacts
|
||||
uv run poe clean
|
||||
```
|
||||
|
||||
Note: `uv run poe unit-tests` first runs tests marked `@pytest.mark.run_alone` in isolation, then runs the rest with `-x` (fail-fast) and parallelism via `pytest-xdist`.
|
||||
|
||||
## Code Style
|
||||
|
||||
- **Linter/formatter**: Ruff with `select = ["ALL"]` and specific ignores
|
||||
- **Line length**: 120 characters
|
||||
- **Quotes**: Single quotes (double for docstrings)
|
||||
- **Docstrings**: Google format (enforced by Ruff)
|
||||
- **Type checker**: ty (Astral's type checker), target Python 3.10
|
||||
- **Async mode**: pytest-asyncio in `auto` mode (no need for `@pytest.mark.asyncio`)
|
||||
- **Commits**: [Conventional Commits](https://www.conventionalcommits.org/) format. Choose the type based on *what* changed, not just *why*:
|
||||
- `feat:` / `fix:` / `perf:` / `refactor:` / `style:` — **source code only**; these trigger a release and appear in the changelog
|
||||
- `test:` — test additions or changes (no release triggered)
|
||||
- `docs:` — documentation changes; also triggers a doc release on master
|
||||
- `ci:` — CI/workflow changes
|
||||
- `chore:` — dependency bumps, tooling, and other housekeeping
|
||||
- `build:` — build system changes
|
||||
|
||||
## Architecture
|
||||
|
||||
### Crawler Hierarchy
|
||||
|
||||
```
|
||||
BasicCrawler[TCrawlingContext, TStatisticsState]
|
||||
├── AbstractHttpCrawler → HttpCrawler, BeautifulSoupCrawler, ParselCrawler
|
||||
├── PlaywrightCrawler
|
||||
└── AdaptivePlaywrightCrawler (extends PlaywrightCrawler)
|
||||
```
|
||||
|
||||
- **BasicCrawler** (`src/crawlee/crawlers/_basic/`): Core request lifecycle, autoscaling pool, retries, session management, router dispatch. Generic over `TCrawlingContext`.
|
||||
- **AbstractHttpCrawler** (`src/crawlee/crawlers/_abstract_http/`): Adds HTTP client integration, response parsing, pre-navigation hooks. Generic over parser result type.
|
||||
- **PlaywrightCrawler** (`src/crawlee/crawlers/_playwright/`): Browser-based crawling with Playwright.
|
||||
|
||||
### Context Pipeline (Middleware Pattern)
|
||||
|
||||
Contexts are progressively enhanced through `ContextPipeline` middleware:
|
||||
|
||||
```
|
||||
BasicCrawlingContext → HttpCrawlingContext → ParsedHttpCrawlingContext → BeautifulSoupCrawlingContext
|
||||
```
|
||||
|
||||
Each middleware is an async generator that wraps the next handler, enabling setup/teardown around request processing.
|
||||
|
||||
### Storage Layer
|
||||
|
||||
Three-tier design:
|
||||
- **High-level**: `Dataset`, `KeyValueStore`, `RequestQueue` in `src/crawlee/storages/`
|
||||
- **Storage clients** (`src/crawlee/storage_clients/`): `FileSystemStorageClient` (default), `MemoryStorageClient`, `SqlStorageClient`, `RedisStorageClient`
|
||||
- **Instance caching**: `StorageInstanceManager` is a global singleton that caches storage instances by ID/name
|
||||
|
||||
### Service Locator
|
||||
|
||||
`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageClient`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`.
|
||||
|
||||
### HTTP Clients
|
||||
|
||||
Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`:
|
||||
- `ImpitHttpClient` (default), `HttpxHttpClient`, `CurlImpersonateHttpClient`
|
||||
- Each provides `crawl()` (for crawler pipeline) and `send_request()` (for in-handler use)
|
||||
|
||||
### Request Model
|
||||
|
||||
`Request` (`src/crawlee/_request.py`) uses `unique_key` for deduplication. Lifecycle states: `UNPROCESSED → DONE`. Crawlee-specific metadata stored in `user_data['__crawlee']`.
|
||||
|
||||
### Router
|
||||
|
||||
```python
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BeautifulSoupCrawlingContext): ...
|
||||
|
||||
@crawler.router.handler(label='detail')
|
||||
async def detail(context: BeautifulSoupCrawlingContext): ...
|
||||
```
|
||||
|
||||
Requests are routed by their `label` field; unmatched requests go to the default handler.
|
||||
|
||||
### Key Directories
|
||||
|
||||
- `src/crawlee/crawlers/` - All crawler implementations
|
||||
- `src/crawlee/storages/` - Dataset, KVS, RequestQueue
|
||||
- `src/crawlee/storage_clients/` - Backend implementations
|
||||
- `src/crawlee/http_clients/` - HTTP client implementations
|
||||
- `src/crawlee/browsers/` - Playwright browser pool and plugins
|
||||
- `src/crawlee/sessions/` - Session management with cookie persistence
|
||||
- `src/crawlee/events/` - Event system (persist state, progress, aborting)
|
||||
- `src/crawlee/_autoscaling/` - Autoscaled pool for concurrency control
|
||||
- `src/crawlee/fingerprint_suite/` - Anti-bot fingerprint generation
|
||||
- `src/crawlee/project_template/` - CLI scaffolding template (excluded from linting)
|
||||
- `tests/unit/` - Unit tests
|
||||
- `tests/e2e/` - End-to-end tests (require `apify-cli` + API token)
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
# Coding guidelines
|
||||
|
||||
This file provides guidance to programming agents when working with code in this repository.
|
||||
|
||||
## Development Commands
|
||||
|
||||
All commands use `uv` (package manager) and `poe` (task runner):
|
||||
|
||||
```bash
|
||||
# Install all dependencies (dev + extras + pre-commit + playwright)
|
||||
uv run poe install-dev
|
||||
|
||||
# Run full check suite (lint + type-check + unit tests)
|
||||
uv run poe check-code
|
||||
|
||||
# Linting (ruff format check + ruff check)
|
||||
uv run poe lint
|
||||
|
||||
# Auto-fix formatting
|
||||
uv run poe format
|
||||
|
||||
# Type checking (ty)
|
||||
uv run poe type-check
|
||||
|
||||
# Run all unit tests
|
||||
uv run poe unit-tests
|
||||
|
||||
# Run a single test file
|
||||
uv run pytest tests/unit/path/to/test_file.py
|
||||
|
||||
# Run a single test by name
|
||||
uv run pytest tests/unit/path/to/test_file.py::test_name -v
|
||||
|
||||
# Run tests with coverage XML report
|
||||
uv run poe unit-tests-cov
|
||||
|
||||
# Build package
|
||||
uv run poe build
|
||||
|
||||
# Clean build artifacts
|
||||
uv run poe clean
|
||||
```
|
||||
|
||||
Note: `uv run poe unit-tests` first runs tests marked `@pytest.mark.run_alone` in isolation, then runs the rest with `-x` (fail-fast) and parallelism via `pytest-xdist`.
|
||||
|
||||
## Code Style
|
||||
|
||||
- **Linter/formatter**: Ruff with `select = ["ALL"]` and specific ignores
|
||||
- **Line length**: 120 characters
|
||||
- **Quotes**: Single quotes (double for docstrings)
|
||||
- **Docstrings**: Google format (enforced by Ruff)
|
||||
- **Type checker**: ty (Astral's type checker), target Python 3.10
|
||||
- **Async mode**: pytest-asyncio in `auto` mode (no need for `@pytest.mark.asyncio`)
|
||||
- **Commits**: [Conventional Commits](https://www.conventionalcommits.org/) format. Choose the type based on *what* changed, not just *why*:
|
||||
- `feat:` / `fix:` / `perf:` / `refactor:` / `style:` — **source code only**; these trigger a release and appear in the changelog
|
||||
- `test:` — test additions or changes (no release triggered)
|
||||
- `docs:` — documentation changes; also triggers a doc release on master
|
||||
- `ci:` — CI/workflow changes
|
||||
- `chore:` — dependency bumps, tooling, and other housekeeping
|
||||
- `build:` — build system changes
|
||||
|
||||
## Architecture
|
||||
|
||||
### Crawler Hierarchy
|
||||
|
||||
```
|
||||
BasicCrawler[TCrawlingContext, TStatisticsState]
|
||||
├── AbstractHttpCrawler → HttpCrawler, BeautifulSoupCrawler, ParselCrawler
|
||||
├── PlaywrightCrawler
|
||||
└── AdaptivePlaywrightCrawler (extends PlaywrightCrawler)
|
||||
```
|
||||
|
||||
- **BasicCrawler** (`src/crawlee/crawlers/_basic/`): Core request lifecycle, autoscaling pool, retries, session management, router dispatch. Generic over `TCrawlingContext`.
|
||||
- **AbstractHttpCrawler** (`src/crawlee/crawlers/_abstract_http/`): Adds HTTP client integration, response parsing, pre-navigation hooks. Generic over parser result type.
|
||||
- **PlaywrightCrawler** (`src/crawlee/crawlers/_playwright/`): Browser-based crawling with Playwright.
|
||||
|
||||
### Context Pipeline (Middleware Pattern)
|
||||
|
||||
Contexts are progressively enhanced through `ContextPipeline` middleware:
|
||||
|
||||
```
|
||||
BasicCrawlingContext → HttpCrawlingContext → ParsedHttpCrawlingContext → BeautifulSoupCrawlingContext
|
||||
```
|
||||
|
||||
Each middleware is an async generator that wraps the next handler, enabling setup/teardown around request processing.
|
||||
|
||||
### Storage Layer
|
||||
|
||||
Three-tier design:
|
||||
- **High-level**: `Dataset`, `KeyValueStore`, `RequestQueue` in `src/crawlee/storages/`
|
||||
- **Storage clients** (`src/crawlee/storage_clients/`): `FileSystemStorageClient` (default), `MemoryStorageClient`, `SqlStorageClient`, `RedisStorageClient`
|
||||
- **Instance caching**: `StorageInstanceManager` is a global singleton that caches storage instances by ID/name
|
||||
|
||||
### Service Locator
|
||||
|
||||
`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageClient`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`.
|
||||
|
||||
### HTTP Clients
|
||||
|
||||
Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`:
|
||||
- `ImpitHttpClient` (default), `HttpxHttpClient`, `CurlImpersonateHttpClient`
|
||||
- Each provides `crawl()` (for crawler pipeline) and `send_request()` (for in-handler use)
|
||||
|
||||
### Request Model
|
||||
|
||||
`Request` (`src/crawlee/_request.py`) uses `unique_key` for deduplication. Lifecycle states: `UNPROCESSED → DONE`. Crawlee-specific metadata stored in `user_data['__crawlee']`.
|
||||
|
||||
### Router
|
||||
|
||||
```python
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BeautifulSoupCrawlingContext): ...
|
||||
|
||||
@crawler.router.handler(label='detail')
|
||||
async def detail(context: BeautifulSoupCrawlingContext): ...
|
||||
```
|
||||
|
||||
Requests are routed by their `label` field; unmatched requests go to the default handler.
|
||||
|
||||
### Key Directories
|
||||
|
||||
- `src/crawlee/crawlers/` - All crawler implementations
|
||||
- `src/crawlee/storages/` - Dataset, KVS, RequestQueue
|
||||
- `src/crawlee/storage_clients/` - Backend implementations
|
||||
- `src/crawlee/http_clients/` - HTTP client implementations
|
||||
- `src/crawlee/browsers/` - Playwright browser pool and plugins
|
||||
- `src/crawlee/sessions/` - Session management with cookie persistence
|
||||
- `src/crawlee/events/` - Event system (persist state, progress, aborting)
|
||||
- `src/crawlee/_autoscaling/` - Autoscaled pool for concurrency control
|
||||
- `src/crawlee/fingerprint_suite/` - Anti-bot fingerprint generation
|
||||
- `src/crawlee/project_template/` - CLI scaffolding template (excluded from linting)
|
||||
- `tests/unit/` - Unit tests
|
||||
- `tests/e2e/` - End-to-end tests (require `apify-cli` + API token)
|
||||
|
|
@ -0,0 +1,878 @@
|
|||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
<!-- git-cliff-unreleased-start -->
|
||||
## 1.7.1 - **not yet released**
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Include `sql_mysql` in the `all` extra ([#1895](https://github.com/apify/crawlee-python/pull/1895)) ([4023314](https://github.com/apify/crawlee-python/commit/4023314132b8942519fdee3795107d2169179423)) by [@vdusek](https://github.com/vdusek)
|
||||
- Update `push_data` and `user_data` annotation with `JsonSerializable` instead of `Any` ([#1889](https://github.com/apify/crawlee-python/pull/1889)) ([662b93b](https://github.com/apify/crawlee-python/commit/662b93b2e6764396ba885d7f1a57c0dba42369a1)) by [@Mantisus](https://github.com/Mantisus), closes [#1191](https://github.com/apify/crawlee-python/issues/1191)
|
||||
- **stagehand:** Inject `--no-sandbox` into Stagehand's Chromium launch when sandbox is disabled ([#1906](https://github.com/apify/crawlee-python/pull/1906)) ([041b92a](https://github.com/apify/crawlee-python/commit/041b92a1cd671eabd7629dbcdba2d5cc30ff1837)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
<!-- git-cliff-unreleased-end -->
|
||||
## [1.7.0](https://github.com/apify/crawlee-python/releases/tag/v1.7.0) (2026-05-12)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `use` to `Router` for middleware support with pre-handler execution ([#1857](https://github.com/apify/crawlee-python/pull/1857)) ([23d7d6c](https://github.com/apify/crawlee-python/commit/23d7d6c5865a05b75bc6c68490e3382d876cde64)) by [@Mantisus](https://github.com/Mantisus), closes [#1742](https://github.com/apify/crawlee-python/issues/1742)
|
||||
- Add opt-in per-domain request throttling for HTTP 429 backoff ([#1762](https://github.com/apify/crawlee-python/pull/1762)) ([c17f4d5](https://github.com/apify/crawlee-python/commit/c17f4d52883763519776d9296b71457b6d3063f0)) by [@MrAliHasan](https://github.com/MrAliHasan), closes [#1437](https://github.com/apify/crawlee-python/issues/1437)
|
||||
- Add pre/post launch hooks to `BrowserPool` ([#1879](https://github.com/apify/crawlee-python/pull/1879)) ([00ffb7e](https://github.com/apify/crawlee-python/commit/00ffb7e52bed73bc4da7ea34102d589741a3fdf3)) by [@Mantisus](https://github.com/Mantisus), closes [#1741](https://github.com/apify/crawlee-python/issues/1741)
|
||||
- Add `StagehandCrawler` with AI-powered browser automation ([#1854](https://github.com/apify/crawlee-python/pull/1854)) ([da84db1](https://github.com/apify/crawlee-python/commit/da84db1282b613ccb2fb205e2f43dfb5a73fea8e)) by [@Mantisus](https://github.com/Mantisus), closes [#1738](https://github.com/apify/crawlee-python/issues/1738)
|
||||
- **cli:** Add Adaptive and Stagehand crawler templates ([#1888](https://github.com/apify/crawlee-python/pull/1888)) ([39b2d24](https://github.com/apify/crawlee-python/commit/39b2d24fd29ffc6d144f937c5b070dd4a693b279)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Reject non-http(s) URL schemes in HTTP clients ([#1862](https://github.com/apify/crawlee-python/pull/1862)) ([ac66b2a](https://github.com/apify/crawlee-python/commit/ac66b2a4851a11db3a5943d85f7091f39b1053f4)) by [@vdusek](https://github.com/vdusek)
|
||||
- Filter sitemap-derived URLs by enqueue strategy ([#1864](https://github.com/apify/crawlee-python/pull/1864)) ([b3db0dc](https://github.com/apify/crawlee-python/commit/b3db0dccbcb679d9e67e7996a97ac2c6ed364456)) by [@vdusek](https://github.com/vdusek)
|
||||
- Bump `BrowserPool` default `operation_timeout` to 60 seconds ([#1877](https://github.com/apify/crawlee-python/pull/1877)) ([38e7dd2](https://github.com/apify/crawlee-python/commit/38e7dd209ed332b55aff4da29859089e6e453d59)) by [@vdusek](https://github.com/vdusek)
|
||||
- **redis:** Prevent counter corruption from concurrent mark handled in Redis RQ ([#1878](https://github.com/apify/crawlee-python/pull/1878)) ([50d70f0](https://github.com/apify/crawlee-python/commit/50d70f06402e76e676dca45e333f0d7580d47add)) by [@Mantisus](https://github.com/Mantisus), closes [#1873](https://github.com/apify/crawlee-python/issues/1873)
|
||||
- Fall back to drop+recreate when `RequestQueue.purge` is unsupported ([#1883](https://github.com/apify/crawlee-python/pull/1883)) ([cd15dce](https://github.com/apify/crawlee-python/commit/cd15dce37cf0ca9419625339e60da54f76aead7b)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [1.6.3](https://github.com/apify/crawlee-python/releases/tag/v1.6.3) (2026-04-27)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix potential deadlocks in `SitemapRequestLoader` and `RequestManagerTandem` ([#1843](https://github.com/apify/crawlee-python/pull/1843)) ([6226d93](https://github.com/apify/crawlee-python/commit/6226d93f4d25a63f3c88b0f6ec3d2c5431165197)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Add retry logic for `RedisStorageClient` and `SqlStorageClient` ([#1838](https://github.com/apify/crawlee-python/pull/1838)) ([b80f562](https://github.com/apify/crawlee-python/commit/b80f56291e1adaa8cc4bc0fb85ef0d6a3fa6c78b)) by [@Mantisus](https://github.com/Mantisus), closes [#1831](https://github.com/apify/crawlee-python/issues/1831)
|
||||
- Fix StorageInstanceManager cache eviction ([#1855](https://github.com/apify/crawlee-python/pull/1855)) ([983f14f](https://github.com/apify/crawlee-python/commit/983f14f1aee28c254e1ad49b98a4adb611741a4d)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Report integer count in 'Experiencing problems' status log ([#1860](https://github.com/apify/crawlee-python/pull/1860)) ([40170a6](https://github.com/apify/crawlee-python/commit/40170a67b37bd2bb2498d02b3068f849370b228b)) by [@vdusek](https://github.com/vdusek)
|
||||
- Preserve `forefront` flag on `RequestQueue` retry path ([#1861](https://github.com/apify/crawlee-python/pull/1861)) ([dc1073a](https://github.com/apify/crawlee-python/commit/dc1073a857b13ff246145dc4fe4ec09845972e0d)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [1.6.2](https://github.com/apify/crawlee-python/releases/tag/v1.6.2) (2026-04-08)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- **file-system:** Reclaim orphaned in-progress requests on RQ recovery ([#1825](https://github.com/apify/crawlee-python/pull/1825)) ([e86794a](https://github.com/apify/crawlee-python/commit/e86794a6e5605432c9331c7cd99edf885527a3eb)) by [@vdusek](https://github.com/vdusek)
|
||||
- Prevent premature `EventManager` shutdown when multiple crawlers share it ([#1810](https://github.com/apify/crawlee-python/pull/1810)) ([2efb668](https://github.com/apify/crawlee-python/commit/2efb668ad54fb3e8d740066446563d1e8a39d2e8)) by [@Mantisus](https://github.com/Mantisus), closes [#1805](https://github.com/apify/crawlee-python/issues/1805), [#1808](https://github.com/apify/crawlee-python/issues/1808)
|
||||
- Apply SQLite optimizations to the custom `connection_string` in `SqlStorageClient` ([#1837](https://github.com/apify/crawlee-python/pull/1837)) ([8b53e27](https://github.com/apify/crawlee-python/commit/8b53e273067e27b4ef4b2b4bb40277b15ef6b058)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Apply `SharedTimeout` to post-navigation hooks ([#1839](https://github.com/apify/crawlee-python/pull/1839)) ([88bd05a](https://github.com/apify/crawlee-python/commit/88bd05a2127ebfe3cd4eb78c514a63fc9e2cd079)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [1.6.1](https://github.com/apify/crawlee-python/releases/tag/v1.6.1) (2026-03-30)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Handle invalid URLs in `RequestList` ([#1803](https://github.com/apify/crawlee-python/pull/1803)) ([0b2e3fc](https://github.com/apify/crawlee-python/commit/0b2e3fc5cbca371131b54085e052a6cda6361b0f)) by [@Mantisus](https://github.com/Mantisus), closes [#1802](https://github.com/apify/crawlee-python/issues/1802)
|
||||
- **playwright:** Filter unsupported context options in persistent browser ([#1796](https://github.com/apify/crawlee-python/pull/1796)) ([69ad22e](https://github.com/apify/crawlee-python/commit/69ad22e60ef558d8c26e84e2bd165fe03f116b7f)) by [@sushant-mutnale](https://github.com/sushant-mutnale), closes [#1784](https://github.com/apify/crawlee-python/issues/1784)
|
||||
- Remove double usage_count increment in Session.retire() ([#1816](https://github.com/apify/crawlee-python/pull/1816)) ([c40d411](https://github.com/apify/crawlee-python/commit/c40d411b024ba2aae531a3c97609f78ad2c2757e)) by [@vdusek](https://github.com/vdusek)
|
||||
- Defer page object cleanup to make it accessible in error handlers ([#1814](https://github.com/apify/crawlee-python/pull/1814)) ([7eeb500](https://github.com/apify/crawlee-python/commit/7eeb5007cfb911901203ea21e1fd40127641feb1)) by [@janbuchar](https://github.com/janbuchar), closes [#1482](https://github.com/apify/crawlee-python/issues/1482)
|
||||
|
||||
### ⚡ Performance
|
||||
|
||||
- Offload BeautifulSoup parsing to a thread via `asyncio.to_thread` ([#1817](https://github.com/apify/crawlee-python/pull/1817)) ([d612ffa](https://github.com/apify/crawlee-python/commit/d612ffa1730f2aacfb7a28ae2b0ce2f4eda77692)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [1.6.0](https://github.com/apify/crawlee-python/releases/tag/v1.6.0) (2026-03-20)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Allow non-href links extract & enqueue ([#1781](https://github.com/apify/crawlee-python/pull/1781)) ([6db365d](https://github.com/apify/crawlee-python/commit/6db365d1625206d8d691256c9cd4b44a821238bb)) by [@kozlice](https://github.com/kozlice)
|
||||
- Add `post_navigation_hooks` to crawlers ([#1795](https://github.com/apify/crawlee-python/pull/1795)) ([38ceda6](https://github.com/apify/crawlee-python/commit/38ceda635a18cb2f14efc7c8e8b67f3adb7e53fd)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Add page lifecycle hooks to `BrowserPool` ([#1791](https://github.com/apify/crawlee-python/pull/1791)) ([6f2ac13](https://github.com/apify/crawlee-python/commit/6f2ac13fea4cfa8a65e6e41430d3e8d28cc3a787)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Expose `BrowserType` and `CrawleePage` ([#1798](https://github.com/apify/crawlee-python/pull/1798)) ([b50b9f2](https://github.com/apify/crawlee-python/commit/b50b9f2a8396dcee2bd7eaf76c94d24912c2bc5f)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Expose `use_state` in `BasicCrawler` ([#1799](https://github.com/apify/crawlee-python/pull/1799)) ([d121873](https://github.com/apify/crawlee-python/commit/d121873a7f5902b911dd04b4aa9eaf75a8449323)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- **redis:** Do not remove handled request data from request queue ([#1787](https://github.com/apify/crawlee-python/pull/1787)) ([3008c61](https://github.com/apify/crawlee-python/commit/3008c61dcbe07ccdf3c43f198b37582cc1356c9a)) by [@kozlice](https://github.com/kozlice)
|
||||
- **redis:** Update actual `Request` state in request queue Redis storage client ([#1789](https://github.com/apify/crawlee-python/pull/1789)) ([787231c](https://github.com/apify/crawlee-python/commit/787231cebeb863ee2b4395964a79a37053dbec01)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [1.5.0](https://github.com/apify/crawlee-python/releases/tag/v1.5.0) (2026-03-06)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Use specialized Playwright docker images in templates ([#1757](https://github.com/apify/crawlee-python/pull/1757)) ([747c0cf](https://github.com/apify/crawlee-python/commit/747c0cf4a82296a2e3ea5cac5ef4c9578ea62a0c)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1756](https://github.com/apify/crawlee-python/issues/1756)
|
||||
- Add `discover_valid_sitemaps` utility ([#1777](https://github.com/apify/crawlee-python/pull/1777)) ([872447b](https://github.com/apify/crawlee-python/commit/872447b60bbdb3926068064a971492807b1bdfbb)) by [@Mantisus](https://github.com/Mantisus), closes [#1740](https://github.com/apify/crawlee-python/issues/1740)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Prevent list modification during iteration in BrowserPool ([#1703](https://github.com/apify/crawlee-python/pull/1703)) ([70309d9](https://github.com/apify/crawlee-python/commit/70309d9bf568d268a26b3ba6392be2b6ff284c65)) by [@vdusek](https://github.com/vdusek)
|
||||
- Fix ` max_requests_per_crawl` excluding failed requests ([#1766](https://github.com/apify/crawlee-python/pull/1766)) ([d6bb0b4](https://github.com/apify/crawlee-python/commit/d6bb0b4a9dc5dd6668d076fbfa1b5e748deaee0d)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1765](https://github.com/apify/crawlee-python/issues/1765)
|
||||
- **playwright:** Dispose of `APIResponse` body for `send_request` ([#1771](https://github.com/apify/crawlee-python/pull/1771)) ([29d301b](https://github.com/apify/crawlee-python/commit/29d301bf9d7795f2fbaddb99235a7157b880f60c)) by [@kozlice](https://github.com/kozlice)
|
||||
- Return `None` from `add_request` when storage client fails to enqueue request ([#1775](https://github.com/apify/crawlee-python/pull/1775)) ([944753a](https://github.com/apify/crawlee-python/commit/944753a71956c30f3ce0896ffa24be7de5348933)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Re-use pre-existing browser context in `PlaywrightBrowserController` ([#1778](https://github.com/apify/crawlee-python/pull/1778)) ([4487543](https://github.com/apify/crawlee-python/commit/44875433df83d433aa69ada458b91df3ad569f5e)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1776](https://github.com/apify/crawlee-python/issues/1776)
|
||||
|
||||
|
||||
## [1.4.0](https://github.com/apify/crawlee-python/releases/tag/v1.4.0) (2026-02-17)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Dynamic memory snapshots ([#1715](https://github.com/apify/crawlee-python/pull/1715)) ([568a7b1](https://github.com/apify/crawlee-python/commit/568a7b186dedda19ad814ee8af3cd8e256cc4ad9)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1704](https://github.com/apify/crawlee-python/issues/1704)
|
||||
- Add `MySQL` and `MariaDB` support for `SqlStorageClient` ([#1749](https://github.com/apify/crawlee-python/pull/1749)) ([202b500](https://github.com/apify/crawlee-python/commit/202b5009ea5d35ea779eb5b8db1fc575f90ca7bb)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Make log levels consistent in ServiceLocator ([#1746](https://github.com/apify/crawlee-python/pull/1746)) ([4163413](https://github.com/apify/crawlee-python/commit/4163413049485b035c38efd6a4a7d41502a44cfc)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Fix `PlaywrightCrawler` unintentionally setting the global configuration ([#1747](https://github.com/apify/crawlee-python/pull/1747)) ([fa58438](https://github.com/apify/crawlee-python/commit/fa58438026eb72a6002c8d494725bf4e48b4407e)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
- Fix `Snapshotter` handling of out of order samples ([#1735](https://github.com/apify/crawlee-python/pull/1735)) ([387c712](https://github.com/apify/crawlee-python/commit/387c712306055d901b1c0df4a9666967f039aefd)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1734](https://github.com/apify/crawlee-python/issues/1734)
|
||||
|
||||
### ⚡ Performance
|
||||
|
||||
- Optimize metadata records processing in `SqlStorageClient` ([#1551](https://github.com/apify/crawlee-python/pull/1551)) ([df1347a](https://github.com/apify/crawlee-python/commit/df1347aacf05c05980000d15b36b65996119ea86)) by [@Mantisus](https://github.com/Mantisus), closes [#1533](https://github.com/apify/crawlee-python/issues/1533)
|
||||
|
||||
|
||||
## [1.3.2](https://github.com/apify/crawlee-python/releases/tag/v1.3.2) (2026-02-09)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Use `max()` instead of `min()` for `request_max_duration` statistic ([#1701](https://github.com/apify/crawlee-python/pull/1701)) ([85c4335](https://github.com/apify/crawlee-python/commit/85c43351a05ada1369b720061f6f1a7e158340b6)) by [@vdusek](https://github.com/vdusek)
|
||||
- Prevent mutation of default URL patterns list in `block_requests` ([#1702](https://github.com/apify/crawlee-python/pull/1702)) ([fcf9adb](https://github.com/apify/crawlee-python/commit/fcf9adb6a0cfeaa87ca482372d4e066584eb28d6)) by [@vdusek](https://github.com/vdusek)
|
||||
- Keep None values for `user_data` in `Request` ([#1707](https://github.com/apify/crawlee-python/pull/1707)) ([3c575bc](https://github.com/apify/crawlee-python/commit/3c575bc2b0f1c89c99d134ad3a3fa7455ccc6910)) by [@Mantisus](https://github.com/Mantisus), closes [#1706](https://github.com/apify/crawlee-python/issues/1706)
|
||||
- Respect `max_open_pages_per_browser` limit for `PlaywrightBrowserController` on concurrent `new_page` calls ([#1712](https://github.com/apify/crawlee-python/pull/1712)) ([2e5534b](https://github.com/apify/crawlee-python/commit/2e5534b98913d5cbd6b721b2423d063772024417)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [1.3.1](https://github.com/apify/crawlee-python/releases/tag/v1.3.1) (2026-01-30)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Reset all counter in metadata with `purge` for `RequestQueue` ([#1686](https://github.com/apify/crawlee-python/pull/1686)) ([ee09260](https://github.com/apify/crawlee-python/commit/ee0926084589f1b6e15840b6185ec5433be3b72f)) by [@Mantisus](https://github.com/Mantisus), closes [#1682](https://github.com/apify/crawlee-python/issues/1682)
|
||||
- Set default `http3=False` for `ImpitHttpClient` ([#1685](https://github.com/apify/crawlee-python/pull/1685)) ([3f390f6](https://github.com/apify/crawlee-python/commit/3f390f677540a3905038d7db6a6d1efad32fd045)) by [@Mantisus](https://github.com/Mantisus), closes [#1683](https://github.com/apify/crawlee-python/issues/1683)
|
||||
- Prevent get_request from permanently blocking requests ([#1684](https://github.com/apify/crawlee-python/pull/1684)) ([da416f9](https://github.com/apify/crawlee-python/commit/da416f98fb453904d62e7d29d8f24611ffb3ba8d)) by [@Mirza-Samad-Ahmed-Baig](https://github.com/Mirza-Samad-Ahmed-Baig)
|
||||
- Do not share state between different crawlers unless requested ([#1669](https://github.com/apify/crawlee-python/pull/1669)) ([64c246b](https://github.com/apify/crawlee-python/commit/64c246bedea14f86e607d23adc5bec644c578364)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1627](https://github.com/apify/crawlee-python/issues/1627)
|
||||
|
||||
|
||||
## [1.3.0](https://github.com/apify/crawlee-python/releases/tag/v1.3.0) (2026-01-20)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Expose `AdaptivePlaywrightCrawlerStatisticState` for `AdaptivePlaywrightCrawler` ([#1635](https://github.com/apify/crawlee-python/pull/1635)) ([1bb4bcb](https://github.com/apify/crawlee-python/commit/1bb4bcb4ccbec347ad9c14f70e9e946d48e3c38e)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Prevent race condition in concurrent storage creation ([#1626](https://github.com/apify/crawlee-python/pull/1626)) ([7f17a43](https://github.com/apify/crawlee-python/commit/7f17a4347d5884962767e757a92ec173688fed7b)) by [@Mantisus](https://github.com/Mantisus), closes [#1621](https://github.com/apify/crawlee-python/issues/1621)
|
||||
- Create correct statistics for `AdaptivePlaywrightCrawler` on initialization with a custom parser ([#1637](https://github.com/apify/crawlee-python/pull/1637)) ([bff7260](https://github.com/apify/crawlee-python/commit/bff726055dd0d7e07a2c546b15cbee22abd85960)) by [@Mantisus](https://github.com/Mantisus), closes [#1630](https://github.com/apify/crawlee-python/issues/1630)
|
||||
- Fix adding extra link for `EnqueueLinksFunction` with `limit` ([#1674](https://github.com/apify/crawlee-python/pull/1674)) ([71d7867](https://github.com/apify/crawlee-python/commit/71d7867b14f7f07cac06899f5da006091af4a954)) by [@Mantisus](https://github.com/Mantisus), closes [#1673](https://github.com/apify/crawlee-python/issues/1673)
|
||||
|
||||
|
||||
## [1.2.1](https://github.com/apify/crawlee-python/releases/tag/v1.2.1) (2025-12-16)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix short error summary ([#1605](https://github.com/apify/crawlee-python/pull/1605)) ([b751208](https://github.com/apify/crawlee-python/commit/b751208d9a56e9d923e4559baeba35e2eede0450)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1602](https://github.com/apify/crawlee-python/issues/1602)
|
||||
- Freeze core `Request` fields ([#1603](https://github.com/apify/crawlee-python/pull/1603)) ([ae6d86b](https://github.com/apify/crawlee-python/commit/ae6d86b8c82900116032596201d94cd7875aaadc)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Respect `enqueue_strategy` after redirects in `enqueue_links` ([#1607](https://github.com/apify/crawlee-python/pull/1607)) ([700df91](https://github.com/apify/crawlee-python/commit/700df91bc9be1299388030a3e48e4dbc6f5b85a0)) by [@Mantisus](https://github.com/Mantisus), closes [#1606](https://github.com/apify/crawlee-python/issues/1606)
|
||||
- Protect `Request` from partial mutations on request handler failure ([#1585](https://github.com/apify/crawlee-python/pull/1585)) ([a69caf8](https://github.com/apify/crawlee-python/commit/a69caf87edecc755287c53c8cc0ca4725af5d411)) by [@Mantisus](https://github.com/Mantisus), closes [#1514](https://github.com/apify/crawlee-python/issues/1514)
|
||||
|
||||
|
||||
|
||||
## [1.2.0](https://github.com/apify/crawlee-python/releases/tag/v1.2.0) (2025-12-08)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add additional kwargs to Crawler's export_data ([#1597](https://github.com/apify/crawlee-python/pull/1597)) ([5977f37](https://github.com/apify/crawlee-python/commit/5977f376b93a7c0d4dd53f0d331a4b04fedba2c6)) by [@vdusek](https://github.com/vdusek), closes [#526](https://github.com/apify/crawlee-python/issues/526)
|
||||
- Add `goto_options` for `PlaywrightCrawler` ([#1599](https://github.com/apify/crawlee-python/pull/1599)) ([0b82f3b](https://github.com/apify/crawlee-python/commit/0b82f3b6fb175223ea2aa5b348afcd5fdb767972)) by [@Mantisus](https://github.com/Mantisus), closes [#1576](https://github.com/apify/crawlee-python/issues/1576)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Only apply requestHandlerTimeout to request handler ([#1474](https://github.com/apify/crawlee-python/pull/1474)) ([0dfb6c2](https://github.com/apify/crawlee-python/commit/0dfb6c2a13b6650736245fa39b3fbff397644df7)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Handle the case when `error_handler` returns `Request` ([#1595](https://github.com/apify/crawlee-python/pull/1595)) ([8a961a2](https://github.com/apify/crawlee-python/commit/8a961a2b07d0d33a7302dbb13c17f3d90999d390)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Align `Request.state` transitions with `Request` lifecycle ([#1601](https://github.com/apify/crawlee-python/pull/1601)) ([383225f](https://github.com/apify/crawlee-python/commit/383225f9f055d95ffb1302b8cf96f42ec264f1fc)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [1.1.1](https://github.com/apify/crawlee-python/releases/tag/v1.1.1) (2025-12-02)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Unify separators in `unique_key` construction ([#1569](https://github.com/apify/crawlee-python/pull/1569)) ([af46a37](https://github.com/apify/crawlee-python/commit/af46a3733b059a8052489296e172f005def953f7)) by [@vdusek](https://github.com/vdusek), closes [#1512](https://github.com/apify/crawlee-python/issues/1512)
|
||||
- Fix `same-domain` strategy ignoring public suffix ([#1572](https://github.com/apify/crawlee-python/pull/1572)) ([3d018b2](https://github.com/apify/crawlee-python/commit/3d018b21a28a4bee493829783057188d6106a69b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1571](https://github.com/apify/crawlee-python/issues/1571)
|
||||
- Make context helpers work in `FailedRequestHandler` and `ErrorHandler` ([#1570](https://github.com/apify/crawlee-python/pull/1570)) ([b830019](https://github.com/apify/crawlee-python/commit/b830019350830ac33075316061659e2854f7f4a5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1532](https://github.com/apify/crawlee-python/issues/1532)
|
||||
- Fix non-ASCII character corruption in `FileSystemStorageClient` on systems without UTF-8 default encoding ([#1580](https://github.com/apify/crawlee-python/pull/1580)) ([f179f86](https://github.com/apify/crawlee-python/commit/f179f8671b0b6af9264450e4fef7e49d1cecd2bd)) by [@Mantisus](https://github.com/Mantisus), closes [#1579](https://github.com/apify/crawlee-python/issues/1579)
|
||||
- Respect `<base>` when enqueuing ([#1590](https://github.com/apify/crawlee-python/pull/1590)) ([de517a1](https://github.com/apify/crawlee-python/commit/de517a1629cc29b20568143eb64018f216d4ba33)) by [@Mantisus](https://github.com/Mantisus), closes [#1589](https://github.com/apify/crawlee-python/issues/1589)
|
||||
|
||||
|
||||
## [1.1.0](https://github.com/apify/crawlee-python/releases/tag/v1.1.0) (2025-11-18)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
|
||||
- Add `RedisStorageClient` based on Redis v8.0+ ([#1406](https://github.com/apify/crawlee-python/pull/1406)) ([d08d13d](https://github.com/apify/crawlee-python/commit/d08d13d39203c24ab61fe254b0956d6744db3b5f)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Add support for Python 3.14 ([#1553](https://github.com/apify/crawlee-python/pull/1553)) ([89e9130](https://github.com/apify/crawlee-python/commit/89e9130cabee0fbc974b29c26483b7fa0edf627c)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Add `transform_request_function` parameter for `SitemapRequestLoader` ([#1525](https://github.com/apify/crawlee-python/pull/1525)) ([dc90127](https://github.com/apify/crawlee-python/commit/dc901271849b239ba2a947e8ebff8e1815e8c4fb)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Improve indexing of the `request_queue_records` table for `SqlRequestQueueClient` ([#1527](https://github.com/apify/crawlee-python/pull/1527)) ([6509534](https://github.com/apify/crawlee-python/commit/65095346a9d8b703b10c91e0510154c3c48a4176)) by [@Mantisus](https://github.com/Mantisus), closes [#1526](https://github.com/apify/crawlee-python/issues/1526)
|
||||
- Improve error handling for `RobotsTxtFile.load` ([#1524](https://github.com/apify/crawlee-python/pull/1524)) ([596a311](https://github.com/apify/crawlee-python/commit/596a31184914a254b3e7a81fd2f48ea8eda7db49)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix `crawler_runtime` not being updated during run and only in the end ([#1540](https://github.com/apify/crawlee-python/pull/1540)) ([0d6c3f6](https://github.com/apify/crawlee-python/commit/0d6c3f6d3337ddb6cab4873747c28cf95605d550)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1541](https://github.com/apify/crawlee-python/issues/1541)
|
||||
- Ensure persist state event emission when exiting `EventManager` context ([#1562](https://github.com/apify/crawlee-python/pull/1562)) ([6a44f17](https://github.com/apify/crawlee-python/commit/6a44f172600cbcacebab899082d6efc9105c4e03)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1560](https://github.com/apify/crawlee-python/issues/1560)
|
||||
|
||||
|
||||
## [1.0.4](https://github.com/apify/crawlee-python/releases/tag/v1.0.4) (2025-10-24)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Respect `enqueue_strategy` in `enqueue_links` ([#1505](https://github.com/apify/crawlee-python/pull/1505)) ([6ee04bc](https://github.com/apify/crawlee-python/commit/6ee04bc08c50a70f2e956a79d4ce5072a726c3a8)) by [@Mantisus](https://github.com/Mantisus), closes [#1504](https://github.com/apify/crawlee-python/issues/1504)
|
||||
- Exclude incorrect links before checking `robots.txt` ([#1502](https://github.com/apify/crawlee-python/pull/1502)) ([3273da5](https://github.com/apify/crawlee-python/commit/3273da5fee62ec9254666b376f382474c3532a56)) by [@Mantisus](https://github.com/Mantisus), closes [#1499](https://github.com/apify/crawlee-python/issues/1499)
|
||||
- Resolve compatibility issue between `SqlStorageClient` and `AdaptivePlaywrightCrawler` ([#1496](https://github.com/apify/crawlee-python/pull/1496)) ([ce172c4](https://github.com/apify/crawlee-python/commit/ce172c425a8643a1d4c919db4f5e5a6e47e91deb)) by [@Mantisus](https://github.com/Mantisus), closes [#1495](https://github.com/apify/crawlee-python/issues/1495)
|
||||
- Fix `BasicCrawler` statistics persistence ([#1490](https://github.com/apify/crawlee-python/pull/1490)) ([1eb1c19](https://github.com/apify/crawlee-python/commit/1eb1c19aa6f9dda4a0e3f7eda23f77a554f95076)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1501](https://github.com/apify/crawlee-python/issues/1501)
|
||||
- Save context state in result for `AdaptivePlaywrightCrawler` after isolated processing in `SubCrawler` ([#1488](https://github.com/apify/crawlee-python/pull/1488)) ([62b7c70](https://github.com/apify/crawlee-python/commit/62b7c70b54085fc65a660062028014f4502beba9)) by [@Mantisus](https://github.com/Mantisus), closes [#1483](https://github.com/apify/crawlee-python/issues/1483)
|
||||
|
||||
|
||||
## [1.0.3](https://github.com/apify/crawlee-python/releases/tag/v1.0.3) (2025-10-17)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Add support for Pydantic v2.12 ([#1471](https://github.com/apify/crawlee-python/pull/1471)) ([35c1108](https://github.com/apify/crawlee-python/commit/35c110878c2f445a2866be2522ea8703e9b371dd)) by [@Mantisus](https://github.com/Mantisus), closes [#1464](https://github.com/apify/crawlee-python/issues/1464)
|
||||
- Fix database version warning message ([#1485](https://github.com/apify/crawlee-python/pull/1485)) ([18a545e](https://github.com/apify/crawlee-python/commit/18a545ee8add92e844acd0068f9cb8580a82e1c9)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix `reclaim_request` in `SqlRequestQueueClient` to correctly update the request state ([#1486](https://github.com/apify/crawlee-python/pull/1486)) ([1502469](https://github.com/apify/crawlee-python/commit/150246957f8f7f1ceb77bb77e3a02a903c50cae1)) by [@Mantisus](https://github.com/Mantisus), closes [#1484](https://github.com/apify/crawlee-python/issues/1484)
|
||||
- Fix `KeyValueStore.auto_saved_value` failing in some scenarios ([#1438](https://github.com/apify/crawlee-python/pull/1438)) ([b35dee7](https://github.com/apify/crawlee-python/commit/b35dee78180e57161b826641d45a61b8d8f6ef51)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1354](https://github.com/apify/crawlee-python/issues/1354)
|
||||
|
||||
|
||||
## [1.0.2](https://github.com/apify/crawlee-python/releases/tag/v1.0.2) (2025-10-08)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Use Self type in the open() method of storage clients ([#1462](https://github.com/apify/crawlee-python/pull/1462)) ([4ec6f6c](https://github.com/apify/crawlee-python/commit/4ec6f6c08f81632197f602ff99151338b3eba6e7)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Add storages name validation ([#1457](https://github.com/apify/crawlee-python/pull/1457)) ([84de11a](https://github.com/apify/crawlee-python/commit/84de11a3a603503076f5b7df487c9abab68a9015)) by [@Mantisus](https://github.com/Mantisus), closes [#1434](https://github.com/apify/crawlee-python/issues/1434)
|
||||
- Pin pydantic version to <2.12.0 to avoid compatibility issues ([#1467](https://github.com/apify/crawlee-python/pull/1467)) ([f11b86f](https://github.com/apify/crawlee-python/commit/f11b86f7ed57f98e83dc1b52f15f2017a919bf59)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [1.0.1](https://github.com/apify/crawlee-python/releases/tag/v1.0.1) (2025-10-06)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix memory leak in `PlaywrightCrawler` on browser context creation ([#1446](https://github.com/apify/crawlee-python/pull/1446)) ([bb181e5](https://github.com/apify/crawlee-python/commit/bb181e58d8070fba38e62d6e57fe981a00e5f035)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1443](https://github.com/apify/crawlee-python/issues/1443)
|
||||
- Update templates to handle optional httpx client ([#1440](https://github.com/apify/crawlee-python/pull/1440)) ([c087efd](https://github.com/apify/crawlee-python/commit/c087efd39baedf46ca3e5cae1ddc1acd6396e6c1)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [1.0.0](https://github.com/apify/crawlee-python/releases/tag/v1.0.0) (2025-09-29)
|
||||
|
||||
- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v1) for more details.
|
||||
- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v1) to ensure a smooth update.
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add utility for load and parse Sitemap and `SitemapRequestLoader` ([#1169](https://github.com/apify/crawlee-python/pull/1169)) ([66599f8](https://github.com/apify/crawlee-python/commit/66599f8d085f3a8622e130019b6fdce2325737de)) by [@Mantisus](https://github.com/Mantisus), closes [#1161](https://github.com/apify/crawlee-python/issues/1161)
|
||||
- Add periodic status logging and `status_message_callback` parameter for customization ([#1265](https://github.com/apify/crawlee-python/pull/1265)) ([b992fb2](https://github.com/apify/crawlee-python/commit/b992fb2a457dedd20fc3014d7a4a8afe14602342)) by [@Mantisus](https://github.com/Mantisus), closes [#96](https://github.com/apify/crawlee-python/issues/96)
|
||||
- Add crawlee-cli option to skip project installation ([#1294](https://github.com/apify/crawlee-python/pull/1294)) ([4d5aef0](https://github.com/apify/crawlee-python/commit/4d5aef05613d10c1442fe449d1cf0f63392c98e3)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1122](https://github.com/apify/crawlee-python/issues/1122)
|
||||
- Improve `Crawlee` CLI help text ([#1297](https://github.com/apify/crawlee-python/pull/1297)) ([afbe10f](https://github.com/apify/crawlee-python/commit/afbe10f15d93353f5bc551bf9f193414179d0dd7)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1295](https://github.com/apify/crawlee-python/issues/1295)
|
||||
- Add basic `OpenTelemetry` instrumentation ([#1255](https://github.com/apify/crawlee-python/pull/1255)) ([a92d8b3](https://github.com/apify/crawlee-python/commit/a92d8b3f843ee795bba7e14710bb1faa1fdbf292)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1254](https://github.com/apify/crawlee-python/issues/1254)
|
||||
- Add `ImpitHttpClient` http-client client using the `impit` library ([#1151](https://github.com/apify/crawlee-python/pull/1151)) ([0d0d268](https://github.com/apify/crawlee-python/commit/0d0d2681a4379c0e7ba54c49c86dabfef641610f)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Prevent overloading system memory when running locally ([#1270](https://github.com/apify/crawlee-python/pull/1270)) ([30de3bd](https://github.com/apify/crawlee-python/commit/30de3bd7722cbc34db9fc582b4bda7dc2dfa90ff)) by [@janbuchar](https://github.com/janbuchar), closes [#1232](https://github.com/apify/crawlee-python/issues/1232)
|
||||
- Expose `PlaywrightPersistentBrowser` class ([#1314](https://github.com/apify/crawlee-python/pull/1314)) ([b5fa955](https://github.com/apify/crawlee-python/commit/b5fa95508d7c099ff3a342577f338439283a975f)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Add `impit` option for Crawlee CLI ([#1312](https://github.com/apify/crawlee-python/pull/1312)) ([508d7ce](https://github.com/apify/crawlee-python/commit/508d7ce4d998f37ab2adcf9c057c3c635a69f863)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Persist RequestList state ([#1274](https://github.com/apify/crawlee-python/pull/1274)) ([cc68014](https://github.com/apify/crawlee-python/commit/cc680147ba3cc8b35b9da70274e53e6f5dd92434)) by [@janbuchar](https://github.com/janbuchar), closes [#99](https://github.com/apify/crawlee-python/issues/99)
|
||||
- Persist `DefaultRenderingTypePredictor` state ([#1340](https://github.com/apify/crawlee-python/pull/1340)) ([fad4c25](https://github.com/apify/crawlee-python/commit/fad4c25fc712915c4a45b24e3290b6f5dbd8a683)) by [@Mantisus](https://github.com/Mantisus), closes [#1272](https://github.com/apify/crawlee-python/issues/1272)
|
||||
- Persist the `SitemapRequestLoader` state ([#1347](https://github.com/apify/crawlee-python/pull/1347)) ([27ef9ad](https://github.com/apify/crawlee-python/commit/27ef9ad194552ea9f1321d91a7a52054be9a8a51)) by [@Mantisus](https://github.com/Mantisus), closes [#1269](https://github.com/apify/crawlee-python/issues/1269)
|
||||
- Add support for NDU storages ([#1401](https://github.com/apify/crawlee-python/pull/1401)) ([5dbd212](https://github.com/apify/crawlee-python/commit/5dbd212663e7abc37535713f4c6e3a5bbf30a12e)) by [@vdusek](https://github.com/vdusek), closes [#1175](https://github.com/apify/crawlee-python/issues/1175)
|
||||
- Add RQ id, name, alias args to `add_requests` and `enqueue_links` methods ([#1413](https://github.com/apify/crawlee-python/pull/1413)) ([1cae2bc](https://github.com/apify/crawlee-python/commit/1cae2bca0b1508fcb3cb419dc239caf33e20a7ef)) by [@Mantisus](https://github.com/Mantisus), closes [#1402](https://github.com/apify/crawlee-python/issues/1402)
|
||||
- Add `SqlStorageClient` based on `sqlalchemy` v2+ ([#1339](https://github.com/apify/crawlee-python/pull/1339)) ([07c75a0](https://github.com/apify/crawlee-python/commit/07c75a078b443b58bfaaeb72eb2aa1439458dc47)) by [@Mantisus](https://github.com/Mantisus), closes [#307](https://github.com/apify/crawlee-python/issues/307)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix memory estimation not working on MacOS ([#1330](https://github.com/apify/crawlee-python/pull/1330)) ([ab020eb](https://github.com/apify/crawlee-python/commit/ab020eb821a75723225b652d64babd84c368183f)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1329](https://github.com/apify/crawlee-python/issues/1329)
|
||||
- Fix retry count to not count the original request ([#1328](https://github.com/apify/crawlee-python/pull/1328)) ([74fa1d9](https://github.com/apify/crawlee-python/commit/74fa1d936cb3c29cf62d87862a96b4266694af2f)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1326](https://github.com/apify/crawlee-python/issues/1326)
|
||||
- [**breaking**] Remove unused "stats" field from RequestQueueMetadata ([#1331](https://github.com/apify/crawlee-python/pull/1331)) ([0a63bef](https://github.com/apify/crawlee-python/commit/0a63bef514b0bdcd3d6f208b386f706d0fe561e6)) by [@vdusek](https://github.com/vdusek)
|
||||
- Ignore unknown parameters passed in cookies ([#1336](https://github.com/apify/crawlee-python/pull/1336)) ([50d3ef7](https://github.com/apify/crawlee-python/commit/50d3ef7540551383d26d40f3404b435bde35b47d)) by [@Mantisus](https://github.com/Mantisus), closes [#1333](https://github.com/apify/crawlee-python/issues/1333)
|
||||
- Fix `timeout` for `stream` method in `ImpitHttpClient` ([#1352](https://github.com/apify/crawlee-python/pull/1352)) ([54b693b](https://github.com/apify/crawlee-python/commit/54b693b838f135a596e1e9493b565bc558b19a3a)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Include reason in the session rotation warning logs ([#1363](https://github.com/apify/crawlee-python/pull/1363)) ([d6d7a45](https://github.com/apify/crawlee-python/commit/d6d7a45dd64a906419d9552c45062d726cbb1a0f)) by [@vdusek](https://github.com/vdusek), closes [#1318](https://github.com/apify/crawlee-python/issues/1318)
|
||||
- Improve crawler statistics logging ([#1364](https://github.com/apify/crawlee-python/pull/1364)) ([1eb6da5](https://github.com/apify/crawlee-python/commit/1eb6da5dd85870124593dcad877284ccaed9c0ce)) by [@vdusek](https://github.com/vdusek), closes [#1317](https://github.com/apify/crawlee-python/issues/1317)
|
||||
- Do not add a request that is already in progress to `MemoryRequestQueueClient` ([#1384](https://github.com/apify/crawlee-python/pull/1384)) ([3af326c](https://github.com/apify/crawlee-python/commit/3af326c9dfa8fffd56a42ca42981374613739e39)) by [@Mantisus](https://github.com/Mantisus), closes [#1383](https://github.com/apify/crawlee-python/issues/1383)
|
||||
- Save `RequestQueueState` for `FileSystemRequestQueueClient` in default KVS ([#1411](https://github.com/apify/crawlee-python/pull/1411)) ([6ee60a0](https://github.com/apify/crawlee-python/commit/6ee60a08ac1f9414e1b792f4935cc3799cb5089a)) by [@Mantisus](https://github.com/Mantisus), closes [#1410](https://github.com/apify/crawlee-python/issues/1410)
|
||||
- Set default desired concurrency for non-browser crawlers to 10 ([#1419](https://github.com/apify/crawlee-python/pull/1419)) ([1cc9401](https://github.com/apify/crawlee-python/commit/1cc940197600d2539bda967880d7f9d241eb8c3e)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
### 🚜 Refactor
|
||||
|
||||
- [**breaking**] Introduce new storage client system ([#1194](https://github.com/apify/crawlee-python/pull/1194)) ([de1c03f](https://github.com/apify/crawlee-python/commit/de1c03f70dbd4ae1773fd49c632b3cfcfab82c26)) by [@vdusek](https://github.com/vdusek), closes [#92](https://github.com/apify/crawlee-python/issues/92), [#147](https://github.com/apify/crawlee-python/issues/147), [#783](https://github.com/apify/crawlee-python/issues/783), [#1247](https://github.com/apify/crawlee-python/issues/1247)
|
||||
- [**breaking**] Split `BrowserType` literal into two different literals based on context ([#1070](https://github.com/apify/crawlee-python/pull/1070)) ([72b5698](https://github.com/apify/crawlee-python/commit/72b5698fa0647ea02b08da5651736cc37c4c0f6a)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
- [**breaking**] Change method `HttpResponse.read` from sync to async ([#1296](https://github.com/apify/crawlee-python/pull/1296)) ([83fa8a4](https://github.com/apify/crawlee-python/commit/83fa8a416b6d2d4e27c678b9bf99bd1b8799f57b)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- [**breaking**] Replace `HttpxHttpClient` with `ImpitHttpClient` as default HTTP client ([#1307](https://github.com/apify/crawlee-python/pull/1307)) ([c803a97](https://github.com/apify/crawlee-python/commit/c803a976776a76846866d533e3a3ee8144e248c4)) by [@Mantisus](https://github.com/Mantisus), closes [#1079](https://github.com/apify/crawlee-python/issues/1079)
|
||||
- [**breaking**] Change Dataset unwind parameter to accept list of strings ([#1357](https://github.com/apify/crawlee-python/pull/1357)) ([862a203](https://github.com/apify/crawlee-python/commit/862a20398f00fe91802fe7a1ccd58b05aee053a1)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Remove `Request.id` field ([#1366](https://github.com/apify/crawlee-python/pull/1366)) ([32f3580](https://github.com/apify/crawlee-python/commit/32f3580e9775a871924ab1233085d0c549c4cd52)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1358](https://github.com/apify/crawlee-python/issues/1358)
|
||||
- [**breaking**] Refactor storage creation and caching, configuration and services ([#1386](https://github.com/apify/crawlee-python/pull/1386)) ([04649bd](https://github.com/apify/crawlee-python/commit/04649bde60d46b2bc18ae4f6e3fd9667d02a9cef)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1379](https://github.com/apify/crawlee-python/issues/1379)
|
||||
|
||||
|
||||
|
||||
## [0.6.12](https://github.com/apify/crawlee-python/releases/tag/v0.6.12) (2025-07-30)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `retire_browser_after_page_count` parameter for `BrowserPool` ([#1266](https://github.com/apify/crawlee-python/pull/1266)) ([603aa2b](https://github.com/apify/crawlee-python/commit/603aa2b192ef4bc42d88244bd009fffdb0614c06)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Use `perf_counter_ns` for request duration tracking ([#1260](https://github.com/apify/crawlee-python/pull/1260)) ([9e92f6b](https://github.com/apify/crawlee-python/commit/9e92f6b54400ce5004fbab770e2e4ac42f73148f)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1256](https://github.com/apify/crawlee-python/issues/1256)
|
||||
- Fix memory estimation not working on MacOS (#1330) ([8558954](https://github.com/apify/crawlee-python/commit/8558954feeb7d5e91378186974a29851fedae9c8)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1329](https://github.com/apify/crawlee-python/issues/1329)
|
||||
- Fix retry count to not count the original request (#1328) ([1aff3aa](https://github.com/apify/crawlee-python/commit/1aff3aaf0cdbe452a3731192449a445e5b2d7a63)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1326](https://github.com/apify/crawlee-python/issues/1326)
|
||||
- Ignore unknown parameters passed in cookies (#1336) ([0f2610c](https://github.com/apify/crawlee-python/commit/0f2610c0ee1154dc004de60fc57fe7c9f478166a)) by [@Mantisus](https://github.com/Mantisus), closes [#1333](https://github.com/apify/crawlee-python/issues/1333)
|
||||
|
||||
|
||||
## [0.6.11](https://github.com/apify/crawlee-python/releases/tag/v0.6.11) (2025-06-23)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `stream` method for `HttpClient` ([#1241](https://github.com/apify/crawlee-python/pull/1241)) ([95c68b0](https://github.com/apify/crawlee-python/commit/95c68b0b2d0bf9e093c1d0ee1002625172f7a868)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix `ClientSnapshot` overload calculation ([#1228](https://github.com/apify/crawlee-python/pull/1228)) ([a4fc1b6](https://github.com/apify/crawlee-python/commit/a4fc1b6e83143650666108c289c084ea0463b80c)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1207](https://github.com/apify/crawlee-python/issues/1207)
|
||||
- Use `PSS` instead of `RSS` to estimate children process memory usage on Linux ([#1210](https://github.com/apify/crawlee-python/pull/1210)) ([436032f](https://github.com/apify/crawlee-python/commit/436032f2de5f7d7fa1016033f1bb224159a8e6bf)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1206](https://github.com/apify/crawlee-python/issues/1206)
|
||||
- Do not raise an error to check 'same-domain' if there is no hostname in the url ([#1251](https://github.com/apify/crawlee-python/pull/1251)) ([a6c3aab](https://github.com/apify/crawlee-python/commit/a6c3aabf5f8341f215275077b6768a56118bc656)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [0.6.10](https://github.com/apify/crawlee-python/releases/tag/v0.6.10) (2025-06-02)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Allow config change on `PlaywrightCrawler` ([#1186](https://github.com/apify/crawlee-python/pull/1186)) ([f17bf31](https://github.com/apify/crawlee-python/commit/f17bf31456b702631aa7e0c26d4f07fd5eb7d1bd)) by [@mylank](https://github.com/mylank), closes [#1185](https://github.com/apify/crawlee-python/issues/1185)
|
||||
- Add `payload` to `SendRequestFunction` to support `POST` request ([#1202](https://github.com/apify/crawlee-python/pull/1202)) ([e7449f2](https://github.com/apify/crawlee-python/commit/e7449f206c580cb8383a66e4c9ff5f67c5ce8409)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix match check for specified enqueue strategy for requests with redirect ([#1199](https://github.com/apify/crawlee-python/pull/1199)) ([d84c30c](https://github.com/apify/crawlee-python/commit/d84c30cbd7c94d6525d3b6e8e86b379050454c0e)) by [@Mantisus](https://github.com/Mantisus), closes [#1198](https://github.com/apify/crawlee-python/issues/1198)
|
||||
- Set `WindowsSelectorEventLoopPolicy` only for curl-impersonate template without `playwright` ([#1209](https://github.com/apify/crawlee-python/pull/1209)) ([f3b839f](https://github.com/apify/crawlee-python/commit/f3b839ffc2ccc1b889b6d5928f35f57b725e27f1)) by [@Mantisus](https://github.com/Mantisus), closes [#1204](https://github.com/apify/crawlee-python/issues/1204)
|
||||
- Add support non-GET requests for `PlaywrightCrawler` ([#1208](https://github.com/apify/crawlee-python/pull/1208)) ([dbb9f44](https://github.com/apify/crawlee-python/commit/dbb9f44c71af15e1f86766fa0ba68281dd85fd9e)) by [@Mantisus](https://github.com/Mantisus), closes [#1201](https://github.com/apify/crawlee-python/issues/1201)
|
||||
- Respect `EnqueueLinksKwargs` for `extract_links` function ([#1213](https://github.com/apify/crawlee-python/pull/1213)) ([c9907d6](https://github.com/apify/crawlee-python/commit/c9907d6ff4c3a4a719b279cea77694c00a5a963d)) by [@Mantisus](https://github.com/Mantisus), closes [#1212](https://github.com/apify/crawlee-python/issues/1212)
|
||||
|
||||
|
||||
## [0.6.9](https://github.com/apify/crawlee-python/releases/tag/v0.6.9) (2025-05-02)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add an internal `HttpClient` to be used in `send_request` for `PlaywrightCrawler` using `APIRequestContext` bound to the browser context ([#1134](https://github.com/apify/crawlee-python/pull/1134)) ([e794f49](https://github.com/apify/crawlee-python/commit/e794f4985d3a018ee76d634fe2b2c735fb450272)) by [@Mantisus](https://github.com/Mantisus), closes [#928](https://github.com/apify/crawlee-python/issues/928)
|
||||
- Make timeout error log cleaner ([#1170](https://github.com/apify/crawlee-python/pull/1170)) ([78ea9d2](https://github.com/apify/crawlee-python/commit/78ea9d23e0b2d73286043b68393e462f636625c9)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1158](https://github.com/apify/crawlee-python/issues/1158)
|
||||
- Add `on_skipped_request` decorator, to process links skipped according to `robots.txt` rules ([#1166](https://github.com/apify/crawlee-python/pull/1166)) ([bd16f14](https://github.com/apify/crawlee-python/commit/bd16f14a834eebf485aea6b6a83f2b18bf16b504)) by [@Mantisus](https://github.com/Mantisus), closes [#1160](https://github.com/apify/crawlee-python/issues/1160)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix handle error without `args` in `_get_error_message` for `ErrorTracker` ([#1181](https://github.com/apify/crawlee-python/pull/1181)) ([21944d9](https://github.com/apify/crawlee-python/commit/21944d908b8404d2ad6c182104e7a8c27be12a6e)) by [@Mantisus](https://github.com/Mantisus), closes [#1179](https://github.com/apify/crawlee-python/issues/1179)
|
||||
- Temporarily add `certifi<=2025.1.31` dependency ([#1183](https://github.com/apify/crawlee-python/pull/1183)) ([25ff961](https://github.com/apify/crawlee-python/commit/25ff961990f9abc9d0673ba6573dfcf46dd6e53f)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.6.8](https://github.com/apify/crawlee-python/releases/tag/v0.6.8) (2025-04-25)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Handle unprocessed requests in `add_requests_batched` ([#1159](https://github.com/apify/crawlee-python/pull/1159)) ([7851175](https://github.com/apify/crawlee-python/commit/7851175304d63e455223b25853021cfbe15d68bd)) by [@Pijukatel](https://github.com/Pijukatel), closes [#456](https://github.com/apify/crawlee-python/issues/456)
|
||||
- Add `respect_robots_txt_file` option ([#1162](https://github.com/apify/crawlee-python/pull/1162)) ([c23f365](https://github.com/apify/crawlee-python/commit/c23f365bfd263b4357edf82c14a7c6ff8dee45e4)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Update `UnprocessedRequest` to match actual data ([#1155](https://github.com/apify/crawlee-python/pull/1155)) ([a15a1f3](https://github.com/apify/crawlee-python/commit/a15a1f3528c7cbcf78d3bda5a236bcee1d492764)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1150](https://github.com/apify/crawlee-python/issues/1150)
|
||||
- Fix the order in which cookies are saved to the `SessionCookies` and the handler is executed for `PlaywrightCrawler` ([#1163](https://github.com/apify/crawlee-python/pull/1163)) ([82ff69a](https://github.com/apify/crawlee-python/commit/82ff69acd8e409f56be56dd061aae0f854ec25b4)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Call `failed_request_handler` for `SessionError` when session rotation count exceeds maximum ([#1147](https://github.com/apify/crawlee-python/pull/1147)) ([b3637b6](https://github.com/apify/crawlee-python/commit/b3637b68ec7eae9de7f1b923fa2f68885da64b90)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [0.6.7](https://github.com/apify/crawlee-python/releases/tag/v0.6.7) (2025-04-17)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `ErrorSnapshotter` to `ErrorTracker` ([#1125](https://github.com/apify/crawlee-python/pull/1125)) ([9666092](https://github.com/apify/crawlee-python/commit/9666092c6a59ac4d34409038d5476e5b6fb58a26)) by [@Pijukatel](https://github.com/Pijukatel), closes [#151](https://github.com/apify/crawlee-python/issues/151)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Improve validation errors in Crawlee CLI ([#1140](https://github.com/apify/crawlee-python/pull/1140)) ([f2d33df](https://github.com/apify/crawlee-python/commit/f2d33dff178a3d3079eb3807feb9645a25cc7a93)) by [@vdusek](https://github.com/vdusek), closes [#1138](https://github.com/apify/crawlee-python/issues/1138)
|
||||
- Disable logger propagation to prevent duplicate logs ([#1156](https://github.com/apify/crawlee-python/pull/1156)) ([0b3648d](https://github.com/apify/crawlee-python/commit/0b3648d5d399f0af23520f7fb8ee635d38b512c4)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.6.6](https://github.com/apify/crawlee-python/releases/tag/v0.6.6) (2025-04-03)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `statistics_log_format` parameter to `BasicCrawler` ([#1061](https://github.com/apify/crawlee-python/pull/1061)) ([635ae4a](https://github.com/apify/crawlee-python/commit/635ae4a56c65e434783ca721f4164203f465abf0)) by [@Mantisus](https://github.com/Mantisus), closes [#700](https://github.com/apify/crawlee-python/issues/700)
|
||||
- Add Session binding capability via `session_id` in `Request` ([#1086](https://github.com/apify/crawlee-python/pull/1086)) ([cda7b31](https://github.com/apify/crawlee-python/commit/cda7b314ffda3104e4fd28a5e85c9e238d8866a4)) by [@Mantisus](https://github.com/Mantisus), closes [#1076](https://github.com/apify/crawlee-python/issues/1076)
|
||||
- Add `requests` argument to `EnqueueLinksFunction` ([#1024](https://github.com/apify/crawlee-python/pull/1024)) ([fc8444c](https://github.com/apify/crawlee-python/commit/fc8444c245c7607d3e378a4835d7d3355c4059be)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Add port for `same-origin` strategy check ([#1096](https://github.com/apify/crawlee-python/pull/1096)) ([9e24598](https://github.com/apify/crawlee-python/commit/9e245987d0aab0ba9c763689f12958b5a332db46)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix handling of loading empty `metadata` file for queue ([#1042](https://github.com/apify/crawlee-python/pull/1042)) ([b00876e](https://github.com/apify/crawlee-python/commit/b00876e8dcb30a12d3737bd31237da9daada46bb)) by [@Mantisus](https://github.com/Mantisus), closes [#1029](https://github.com/apify/crawlee-python/issues/1029)
|
||||
- Update favicon ([#1114](https://github.com/apify/crawlee-python/pull/1114)) ([eba900f](https://github.com/apify/crawlee-python/commit/eba900fc1e8d918c6fc464657c53004a3e0fe668)) by [@baldasseva](https://github.com/baldasseva)
|
||||
- **website:** Use correct image source ([#1115](https://github.com/apify/crawlee-python/pull/1115)) ([ee7806f](https://github.com/apify/crawlee-python/commit/ee7806fc2f9b7b590d9668cc9f86009a898a3da6)) by [@baldasseva](https://github.com/baldasseva)
|
||||
|
||||
|
||||
## [0.6.5](https://github.com/apify/crawlee-python/releases/tag/v0.6.5) (2025-03-13)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Update to `browserforge` workaround ([#1075](https://github.com/apify/crawlee-python/pull/1075)) ([2378cf8](https://github.com/apify/crawlee-python/commit/2378cf84ab1ed06473049a9ddfca2ba6f166306d)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.6.4](https://github.com/apify/crawlee-python/releases/tag/v0.6.4) (2025-03-12)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Add a check thread before set `add_signal_handler` ([#1068](https://github.com/apify/crawlee-python/pull/1068)) ([6983bda](https://github.com/apify/crawlee-python/commit/6983bda2dbc202b3ecbf7db62b11deee007b4b5f)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Temporary workaround for `browserforge` import time code execution ([#1073](https://github.com/apify/crawlee-python/pull/1073)) ([17d914f](https://github.com/apify/crawlee-python/commit/17d914f78242078f88c07d686a567d1091255eb1)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.6.3](https://github.com/apify/crawlee-python/releases/tag/v0.6.3) (2025-03-07)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add project template with `uv` package manager ([#1057](https://github.com/apify/crawlee-python/pull/1057)) ([9ec06e5](https://github.com/apify/crawlee-python/commit/9ec06e58032aa11af46ac9cd1ea7bb002a18eb13)) by [@Mantisus](https://github.com/Mantisus), closes [#1053](https://github.com/apify/crawlee-python/issues/1053)
|
||||
- Use fingerprint generator in `PlaywrightCrawler` by default ([#1060](https://github.com/apify/crawlee-python/pull/1060)) ([09cec53](https://github.com/apify/crawlee-python/commit/09cec532911043623eeb475aa8552c70bd94f8b7)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1054](https://github.com/apify/crawlee-python/issues/1054)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Update project templates for Poetry v2.x compatibility ([#1049](https://github.com/apify/crawlee-python/pull/1049)) ([96dc2f9](https://github.com/apify/crawlee-python/commit/96dc2f9b53b0a2d0f1d0c73d10e5244114e849ff)) by [@Mantisus](https://github.com/Mantisus), closes [#954](https://github.com/apify/crawlee-python/issues/954)
|
||||
- Remove tmp folder for PlaywrightCrawler in non-headless mode ([#1046](https://github.com/apify/crawlee-python/pull/1046)) ([3a7f444](https://github.com/apify/crawlee-python/commit/3a7f444fb7ee9a0ab1867c8c9586b15aab1e7df2)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [0.6.2](https://github.com/apify/crawlee-python/releases/tag/v0.6.2) (2025-03-05)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Extend ErrorTracker with error grouping ([#1014](https://github.com/apify/crawlee-python/pull/1014)) ([561de5c](https://github.com/apify/crawlee-python/commit/561de5c6b76af386cad5ac804a22fb7af227e460)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.6.1](https://github.com/apify/crawlee-python/releases/tag/v0.6.1) (2025-03-03)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Add `browserforge` to mandatory dependencies ([#1044](https://github.com/apify/crawlee-python/pull/1044)) ([ddfbde8](https://github.com/apify/crawlee-python/commit/ddfbde89dd3e3cbef0f3954936f4a41c3d6df909)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.6.0](https://github.com/apify/crawlee-python/releases/tag/v0.6.0) (2025-03-03)
|
||||
|
||||
- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v06) for more details.
|
||||
- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v06) to ensure a smooth update.
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Integrate browserforge fingerprints ([#829](https://github.com/apify/crawlee-python/pull/829)) ([2b156b4](https://github.com/apify/crawlee-python/commit/2b156b4ba688f9111195422e6058dff30eb1f782)) by [@Pijukatel](https://github.com/Pijukatel), closes [#549](https://github.com/apify/crawlee-python/issues/549)
|
||||
- Add AdaptivePlaywrightCrawler ([#872](https://github.com/apify/crawlee-python/pull/872)) ([5ba70b6](https://github.com/apify/crawlee-python/commit/5ba70b6e846a908a55db461ab0c85e3946f2bc7c)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
- Implement `_snapshot_client` for `Snapshotter` ([#957](https://github.com/apify/crawlee-python/pull/957)) ([ba4d384](https://github.com/apify/crawlee-python/commit/ba4d384228d030c20c580ed01fae0e78af3a9543)) by [@Mantisus](https://github.com/Mantisus), closes [#60](https://github.com/apify/crawlee-python/issues/60)
|
||||
- Add adaptive context helpers ([#964](https://github.com/apify/crawlee-python/pull/964)) ([e248f17](https://github.com/apify/crawlee-python/commit/e248f17fad7b6d1fc5e23a0a1e961db66068a411)) by [@Pijukatel](https://github.com/Pijukatel), closes [#249](https://github.com/apify/crawlee-python/issues/249)
|
||||
- [**breaking**] Enable additional status codes arguments to PlaywrightCrawler ([#959](https://github.com/apify/crawlee-python/pull/959)) ([87cf446](https://github.com/apify/crawlee-python/commit/87cf446a7cbaa900e28abd93d4c8a2e0d1747059)) by [@Pijukatel](https://github.com/Pijukatel), closes [#953](https://github.com/apify/crawlee-python/issues/953)
|
||||
- Replace `HeaderGenerator` implementation by `browserforge` implementation ([#960](https://github.com/apify/crawlee-python/pull/960)) ([c2f8c93](https://github.com/apify/crawlee-python/commit/c2f8c93a4ad57c4ede354545bf925bf3707899c9)) by [@Pijukatel](https://github.com/Pijukatel), closes [#937](https://github.com/apify/crawlee-python/issues/937)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix playwright template and dockerfile ([#972](https://github.com/apify/crawlee-python/pull/972)) ([c33b34d](https://github.com/apify/crawlee-python/commit/c33b34dd6e253b1261c700857bb5c4bbec6d5c14)) by [@janbuchar](https://github.com/janbuchar), closes [#969](https://github.com/apify/crawlee-python/issues/969)
|
||||
- Fix installing dependencies via pip in project template ([#977](https://github.com/apify/crawlee-python/pull/977)) ([1e3b8eb](https://github.com/apify/crawlee-python/commit/1e3b8eb1cdb57bf2f7256e8ae5f0706b0afc3ba9)) by [@janbuchar](https://github.com/janbuchar), closes [#975](https://github.com/apify/crawlee-python/issues/975)
|
||||
- Fix default migration storage ([#1018](https://github.com/apify/crawlee-python/pull/1018)) ([6a0c4d9](https://github.com/apify/crawlee-python/commit/6a0c4d94593f7e94f24eee8a97fc7bc83c4d02e1)) by [@Pijukatel](https://github.com/Pijukatel), closes [#991](https://github.com/apify/crawlee-python/issues/991)
|
||||
- Fix logger name for http based loggers ([#1023](https://github.com/apify/crawlee-python/pull/1023)) ([bfb3944](https://github.com/apify/crawlee-python/commit/bfb394446351c8f3b9879a9905607f7c929f2542)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1021](https://github.com/apify/crawlee-python/issues/1021)
|
||||
- Remove allow_redirects override in CurlImpersonateHttpClient ([#1017](https://github.com/apify/crawlee-python/pull/1017)) ([01d855a](https://github.com/apify/crawlee-python/commit/01d855a43389a6b4b16ec74767624fa7eb13151f)) by [@2tunnels](https://github.com/2tunnels), closes [#1016](https://github.com/apify/crawlee-python/issues/1016)
|
||||
- Remove follow_redirects override in HttpxHttpClient ([#1015](https://github.com/apify/crawlee-python/pull/1015)) ([88afda3](https://github.com/apify/crawlee-python/commit/88afda33e77be84bc91ad1239740b8e661bef2a2)) by [@2tunnels](https://github.com/2tunnels), closes [#1013](https://github.com/apify/crawlee-python/issues/1013)
|
||||
- Fix flaky test_common_headers_and_user_agent ([#1030](https://github.com/apify/crawlee-python/pull/1030)) ([58aa70e](https://github.com/apify/crawlee-python/commit/58aa70e9600d313b823a1376ab9b36fb416c1c4a)) by [@Pijukatel](https://github.com/Pijukatel), closes [#1027](https://github.com/apify/crawlee-python/issues/1027)
|
||||
|
||||
### 🚜 Refactor
|
||||
|
||||
- [**breaking**] Remove unused config properties ([#978](https://github.com/apify/crawlee-python/pull/978)) ([4b7fe29](https://github.com/apify/crawlee-python/commit/4b7fe2930540a5fbd753135e3ce29dc80f80c543)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Remove Base prefix from abstract class names ([#980](https://github.com/apify/crawlee-python/pull/980)) ([8ccb5d4](https://github.com/apify/crawlee-python/commit/8ccb5d41a1dae9b02088b433266ac89bd089561a)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Сhange default `incognito context` to `persistent context` for `Playwright` ([#985](https://github.com/apify/crawlee-python/pull/985)) ([f01520d](https://github.com/apify/crawlee-python/commit/f01520d22b31af9f0f13ca162cc47e6aa9744c6d)) by [@Mantisus](https://github.com/Mantisus), closes [#721](https://github.com/apify/crawlee-python/issues/721), [#963](https://github.com/apify/crawlee-python/issues/963)
|
||||
- [**breaking**] Change `Session` cookies from `dict` to `SessionCookies` with `CookieJar` ([#984](https://github.com/apify/crawlee-python/pull/984)) ([6523b3a](https://github.com/apify/crawlee-python/commit/6523b3ade0eed53b0363ddce250c557024339b5e)) by [@Mantisus](https://github.com/Mantisus), closes [#710](https://github.com/apify/crawlee-python/issues/710), [#933](https://github.com/apify/crawlee-python/issues/933)
|
||||
- [**breaking**] Replace enum with literal for `EnqueueStrategy` ([#1019](https://github.com/apify/crawlee-python/pull/1019)) ([d2481ef](https://github.com/apify/crawlee-python/commit/d2481ef71d3539979c5b1129387e72b4126fe366)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Update status code handling ([#1028](https://github.com/apify/crawlee-python/pull/1028)) ([6b59471](https://github.com/apify/crawlee-python/commit/6b5947125e63abdfff481b0669398fc9a7293e55)) by [@Mantisus](https://github.com/Mantisus), closes [#830](https://github.com/apify/crawlee-python/issues/830), [#998](https://github.com/apify/crawlee-python/issues/998)
|
||||
- [**breaking**] Move `cli` dependencies to optional dependencies ([#1011](https://github.com/apify/crawlee-python/pull/1011)) ([4382959](https://github.com/apify/crawlee-python/commit/43829590c6b4efd1dc9b833373f82a842a0a1a8e)) by [@Mantisus](https://github.com/Mantisus), closes [#703](https://github.com/apify/crawlee-python/issues/703), [#1010](https://github.com/apify/crawlee-python/issues/1010)
|
||||
|
||||
|
||||
## [0.5.4](https://github.com/apify/crawlee-python/releases/tag/v0.5.4) (2025-02-05)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add support `use_incognito_pages` for `browser_launch_options` in `PlaywrightCrawler` ([#941](https://github.com/apify/crawlee-python/pull/941)) ([eae3a33](https://github.com/apify/crawlee-python/commit/eae3a33a1842ebbdac5f9c51866a4be4bcf1ae2c)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix session management with retire ([#947](https://github.com/apify/crawlee-python/pull/947)) ([caee03f](https://github.com/apify/crawlee-python/commit/caee03fe3a43cc1d7a8d3f9e19b42df1bdb1c0aa)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix templates - poetry-plugin-export version and camoufox template name ([#952](https://github.com/apify/crawlee-python/pull/952)) ([7addea6](https://github.com/apify/crawlee-python/commit/7addea6605359cceba208e16ec9131724bdb3e9b)) by [@Pijukatel](https://github.com/Pijukatel), closes [#951](https://github.com/apify/crawlee-python/issues/951)
|
||||
- Fix convert relative link to absolute in `enqueue_links` for response with redirect ([#956](https://github.com/apify/crawlee-python/pull/956)) ([694102e](https://github.com/apify/crawlee-python/commit/694102e163bb9021a4830d2545d153f6f8f3de90)) by [@Mantisus](https://github.com/Mantisus), closes [#955](https://github.com/apify/crawlee-python/issues/955)
|
||||
- Fix `CurlImpersonateHttpClient` cookies handler ([#946](https://github.com/apify/crawlee-python/pull/946)) ([ed415c4](https://github.com/apify/crawlee-python/commit/ed415c433da2a40b0ee62534f0730d0737e991b8)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [0.5.3](https://github.com/apify/crawlee-python/releases/tag/v0.5.3) (2025-01-31)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add keep_alive flag to `crawler.__init__` ([#921](https://github.com/apify/crawlee-python/pull/921)) ([7a82d0c](https://github.com/apify/crawlee-python/commit/7a82d0cbdbe6c8739d4bf6a9b014e31f07e5a520)) by [@Pijukatel](https://github.com/Pijukatel), closes [#891](https://github.com/apify/crawlee-python/issues/891)
|
||||
- Add `block_requests` helper for `PlaywrightCrawler` ([#919](https://github.com/apify/crawlee-python/pull/919)) ([1030459](https://github.com/apify/crawlee-python/commit/103045994908f80cffee5ccfff91a040e0042f48)) by [@Mantisus](https://github.com/Mantisus), closes [#848](https://github.com/apify/crawlee-python/issues/848)
|
||||
- Return request handlers from decorator methods to allow further decoration ([#934](https://github.com/apify/crawlee-python/pull/934)) ([9ec0aae](https://github.com/apify/crawlee-python/commit/9ec0aae54e2a340d29c893567ae80bf8bd4510a9)) by [@mylank](https://github.com/mylank)
|
||||
- Add `transform_request_function` for `enqueue_links` ([#923](https://github.com/apify/crawlee-python/pull/923)) ([6b15957](https://github.com/apify/crawlee-python/commit/6b159578f612251e6d2253a72b6521430f4f9b09)) by [@Mantisus](https://github.com/Mantisus), closes [#894](https://github.com/apify/crawlee-python/issues/894)
|
||||
- Add `time_remaining_secs` property to `MIGRATING` event data ([#940](https://github.com/apify/crawlee-python/pull/940)) ([b44501b](https://github.com/apify/crawlee-python/commit/b44501bcadbd12673a8f47aa92f12da8e404f60b)) by [@fnesveda](https://github.com/fnesveda)
|
||||
- Add LogisticalRegressionPredictor - rendering type predictor for adaptive crawling ([#930](https://github.com/apify/crawlee-python/pull/930)) ([8440499](https://github.com/apify/crawlee-python/commit/8440499468db115a4c478e9bcdb692554d1655c5)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix crawler not retrying user handler if there was timeout in the handler ([#909](https://github.com/apify/crawlee-python/pull/909)) ([f4090ef](https://github.com/apify/crawlee-python/commit/f4090ef0ea0281d53dab16a77ceea2ef6ac43d76)) by [@Pijukatel](https://github.com/Pijukatel), closes [#907](https://github.com/apify/crawlee-python/issues/907)
|
||||
- Optimize memory consumption for `HttpxHttpClient`, fix proxy handling ([#905](https://github.com/apify/crawlee-python/pull/905)) ([d7ad480](https://github.com/apify/crawlee-python/commit/d7ad480834263ae0480049cb0a8db4dfc3946d8d)) by [@Mantisus](https://github.com/Mantisus), closes [#895](https://github.com/apify/crawlee-python/issues/895)
|
||||
- Fix `BrowserPool` and `PlaywrightBrowserPlugin` closure ([#932](https://github.com/apify/crawlee-python/pull/932)) ([997543d](https://github.com/apify/crawlee-python/commit/997543d2fa5afba49929f4407ee95d7a4933a50d)) by [@Mantisus](https://github.com/Mantisus)
|
||||
|
||||
|
||||
## [0.5.2](https://github.com/apify/crawlee-python/releases/tag/v0.5.2) (2025-01-17)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Avoid `use_state` race conditions. Remove key argument to `use_state` ([#868](https://github.com/apify/crawlee-python/pull/868)) ([000b976](https://github.com/apify/crawlee-python/commit/000b9761211502d86a893a31e3ca21998a6e3b99)) by [@Pijukatel](https://github.com/Pijukatel), closes [#856](https://github.com/apify/crawlee-python/issues/856)
|
||||
- Restore proxy functionality for PlaywrightCrawler broken in v0.5 ([#889](https://github.com/apify/crawlee-python/pull/889)) ([908c944](https://github.com/apify/crawlee-python/commit/908c944ff9b1fc8ed7eb35f0078a1de71e34d5c5)) by [@Mantisus](https://github.com/Mantisus), closes [#887](https://github.com/apify/crawlee-python/issues/887)
|
||||
- Fix the usage of Configuration ([#899](https://github.com/apify/crawlee-python/pull/899)) ([0f1cf6f](https://github.com/apify/crawlee-python/commit/0f1cf6f0b52c92ca4e465a2a01f8111cd9ab42ec)) by [@vdusek](https://github.com/vdusek), closes [#670](https://github.com/apify/crawlee-python/issues/670)
|
||||
|
||||
|
||||
## [0.5.1](https://github.com/apify/crawlee-python/releases/tag/v0.5.1) (2025-01-07)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Make result of RequestList.is_empty independent of fetch_next_request calls ([#876](https://github.com/apify/crawlee-python/pull/876)) ([d50249e](https://github.com/apify/crawlee-python/commit/d50249ecbfe2a04f508fcdc3261e050349bd0da2)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
|
||||
## [0.5.0](https://github.com/apify/crawlee-python/releases/tag/v0.5.0) (2025-01-02)
|
||||
|
||||
- Check out the [Release blog post](https://crawlee.dev/blog/crawlee-for-python-v05) for more details.
|
||||
- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v05) to ensure a smooth update.
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add possibility to use None as no proxy in tiered proxies ([#760](https://github.com/apify/crawlee-python/pull/760)) ([0fbd017](https://github.com/apify/crawlee-python/commit/0fbd01723b9fe2e3410e0f358cab2f22848b08d0)) by [@Pijukatel](https://github.com/Pijukatel), closes [#687](https://github.com/apify/crawlee-python/issues/687)
|
||||
- Add `use_state` context method ([#682](https://github.com/apify/crawlee-python/pull/682)) ([868b41e](https://github.com/apify/crawlee-python/commit/868b41ebd4c8003fa60ab07887577d0fb85b6ecc)) by [@Mantisus](https://github.com/Mantisus), closes [#191](https://github.com/apify/crawlee-python/issues/191)
|
||||
- Add pre-navigation hooks router to AbstractHttpCrawler ([#791](https://github.com/apify/crawlee-python/pull/791)) ([0f23205](https://github.com/apify/crawlee-python/commit/0f23205923065074c522b3de9d47218a204dfa78)) by [@Pijukatel](https://github.com/Pijukatel), closes [#635](https://github.com/apify/crawlee-python/issues/635)
|
||||
- Add example of how to integrate Camoufox into PlaywrightCrawler ([#789](https://github.com/apify/crawlee-python/pull/789)) ([246cfc4](https://github.com/apify/crawlee-python/commit/246cfc4ebc8bce1d15e1dddd62d652bd65869328)) by [@Pijukatel](https://github.com/Pijukatel), closes [#684](https://github.com/apify/crawlee-python/issues/684)
|
||||
- Expose event types, improve on/emit signature, allow parameterless listeners ([#800](https://github.com/apify/crawlee-python/pull/800)) ([c102c4c](https://github.com/apify/crawlee-python/commit/c102c4c894a00b09adfd5f4911563c81cf3e98b4)) by [@janbuchar](https://github.com/janbuchar), closes [#561](https://github.com/apify/crawlee-python/issues/561)
|
||||
- Add stop method to BasicCrawler ([#807](https://github.com/apify/crawlee-python/pull/807)) ([6d01af4](https://github.com/apify/crawlee-python/commit/6d01af4231d02b4349a8719f5ed18d812843fde5)) by [@Pijukatel](https://github.com/Pijukatel), closes [#651](https://github.com/apify/crawlee-python/issues/651)
|
||||
- Add `html_to_text` helper function ([#792](https://github.com/apify/crawlee-python/pull/792)) ([2b9d970](https://github.com/apify/crawlee-python/commit/2b9d97009dd653870681bb3cadbb46b214ff1a73)) by [@Pijukatel](https://github.com/Pijukatel), closes [#659](https://github.com/apify/crawlee-python/issues/659)
|
||||
- [**breaking**] Implement `RequestManagerTandem`, remove `add_request` from `RequestList`, accept any iterable in `RequestList` constructor ([#777](https://github.com/apify/crawlee-python/pull/777)) ([4172652](https://github.com/apify/crawlee-python/commit/4172652079e5e91190c1cc5e2138fd41a7c84a6b)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix circular import in `KeyValueStore` ([#805](https://github.com/apify/crawlee-python/pull/805)) ([8bdf49d](https://github.com/apify/crawlee-python/commit/8bdf49d1cb2a94b66f69fd1b77063a4113517fae)) by [@Mantisus](https://github.com/Mantisus), closes [#804](https://github.com/apify/crawlee-python/issues/804)
|
||||
- [**breaking**] Refactor service usage to rely on `service_locator` ([#691](https://github.com/apify/crawlee-python/pull/691)) ([1d31c6c](https://github.com/apify/crawlee-python/commit/1d31c6c7e7a9ec7cee5b2de900568d9f77db65ba)) by [@vdusek](https://github.com/vdusek), closes [#369](https://github.com/apify/crawlee-python/issues/369), [#539](https://github.com/apify/crawlee-python/issues/539), [#699](https://github.com/apify/crawlee-python/issues/699)
|
||||
- Pass `verify` in httpx client ([#802](https://github.com/apify/crawlee-python/pull/802)) ([074d083](https://github.com/apify/crawlee-python/commit/074d0836b55e52f13726e7cd1c21602623fda4fc)) by [@Mantisus](https://github.com/Mantisus), closes [#798](https://github.com/apify/crawlee-python/issues/798)
|
||||
- Fix `page_options` for `PlaywrightBrowserPlugin` ([#796](https://github.com/apify/crawlee-python/pull/796)) ([bd3bdd4](https://github.com/apify/crawlee-python/commit/bd3bdd4046c2ddea62feb77322033cad50f382dd)) by [@Mantisus](https://github.com/Mantisus), closes [#755](https://github.com/apify/crawlee-python/issues/755)
|
||||
- Fix event migrating handler in `RequestQueue` ([#825](https://github.com/apify/crawlee-python/pull/825)) ([fd6663f](https://github.com/apify/crawlee-python/commit/fd6663f903bc7eecd1000da89e06197b43dfb962)) by [@Mantisus](https://github.com/Mantisus), closes [#815](https://github.com/apify/crawlee-python/issues/815)
|
||||
- Respect user configuration for work with status codes ([#812](https://github.com/apify/crawlee-python/pull/812)) ([8daf4bd](https://github.com/apify/crawlee-python/commit/8daf4bd49c1b09a0924f827daedebf7600ac609b)) by [@Mantisus](https://github.com/Mantisus), closes [#708](https://github.com/apify/crawlee-python/issues/708), [#756](https://github.com/apify/crawlee-python/issues/756)
|
||||
- `abort-on-error` for successive runs ([#834](https://github.com/apify/crawlee-python/pull/834)) ([0cea673](https://github.com/apify/crawlee-python/commit/0cea67387bf366800b447de784af580159b199ee)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Relax ServiceLocator restrictions ([#837](https://github.com/apify/crawlee-python/pull/837)) ([aa3667f](https://github.com/apify/crawlee-python/commit/aa3667f344d78945df3eca77431e1409f43f8bb5)) by [@janbuchar](https://github.com/janbuchar), closes [#806](https://github.com/apify/crawlee-python/issues/806)
|
||||
- Fix typo in exports ([#841](https://github.com/apify/crawlee-python/pull/841)) ([8fa6ac9](https://github.com/apify/crawlee-python/commit/8fa6ac994fe4f3f6430cb796a0c6a732c93c672b)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
### 🚜 Refactor
|
||||
|
||||
- [**breaking**] Refactor HttpCrawler, BeautifulSoupCrawler, ParselCrawler inheritance ([#746](https://github.com/apify/crawlee-python/pull/746)) ([9d3c269](https://github.com/apify/crawlee-python/commit/9d3c2697c91ce93028ca86a91d85d465d36c1ad7)) by [@Pijukatel](https://github.com/Pijukatel), closes [#350](https://github.com/apify/crawlee-python/issues/350)
|
||||
- [**breaking**] Remove `json_` and `order_no` from `Request` ([#788](https://github.com/apify/crawlee-python/pull/788)) ([5381d13](https://github.com/apify/crawlee-python/commit/5381d13aa51a757fc1906f400788555df090a1af)) by [@Mantisus](https://github.com/Mantisus), closes [#94](https://github.com/apify/crawlee-python/issues/94)
|
||||
- [**breaking**] Rename PwPreNavContext to PwPreNavCrawlingContext ([#827](https://github.com/apify/crawlee-python/pull/827)) ([84b61a3](https://github.com/apify/crawlee-python/commit/84b61a3d25bee42faed4e81cd156663f251b3d3d)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Rename PlaywrightCrawler kwargs: browser_options, page_options ([#831](https://github.com/apify/crawlee-python/pull/831)) ([ffc6048](https://github.com/apify/crawlee-python/commit/ffc6048e9dc5c5e862271fa50c48bb0fb6f0a18f)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
- [**breaking**] Update the crawlers & storage clients structure ([#828](https://github.com/apify/crawlee-python/pull/828)) ([0ba04d1](https://github.com/apify/crawlee-python/commit/0ba04d1633881043928a408678932c46fb90e21f)) by [@vdusek](https://github.com/vdusek), closes [#764](https://github.com/apify/crawlee-python/issues/764)
|
||||
|
||||
|
||||
## [0.4.5](https://github.com/apify/crawlee-python/releases/tag/v0.4.5) (2024-12-06)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Improve project bootstrapping ([#538](https://github.com/apify/crawlee-python/pull/538)) ([367899c](https://github.com/apify/crawlee-python/commit/367899cbad5021674f6e41c4dd7eb2266fe043aa)) by [@janbuchar](https://github.com/janbuchar), closes [#317](https://github.com/apify/crawlee-python/issues/317), [#414](https://github.com/apify/crawlee-python/issues/414), [#495](https://github.com/apify/crawlee-python/issues/495), [#511](https://github.com/apify/crawlee-python/issues/511)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Add upper bound of HTTPX version ([#775](https://github.com/apify/crawlee-python/pull/775)) ([b59e34d](https://github.com/apify/crawlee-python/commit/b59e34d6301e26825d88608152ffb337ef602a9f)) by [@vdusek](https://github.com/vdusek)
|
||||
- Fix incorrect use of desired concurrency ratio ([#780](https://github.com/apify/crawlee-python/pull/780)) ([d1f8bfb](https://github.com/apify/crawlee-python/commit/d1f8bfb68ce2ef13b550ce415a3689858112a4c7)) by [@Pijukatel](https://github.com/Pijukatel), closes [#759](https://github.com/apify/crawlee-python/issues/759)
|
||||
- Remove pydantic constraint <2.10.0 and update timedelta validator, serializer type hints ([#757](https://github.com/apify/crawlee-python/pull/757)) ([c0050c0](https://github.com/apify/crawlee-python/commit/c0050c0ee76e5deb28f174ecf276b0e6abf68b9d)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.4.4](https://github.com/apify/crawlee-python/releases/tag/v0.4.4) (2024-11-29)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Expose browser_options and page_options to PlaywrightCrawler ([#730](https://github.com/apify/crawlee-python/pull/730)) ([dbe85b9](https://github.com/apify/crawlee-python/commit/dbe85b90e59def281cfc6617a0eb869a4adf2fc0)) by [@vdusek](https://github.com/vdusek), closes [#719](https://github.com/apify/crawlee-python/issues/719)
|
||||
- Add `abort_on_error` property ([#731](https://github.com/apify/crawlee-python/pull/731)) ([6dae03a](https://github.com/apify/crawlee-python/commit/6dae03a68a2d23c68c78d8d44611d43e40eb9404)) by [@Mantisus](https://github.com/Mantisus), closes [#704](https://github.com/apify/crawlee-python/issues/704)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix init of context managers and context handling in `BasicCrawler` ([#714](https://github.com/apify/crawlee-python/pull/714)) ([486fe6d](https://github.com/apify/crawlee-python/commit/486fe6d6cd56cb560ab51a32ec0286d9e32267cb)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.4.3](https://github.com/apify/crawlee-python/releases/tag/v0.4.3) (2024-11-21)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Pydantic 2.10.0 issues ([#716](https://github.com/apify/crawlee-python/pull/716)) ([8d8b3fc](https://github.com/apify/crawlee-python/commit/8d8b3fcff8be10edf5351f5324c7ba112c1d2ba0)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.4.2](https://github.com/apify/crawlee-python/releases/tag/v0.4.2) (2024-11-20)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Respect custom HTTP headers in `PlaywrightCrawler` ([#685](https://github.com/apify/crawlee-python/pull/685)) ([a84125f](https://github.com/apify/crawlee-python/commit/a84125f031347426de44b8f015c87882c8f96f72)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Fix serialization payload in Request. Fix Docs for Post Request ([#683](https://github.com/apify/crawlee-python/pull/683)) ([e8b4d2d](https://github.com/apify/crawlee-python/commit/e8b4d2d4989fd9967403b828c914cb7ae2ef9b8b)) by [@Mantisus](https://github.com/Mantisus), closes [#668](https://github.com/apify/crawlee-python/issues/668)
|
||||
- Accept string payload in the Request constructor ([#697](https://github.com/apify/crawlee-python/pull/697)) ([19f5add](https://github.com/apify/crawlee-python/commit/19f5addc0223d68389eea47864830c709335ab6e)) by [@vdusek](https://github.com/vdusek)
|
||||
- Fix snapshots handling ([#692](https://github.com/apify/crawlee-python/pull/692)) ([4016c0d](https://github.com/apify/crawlee-python/commit/4016c0d8121a8950ab1df22188eac838a011c39f)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.4.1](https://github.com/apify/crawlee-python/releases/tag/v0.4.1) (2024-11-11)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add `max_crawl_depth` option to `BasicCrawler` ([#637](https://github.com/apify/crawlee-python/pull/637)) ([77deaa9](https://github.com/apify/crawlee-python/commit/77deaa964e2c1e74af1c5117a13d8d8257f0e27e)) by [@Prathamesh010](https://github.com/Prathamesh010), closes [#460](https://github.com/apify/crawlee-python/issues/460)
|
||||
- Add BeautifulSoupParser type alias ([#674](https://github.com/apify/crawlee-python/pull/674)) ([b2cf88f](https://github.com/apify/crawlee-python/commit/b2cf88ffea8d75808c9210850a03fcc70b0b9e3d)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Fix total_size usage in memory size monitoring ([#661](https://github.com/apify/crawlee-python/pull/661)) ([c2a3239](https://github.com/apify/crawlee-python/commit/c2a32397eecd5cc7f412c2af7269b004a8b2eaf2)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Add HttpHeaders to module exports ([#664](https://github.com/apify/crawlee-python/pull/664)) ([f0c5ca7](https://github.com/apify/crawlee-python/commit/f0c5ca717d9f9e304d375da2c23552c26ca870da)) by [@vdusek](https://github.com/vdusek), closes [#663](https://github.com/apify/crawlee-python/issues/663)
|
||||
- Fix unhandled ValueError in request handler result processing ([#666](https://github.com/apify/crawlee-python/pull/666)) ([0a99d7f](https://github.com/apify/crawlee-python/commit/0a99d7f693245eb9a065016fb6f2d268f6956805)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Fix BaseDatasetClient.iter_items type hints ([#680](https://github.com/apify/crawlee-python/pull/680)) ([a968b1b](https://github.com/apify/crawlee-python/commit/a968b1be6fceb56676b0198a044c8fceac7c92a6)) by [@Pijukatel](https://github.com/Pijukatel)
|
||||
|
||||
|
||||
## [0.4.0](https://github.com/apify/crawlee-python/releases/tag/v0.4.0) (2024-11-01)
|
||||
|
||||
- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v04) to ensure a smooth update.
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- [**breaking**] Add headers in unique key computation ([#609](https://github.com/apify/crawlee-python/pull/609)) ([6c4746f](https://github.com/apify/crawlee-python/commit/6c4746fa8ff86952a812b32a1d70dc910e76b43e)) by [@Prathamesh010](https://github.com/Prathamesh010), closes [#548](https://github.com/apify/crawlee-python/issues/548)
|
||||
- Add `pre_navigation_hooks` to `PlaywrightCrawler` ([#631](https://github.com/apify/crawlee-python/pull/631)) ([5dd5b60](https://github.com/apify/crawlee-python/commit/5dd5b60e2a44d5bd3748b613790e1bee3232d6f3)) by [@Prathamesh010](https://github.com/Prathamesh010), closes [#427](https://github.com/apify/crawlee-python/issues/427)
|
||||
- Add `always_enqueue` option to bypass URL deduplication ([#621](https://github.com/apify/crawlee-python/pull/621)) ([4e59fa4](https://github.com/apify/crawlee-python/commit/4e59fa46daaec05e52262cf62c26f28ddcd772af)) by [@Rutam21](https://github.com/Rutam21), closes [#547](https://github.com/apify/crawlee-python/issues/547)
|
||||
- Split and add extra configuration to export_data method ([#580](https://github.com/apify/crawlee-python/pull/580)) ([6751635](https://github.com/apify/crawlee-python/commit/6751635e1785a4a27f60092c82f5dd0c40193d52)) by [@deshansh](https://github.com/deshansh), closes [#526](https://github.com/apify/crawlee-python/issues/526)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Use strip in headers normalization ([#614](https://github.com/apify/crawlee-python/pull/614)) ([a15b21e](https://github.com/apify/crawlee-python/commit/a15b21e51deaf2b67738f95bc2b15c1c16d1775f)) by [@vdusek](https://github.com/vdusek)
|
||||
- [**breaking**] Merge payload and data fields of Request ([#542](https://github.com/apify/crawlee-python/pull/542)) ([d06fcef](https://github.com/apify/crawlee-python/commit/d06fcef3fee44616ded5f587b9c7313b82a57cc7)) by [@vdusek](https://github.com/vdusek), closes [#560](https://github.com/apify/crawlee-python/issues/560)
|
||||
- Default ProxyInfo port if httpx.URL port is None ([#619](https://github.com/apify/crawlee-python/pull/619)) ([8107a6f](https://github.com/apify/crawlee-python/commit/8107a6f97e8f16a330e7d02d3fc6ea34c5f78d77)) by [@steffansafey](https://github.com/steffansafey), closes [#618](https://github.com/apify/crawlee-python/issues/618)
|
||||
|
||||
### ⚙️ Miscellaneous Tasks
|
||||
|
||||
- [**breaking**] Remove Request.query_params field ([#639](https://github.com/apify/crawlee-python/pull/639)) ([6ec0ec4](https://github.com/apify/crawlee-python/commit/6ec0ec4fa0cef9b8bf893e70d99f068675c9c54c)) by [@vdusek](https://github.com/vdusek), closes [#615](https://github.com/apify/crawlee-python/issues/615)
|
||||
|
||||
|
||||
## [0.3.9](https://github.com/apify/crawlee-python/releases/tag/v0.3.9) (2024-10-23)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Key-value store context helpers ([#584](https://github.com/apify/crawlee-python/pull/584)) ([fc15622](https://github.com/apify/crawlee-python/commit/fc156222c3747fc4cc7bd7666a21769845c7d0d5)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Added get_public_url method to KeyValueStore ([#572](https://github.com/apify/crawlee-python/pull/572)) ([3a4ba8f](https://github.com/apify/crawlee-python/commit/3a4ba8f459903b6288aec40de2c3ca862e36abec)) by [@akshay11298](https://github.com/akshay11298), closes [#514](https://github.com/apify/crawlee-python/issues/514)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Workaround for JSON value typing problems ([#581](https://github.com/apify/crawlee-python/pull/581)) ([403496a](https://github.com/apify/crawlee-python/commit/403496a53c12810351139a6e073238143ecc5930)) by [@janbuchar](https://github.com/janbuchar), closes [#563](https://github.com/apify/crawlee-python/issues/563)
|
||||
|
||||
|
||||
## [0.3.8](https://github.com/apify/crawlee-python/releases/tag/v0.3.8) (2024-10-02)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Mask Playwright's "headless" headers ([#545](https://github.com/apify/crawlee-python/pull/545)) ([d1445e4](https://github.com/apify/crawlee-python/commit/d1445e4858fd804bb4a2e35efa1d2f5254d8df6b)) by [@vdusek](https://github.com/vdusek), closes [#401](https://github.com/apify/crawlee-python/issues/401)
|
||||
- Add new model for `HttpHeaders` ([#544](https://github.com/apify/crawlee-python/pull/544)) ([854f2c1](https://github.com/apify/crawlee-python/commit/854f2c1e2e09cf398e04b1e153534282add1247e)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Call `error_handler` for `SessionError` ([#557](https://github.com/apify/crawlee-python/pull/557)) ([e75ac4b](https://github.com/apify/crawlee-python/commit/e75ac4b70cd48a4ca9f8245cea3c5f3c188b8824)) by [@vdusek](https://github.com/vdusek), closes [#546](https://github.com/apify/crawlee-python/issues/546)
|
||||
- Extend from `StrEnum` in `RequestState` to fix serialization ([#556](https://github.com/apify/crawlee-python/pull/556)) ([6bf35ba](https://github.com/apify/crawlee-python/commit/6bf35ba4a6913819706ebd1d2c1156a4c62f944e)) by [@vdusek](https://github.com/vdusek), closes [#551](https://github.com/apify/crawlee-python/issues/551)
|
||||
- Add equality check to UserData model ([#562](https://github.com/apify/crawlee-python/pull/562)) ([899a25c](https://github.com/apify/crawlee-python/commit/899a25ca63f570b3c4d8d56c85a838b371fd3924)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
|
||||
## [0.3.7](https://github.com/apify/crawlee-python/releases/tag/v0.3.7) (2024-09-25)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Improve `Request.user_data` serialization ([#540](https://github.com/apify/crawlee-python/pull/540)) ([de29c0e](https://github.com/apify/crawlee-python/commit/de29c0e6b737a9d2544c5382472618dde76eb2a5)) by [@janbuchar](https://github.com/janbuchar), closes [#524](https://github.com/apify/crawlee-python/issues/524)
|
||||
- Adopt new version of curl-cffi ([#543](https://github.com/apify/crawlee-python/pull/543)) ([f6fcf48](https://github.com/apify/crawlee-python/commit/f6fcf48d99bfcb4b8e75c5c9c38dc8c265164a10)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.3.6](https://github.com/apify/crawlee-python/releases/tag/v0.3.6) (2024-09-19)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add HTTP/2 support for HTTPX client ([#513](https://github.com/apify/crawlee-python/pull/513)) ([0eb0a33](https://github.com/apify/crawlee-python/commit/0eb0a33411096011198e52c393f35730f1a0b6ac)) by [@vdusek](https://github.com/vdusek), closes [#512](https://github.com/apify/crawlee-python/issues/512)
|
||||
- Expose extended unique key when creating a new Request ([#515](https://github.com/apify/crawlee-python/pull/515)) ([1807f41](https://github.com/apify/crawlee-python/commit/1807f419e47a815dd706d09acb0f3b3af8cfc691)) by [@vdusek](https://github.com/vdusek)
|
||||
- Add header generator and integrate it into HTTPX client ([#530](https://github.com/apify/crawlee-python/pull/530)) ([b63f9f9](https://github.com/apify/crawlee-python/commit/b63f9f98c6613e095546ef544eab271d433e3379)) by [@vdusek](https://github.com/vdusek), closes [#402](https://github.com/apify/crawlee-python/issues/402)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Use explicitly UTF-8 encoding in local storage ([#533](https://github.com/apify/crawlee-python/pull/533)) ([a3a0ab2](https://github.com/apify/crawlee-python/commit/a3a0ab2f6809b7a06319a77dfbf289df78638dea)) by [@vdusek](https://github.com/vdusek), closes [#532](https://github.com/apify/crawlee-python/issues/532)
|
||||
|
||||
|
||||
## [0.3.5](https://github.com/apify/crawlee-python/releases/tag/v0.3.5) (2024-09-10)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Memory usage limit configuration via environment variables ([#502](https://github.com/apify/crawlee-python/pull/502)) ([c62e554](https://github.com/apify/crawlee-python/commit/c62e5545de6a1836f0514ebd3dd695e4fd856844)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Http clients detect 4xx as errors by default ([#498](https://github.com/apify/crawlee-python/pull/498)) ([1895dca](https://github.com/apify/crawlee-python/commit/1895dca538f415feca37b4a030525c7c0d32f114)) by [@vdusek](https://github.com/vdusek), closes [#496](https://github.com/apify/crawlee-python/issues/496)
|
||||
- Correctly handle log level configuration ([#508](https://github.com/apify/crawlee-python/pull/508)) ([7ea8fe6](https://github.com/apify/crawlee-python/commit/7ea8fe69f4a6146a1e417bebff60c08a85e2ca27)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
|
||||
## [0.3.4](https://github.com/apify/crawlee-python/releases/tag/v0.3.4) (2024-09-05)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Expose basic crawling context ([#501](https://github.com/apify/crawlee-python/pull/501)) ([b484535](https://github.com/apify/crawlee-python/commit/b484535dbacc5d206a026f55a1d3e58edd375e91)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.3.3](https://github.com/apify/crawlee-python/releases/tag/v0.3.3) (2024-09-05)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Deduplicate requests by unique key before submitting them to the queue ([#499](https://github.com/apify/crawlee-python/pull/499)) ([6a3e0e7](https://github.com/apify/crawlee-python/commit/6a3e0e78490851c43cefb0497ce34ca52a31a25c)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
|
||||
## [0.3.2](https://github.com/apify/crawlee-python/releases/tag/v0.3.2) (2024-09-02)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Double incrementation of `item_count` ([#443](https://github.com/apify/crawlee-python/pull/443)) ([cd9adf1](https://github.com/apify/crawlee-python/commit/cd9adf15731e8c4a39cb142b6d1a62909cafdc51)) by [@cadlagtrader](https://github.com/cadlagtrader), closes [#442](https://github.com/apify/crawlee-python/issues/442)
|
||||
- Field alias in `BatchRequestsOperationResponse` ([#485](https://github.com/apify/crawlee-python/pull/485)) ([126a862](https://github.com/apify/crawlee-python/commit/126a8629cb5b989a0f9fe22156fb09731a34acd2)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- JSON handling with Parsel ([#490](https://github.com/apify/crawlee-python/pull/490)) ([ebf5755](https://github.com/apify/crawlee-python/commit/ebf575539ffb631ae131a1b801cec8f21dd0cf4c)) by [@janbuchar](https://github.com/janbuchar), closes [#488](https://github.com/apify/crawlee-python/issues/488)
|
||||
|
||||
|
||||
## [0.3.1](https://github.com/apify/crawlee-python/releases/tag/v0.3.1) (2024-08-30)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Curl http client selects chrome impersonation by default ([#473](https://github.com/apify/crawlee-python/pull/473)) ([82dc939](https://github.com/apify/crawlee-python/commit/82dc93957b1a380ea975564dea5c6ba4639be548)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.3.0](https://github.com/apify/crawlee-python/releases/tag/v0.3.0) (2024-08-27)
|
||||
|
||||
- Check out the [Upgrading guide](https://crawlee.dev/python/docs/upgrading/upgrading-to-v0x#upgrading-to-v03) to ensure a smooth update.
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Implement ParselCrawler that adds support for Parsel ([#348](https://github.com/apify/crawlee-python/pull/348)) ([a3832e5](https://github.com/apify/crawlee-python/commit/a3832e527f022f32cce4a80055da3b7967b74522)) by [@asymness](https://github.com/asymness), closes [#335](https://github.com/apify/crawlee-python/issues/335)
|
||||
- Add support for filling a web form ([#453](https://github.com/apify/crawlee-python/pull/453)) ([5a125b4](https://github.com/apify/crawlee-python/commit/5a125b464b2619000b92dacad4c3a7faa1869f29)) by [@vdusek](https://github.com/vdusek), closes [#305](https://github.com/apify/crawlee-python/issues/305)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Remove indentation from statistics logging and print the data in tables ([#322](https://github.com/apify/crawlee-python/pull/322)) ([359b515](https://github.com/apify/crawlee-python/commit/359b515d647f064886f91441c2c01d3099e21035)) by [@TymeeK](https://github.com/TymeeK), closes [#306](https://github.com/apify/crawlee-python/issues/306)
|
||||
- Remove redundant log, fix format ([#408](https://github.com/apify/crawlee-python/pull/408)) ([8d27e39](https://github.com/apify/crawlee-python/commit/8d27e3928c605d6eceb51a948453a15024fa2aa2)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Dequeue items from RequestQueue in the correct order ([#411](https://github.com/apify/crawlee-python/pull/411)) ([96fc33e](https://github.com/apify/crawlee-python/commit/96fc33e2cc4631cae3c50dad9eace6407103a2a9)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Relative URLS supports & If not a URL, pass #417 ([#431](https://github.com/apify/crawlee-python/pull/431)) ([ccd8145](https://github.com/apify/crawlee-python/commit/ccd81454166ece68391cdffedb8efe9e663361d9)) by [@black7375](https://github.com/black7375), closes [#417](https://github.com/apify/crawlee-python/issues/417)
|
||||
- Typo in ProlongRequestLockResponse ([#458](https://github.com/apify/crawlee-python/pull/458)) ([30ccc3a](https://github.com/apify/crawlee-python/commit/30ccc3a4763bc3706a3bbeaedc95f9648f5ba09a)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Add missing __all__ to top-level __init__.py file ([#463](https://github.com/apify/crawlee-python/pull/463)) ([353a1ce](https://github.com/apify/crawlee-python/commit/353a1ce28cd38c97ffb36dc1e6b0e86d3aef1a48)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
### 🚜 Refactor
|
||||
|
||||
- [**breaking**] RequestQueue and service management rehaul ([#429](https://github.com/apify/crawlee-python/pull/429)) ([b155a9f](https://github.com/apify/crawlee-python/commit/b155a9f602a163e891777bef5608072fb5d0156f)) by [@janbuchar](https://github.com/janbuchar), closes [#83](https://github.com/apify/crawlee-python/issues/83), [#174](https://github.com/apify/crawlee-python/issues/174), [#203](https://github.com/apify/crawlee-python/issues/203), [#423](https://github.com/apify/crawlee-python/issues/423)
|
||||
- [**breaking**] Declare private and public interface ([#456](https://github.com/apify/crawlee-python/pull/456)) ([d6738df](https://github.com/apify/crawlee-python/commit/d6738df30586934e8d1aba50b9cd437a0ea40400)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.2.1](https://github.com/apify/crawlee-python/releases/tag/v0.2.1) (2024-08-05)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Do not import curl impersonate in http clients init ([#396](https://github.com/apify/crawlee-python/pull/396)) ([3bb8009](https://github.com/apify/crawlee-python/commit/3bb80093e61c1615f869ecd5ab80b061e0e5db36)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.2.0](https://github.com/apify/crawlee-python/releases/tag/v0.2.0) (2024-08-05)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add new curl impersonate HTTP client ([#387](https://github.com/apify/crawlee-python/pull/387)) ([9c06260](https://github.com/apify/crawlee-python/commit/9c06260c0ee958522caa9322001a3186e9e43af4)) by [@vdusek](https://github.com/vdusek), closes [#292](https://github.com/apify/crawlee-python/issues/292)
|
||||
- **playwright:** `infinite_scroll` helper ([#393](https://github.com/apify/crawlee-python/pull/393)) ([34f74bd](https://github.com/apify/crawlee-python/commit/34f74bdcffb42a6c876a856e1c89923d9b3e60bd)) by [@janbuchar](https://github.com/janbuchar)
|
||||
|
||||
|
||||
## [0.1.2](https://github.com/apify/crawlee-python/releases/tag/v0.1.2) (2024-07-30)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Add URL validation ([#343](https://github.com/apify/crawlee-python/pull/343)) ([1514538](https://github.com/apify/crawlee-python/commit/15145388009c85ab54dc72ea8f2d07efd78f80fd)) by [@vdusek](https://github.com/vdusek), closes [#300](https://github.com/apify/crawlee-python/issues/300)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Minor log fix ([#341](https://github.com/apify/crawlee-python/pull/341)) ([0688bf1](https://github.com/apify/crawlee-python/commit/0688bf1860534ab6b2a85dc850bf3d56507ab154)) by [@souravjain540](https://github.com/souravjain540)
|
||||
- Also use error_handler for context pipeline errors ([#331](https://github.com/apify/crawlee-python/pull/331)) ([7a66445](https://github.com/apify/crawlee-python/commit/7a664456b45c7e429b4c90aaf1c09d5796b93e3d)) by [@janbuchar](https://github.com/janbuchar), closes [#296](https://github.com/apify/crawlee-python/issues/296)
|
||||
- Strip whitespace from href in enqueue_links ([#346](https://github.com/apify/crawlee-python/pull/346)) ([8a3174a](https://github.com/apify/crawlee-python/commit/8a3174aed24f9eb4f9ac415a79a58685a081cde2)) by [@janbuchar](https://github.com/janbuchar), closes [#337](https://github.com/apify/crawlee-python/issues/337)
|
||||
- Warn instead of crashing when an empty dataset is being exported ([#342](https://github.com/apify/crawlee-python/pull/342)) ([22b95d1](https://github.com/apify/crawlee-python/commit/22b95d1948d4acd23a010898fa6af2f491e7f514)) by [@janbuchar](https://github.com/janbuchar), closes [#334](https://github.com/apify/crawlee-python/issues/334)
|
||||
- Avoid Github rate limiting in project bootstrapping test ([#364](https://github.com/apify/crawlee-python/pull/364)) ([992f07f](https://github.com/apify/crawlee-python/commit/992f07f266f7b8433d99e9a179f277995f81eb17)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Pass crawler configuration to storages ([#375](https://github.com/apify/crawlee-python/pull/375)) ([b2d3a52](https://github.com/apify/crawlee-python/commit/b2d3a52712abe21f4a4a5db4e20c80afe72c27de)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Purge request queue on repeated crawler runs ([#377](https://github.com/apify/crawlee-python/pull/377)) ([7ad3d69](https://github.com/apify/crawlee-python/commit/7ad3d6908e153c590bff72478af7ee3239a249bc)) by [@janbuchar](https://github.com/janbuchar), closes [#152](https://github.com/apify/crawlee-python/issues/152)
|
||||
|
||||
|
||||
## [0.1.1](https://github.com/apify/crawlee-python/releases/tag/v0.1.1) (2024-07-19)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Expose crawler log ([#316](https://github.com/apify/crawlee-python/pull/316)) ([ae475fa](https://github.com/apify/crawlee-python/commit/ae475fa450c4fe053620d7b7eb475f3d58804674)) by [@vdusek](https://github.com/vdusek), closes [#303](https://github.com/apify/crawlee-python/issues/303)
|
||||
- Integrate proxies into `PlaywrightCrawler` ([#325](https://github.com/apify/crawlee-python/pull/325)) ([2e072b6](https://github.com/apify/crawlee-python/commit/2e072b6ad7d5d82d96a7b489cafb87e7bfaf6e83)) by [@vdusek](https://github.com/vdusek)
|
||||
- Blocking detection for playwright crawler ([#328](https://github.com/apify/crawlee-python/pull/328)) ([49ff6e2](https://github.com/apify/crawlee-python/commit/49ff6e25c12a97550eee718d64bb4130f9990189)) by [@vdusek](https://github.com/vdusek), closes [#239](https://github.com/apify/crawlee-python/issues/239)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Pylance reportPrivateImportUsage errors ([#313](https://github.com/apify/crawlee-python/pull/313)) ([09d7203](https://github.com/apify/crawlee-python/commit/09d72034d5db8c47f461111ec093761935a3e2ef)) by [@vdusek](https://github.com/vdusek), closes [#283](https://github.com/apify/crawlee-python/issues/283)
|
||||
- Set httpx logging to warning ([#314](https://github.com/apify/crawlee-python/pull/314)) ([1585def](https://github.com/apify/crawlee-python/commit/1585defffb2c0c844fab39bbc0e0b793d6169cbf)) by [@vdusek](https://github.com/vdusek), closes [#302](https://github.com/apify/crawlee-python/issues/302)
|
||||
- Byte size serialization in MemoryInfo ([#245](https://github.com/apify/crawlee-python/pull/245)) ([a030174](https://github.com/apify/crawlee-python/commit/a0301746c2df076d281708344fb906e1c42e0790)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Project bootstrapping in existing folder ([#318](https://github.com/apify/crawlee-python/pull/318)) ([c630818](https://github.com/apify/crawlee-python/commit/c630818538e0c37217ab73f6c6da05505ed8b364)) by [@janbuchar](https://github.com/janbuchar), closes [#301](https://github.com/apify/crawlee-python/issues/301)
|
||||
|
||||
|
||||
## [0.1.0](https://github.com/apify/crawlee-python/releases/tag/v0.1.0) (2024-07-08)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Project templates ([#237](https://github.com/apify/crawlee-python/pull/237)) ([c23c12c](https://github.com/apify/crawlee-python/commit/c23c12c66688f825f74deb39702f07cc6c6bbc46)) by [@janbuchar](https://github.com/janbuchar), closes [#215](https://github.com/apify/crawlee-python/issues/215)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- CLI UX improvements ([#271](https://github.com/apify/crawlee-python/pull/271)) ([123d515](https://github.com/apify/crawlee-python/commit/123d515b224c663577bfe0fab387d0aa11e5e4d4)) by [@janbuchar](https://github.com/janbuchar), closes [#267](https://github.com/apify/crawlee-python/issues/267)
|
||||
- Error handling in CLI and templates documentation ([#273](https://github.com/apify/crawlee-python/pull/273)) ([61083c3](https://github.com/apify/crawlee-python/commit/61083c33434d431a118538f15bfa9a68c312ab03)) by [@vdusek](https://github.com/vdusek), closes [#268](https://github.com/apify/crawlee-python/issues/268)
|
||||
|
||||
|
||||
## [0.0.7](https://github.com/apify/crawlee-python/releases/tag/v0.0.7) (2024-06-27)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Do not wait for consistency in request queue ([#235](https://github.com/apify/crawlee-python/pull/235)) ([03ff138](https://github.com/apify/crawlee-python/commit/03ff138aadaf8e915abc7fafb854fe12947b9696)) by [@vdusek](https://github.com/vdusek)
|
||||
- Selector handling in BeautifulSoupCrawler enqueue_links ([#231](https://github.com/apify/crawlee-python/pull/231)) ([896501e](https://github.com/apify/crawlee-python/commit/896501edb44f801409fec95cb3e5f2bcfcb4188d)) by [@janbuchar](https://github.com/janbuchar), closes [#230](https://github.com/apify/crawlee-python/issues/230)
|
||||
- Handle blocked request ([#234](https://github.com/apify/crawlee-python/pull/234)) ([f8ef79f](https://github.com/apify/crawlee-python/commit/f8ef79ffcb7410713182af716d37dbbaad66fdbc)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- Improve AutoscaledPool state management ([#241](https://github.com/apify/crawlee-python/pull/241)) ([fdea3d1](https://github.com/apify/crawlee-python/commit/fdea3d16b13afe70039d864de861486c760aa0ba)) by [@janbuchar](https://github.com/janbuchar), closes [#236](https://github.com/apify/crawlee-python/issues/236)
|
||||
|
||||
|
||||
## [0.0.6](https://github.com/apify/crawlee-python/releases/tag/v0.0.6) (2024-06-25)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Maintain a global configuration instance ([#207](https://github.com/apify/crawlee-python/pull/207)) ([e003aa6](https://github.com/apify/crawlee-python/commit/e003aa63d859bec8199d0c890b5c9604f163ccd3)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Add max requests per crawl to `BasicCrawler` ([#198](https://github.com/apify/crawlee-python/pull/198)) ([b5b3053](https://github.com/apify/crawlee-python/commit/b5b3053f43381601274e4034d07b4bf41720c7c2)) by [@vdusek](https://github.com/vdusek)
|
||||
- Add support decompress *br* response content ([#226](https://github.com/apify/crawlee-python/pull/226)) ([a3547b9](https://github.com/apify/crawlee-python/commit/a3547b9c882dc5333a4fcd1223687ef85e79138d)) by [@Mantisus](https://github.com/Mantisus)
|
||||
- BasicCrawler.export_data helper ([#222](https://github.com/apify/crawlee-python/pull/222)) ([237ec78](https://github.com/apify/crawlee-python/commit/237ec789b7dccc17cc57ef47ec56bcf73c6ca006)) by [@janbuchar](https://github.com/janbuchar), closes [#211](https://github.com/apify/crawlee-python/issues/211)
|
||||
- Automatic logging setup ([#229](https://github.com/apify/crawlee-python/pull/229)) ([a67b72f](https://github.com/apify/crawlee-python/commit/a67b72faacd75674071bae496d59e1c60636350c)) by [@janbuchar](https://github.com/janbuchar), closes [#214](https://github.com/apify/crawlee-python/issues/214)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Handling of relative URLs in add_requests ([#213](https://github.com/apify/crawlee-python/pull/213)) ([8aa8c57](https://github.com/apify/crawlee-python/commit/8aa8c57f44149caa0e01950a5d773726f261699a)) by [@janbuchar](https://github.com/janbuchar), closes [#202](https://github.com/apify/crawlee-python/issues/202), [#204](https://github.com/apify/crawlee-python/issues/204)
|
||||
- Graceful exit in BasicCrawler.run ([#224](https://github.com/apify/crawlee-python/pull/224)) ([337286e](https://github.com/apify/crawlee-python/commit/337286e1b721cf61f57bc0ff3ead08df1f4f5448)) by [@janbuchar](https://github.com/janbuchar), closes [#212](https://github.com/apify/crawlee-python/issues/212)
|
||||
|
||||
|
||||
## [0.0.5](https://github.com/apify/crawlee-python/releases/tag/v0.0.5) (2024-06-21)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Browser rotation and better browser abstraction ([#177](https://github.com/apify/crawlee-python/pull/177)) ([a42ae6f](https://github.com/apify/crawlee-python/commit/a42ae6f53c5e24678f04011c3684290b68684016)) by [@vdusek](https://github.com/vdusek), closes [#131](https://github.com/apify/crawlee-python/issues/131)
|
||||
- Add emit persist state event to event manager ([#181](https://github.com/apify/crawlee-python/pull/181)) ([97f6c68](https://github.com/apify/crawlee-python/commit/97f6c68275b65f76c62b6d16d94354fc7f00d336)) by [@vdusek](https://github.com/vdusek)
|
||||
- Batched request addition in RequestQueue ([#186](https://github.com/apify/crawlee-python/pull/186)) ([f48c806](https://github.com/apify/crawlee-python/commit/f48c8068fe16ce3dd4c46fc248733346c0621411)) by [@vdusek](https://github.com/vdusek)
|
||||
- Add storage helpers to crawler & context ([#192](https://github.com/apify/crawlee-python/pull/192)) ([f8f4066](https://github.com/apify/crawlee-python/commit/f8f4066d8b32d6e7dc0d999a5aa8db75f99b43b8)) by [@vdusek](https://github.com/vdusek), closes [#98](https://github.com/apify/crawlee-python/issues/98), [#100](https://github.com/apify/crawlee-python/issues/100), [#172](https://github.com/apify/crawlee-python/issues/172)
|
||||
- Handle all supported configuration options ([#199](https://github.com/apify/crawlee-python/pull/199)) ([23c901c](https://github.com/apify/crawlee-python/commit/23c901cd68cf14b4041ee03568622ee32822e94b)) by [@janbuchar](https://github.com/janbuchar), closes [#84](https://github.com/apify/crawlee-python/issues/84)
|
||||
- Add Playwright's enqueue links helper ([#196](https://github.com/apify/crawlee-python/pull/196)) ([849d73c](https://github.com/apify/crawlee-python/commit/849d73cc7d137171b98f9f2ab85374e8beec0dad)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- Tmp path in tests is working ([#164](https://github.com/apify/crawlee-python/pull/164)) ([382b6f4](https://github.com/apify/crawlee-python/commit/382b6f48174bdac3931cc379eaf770ab06f826dc)) by [@vdusek](https://github.com/vdusek), closes [#159](https://github.com/apify/crawlee-python/issues/159)
|
||||
- Add explicit err msgs for missing pckg extras during import ([#165](https://github.com/apify/crawlee-python/pull/165)) ([200ebfa](https://github.com/apify/crawlee-python/commit/200ebfa63d6e20e17c8ca29544ef7229ed0df308)) by [@vdusek](https://github.com/vdusek), closes [#155](https://github.com/apify/crawlee-python/issues/155)
|
||||
- Make timedelta_ms accept string-encoded numbers ([#190](https://github.com/apify/crawlee-python/pull/190)) ([d8426ff](https://github.com/apify/crawlee-python/commit/d8426ff41e36f701af459ad17552fee39637674d)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- **deps:** Update dependency psutil to v6 ([#193](https://github.com/apify/crawlee-python/pull/193)) ([eb91f51](https://github.com/apify/crawlee-python/commit/eb91f51e19da406e3f9293e5336c1f85fc7885a4)) by [@renovate[bot]](https://github.com/renovate[bot])
|
||||
- Improve compatibility between ProxyConfiguration and its SDK counterpart ([#201](https://github.com/apify/crawlee-python/pull/201)) ([1a76124](https://github.com/apify/crawlee-python/commit/1a76124080d561e0153a4dda0bdb0d9863c3aab6)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Correct return type of storage get_info methods ([#200](https://github.com/apify/crawlee-python/pull/200)) ([332673c](https://github.com/apify/crawlee-python/commit/332673c4fb519b80846df7fb8cd8bb521538a8a4)) by [@janbuchar](https://github.com/janbuchar)
|
||||
- Type error in statistics persist state ([#206](https://github.com/apify/crawlee-python/pull/206)) ([96ceef6](https://github.com/apify/crawlee-python/commit/96ceef697769cd57bd1a50b6615cf1e70549bd2d)) by [@vdusek](https://github.com/vdusek), closes [#194](https://github.com/apify/crawlee-python/issues/194)
|
||||
|
||||
|
||||
## [0.0.4](https://github.com/apify/crawlee-python/releases/tag/v0.0.4) (2024-05-30)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- Capture statistics about the crawler run ([#142](https://github.com/apify/crawlee-python/pull/142)) ([eeebe9b](https://github.com/apify/crawlee-python/commit/eeebe9b1e24338d68a0a55228bbfc717f4d9d295)) by [@janbuchar](https://github.com/janbuchar), closes [#97](https://github.com/apify/crawlee-python/issues/97)
|
||||
- Proxy configuration ([#156](https://github.com/apify/crawlee-python/pull/156)) ([5c3753a](https://github.com/apify/crawlee-python/commit/5c3753a5527b1d01f7260b9e4c566e43f956a5e8)) by [@janbuchar](https://github.com/janbuchar), closes [#136](https://github.com/apify/crawlee-python/issues/136)
|
||||
- Add first version of browser pool and playwright crawler ([#161](https://github.com/apify/crawlee-python/pull/161)) ([2d2a050](https://github.com/apify/crawlee-python/commit/2d2a0505b1c2b1529a8835163ca97d1ec2a6e44a)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
## [0.0.3](https://github.com/apify/crawlee-python/releases/tag/v0.0.3) (2024-05-13)
|
||||
|
||||
### 🚀 Features
|
||||
|
||||
- AutoscaledPool implementation ([#55](https://github.com/apify/crawlee-python/pull/55)) ([621ada2](https://github.com/apify/crawlee-python/commit/621ada2bd1ba4e2346fb948dc02686e2b37e3856)) by [@janbuchar](https://github.com/janbuchar), closes [#19](https://github.com/apify/crawlee-python/issues/19)
|
||||
- Add Snapshotter ([#20](https://github.com/apify/crawlee-python/pull/20)) ([492ee38](https://github.com/apify/crawlee-python/commit/492ee38c893b8f54e9583dd492576c5106e29881)) by [@vdusek](https://github.com/vdusek)
|
||||
- Implement BasicCrawler ([#56](https://github.com/apify/crawlee-python/pull/56)) ([6da971f](https://github.com/apify/crawlee-python/commit/6da971fcddbf8b6795346c88e295dada28e7b1d3)) by [@janbuchar](https://github.com/janbuchar), closes [#30](https://github.com/apify/crawlee-python/issues/30)
|
||||
- BeautifulSoupCrawler ([#107](https://github.com/apify/crawlee-python/pull/107)) ([4974dfa](https://github.com/apify/crawlee-python/commit/4974dfa20c7911ee073438fd388e60ba4b2c07db)) by [@janbuchar](https://github.com/janbuchar), closes [#31](https://github.com/apify/crawlee-python/issues/31)
|
||||
- Add_requests and enqueue_links context helpers ([#120](https://github.com/apify/crawlee-python/pull/120)) ([dc850a5](https://github.com/apify/crawlee-python/commit/dc850a5778b105ff09e19eaecbb0a12d94798a62)) by [@janbuchar](https://github.com/janbuchar), closes [#5](https://github.com/apify/crawlee-python/issues/5)
|
||||
- Use SessionPool in BasicCrawler ([#128](https://github.com/apify/crawlee-python/pull/128)) ([9fc4648](https://github.com/apify/crawlee-python/commit/9fc464837e596b3b5a7cd818b6d617550e249352)) by [@janbuchar](https://github.com/janbuchar), closes [#110](https://github.com/apify/crawlee-python/issues/110)
|
||||
- Add base storage client and resource subclients ([#138](https://github.com/apify/crawlee-python/pull/138)) ([44d6597](https://github.com/apify/crawlee-python/commit/44d65974e4837576918069d7e63f8b804964971a)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
### 🐛 Bug Fixes
|
||||
|
||||
- **deps:** Update dependency docutils to ^0.21.0 ([#101](https://github.com/apify/crawlee-python/pull/101)) ([534b613](https://github.com/apify/crawlee-python/commit/534b613f7cdfe7adf38b548ee48537db3167d1ec)) by [@renovate[bot]](https://github.com/renovate[bot])
|
||||
- **deps:** Update dependency eval-type-backport to ^0.2.0 ([#124](https://github.com/apify/crawlee-python/pull/124)) ([c9e69a8](https://github.com/apify/crawlee-python/commit/c9e69a8534f4d82d9a6314947d76a86bcb744607)) by [@renovate[bot]](https://github.com/renovate[bot])
|
||||
- Fire local SystemInfo events every second ([#144](https://github.com/apify/crawlee-python/pull/144)) ([f1359fa](https://github.com/apify/crawlee-python/commit/f1359fa7eea23f8153ad711287c073e45d498401)) by [@vdusek](https://github.com/vdusek)
|
||||
- Storage manager & purging the defaults ([#150](https://github.com/apify/crawlee-python/pull/150)) ([851042f](https://github.com/apify/crawlee-python/commit/851042f25ad07e25651768e476f098ef0ed21914)) by [@vdusek](https://github.com/vdusek)
|
||||
|
||||
|
||||
<!-- generated by git-cliff -->
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
# Coding guidelines
|
||||
|
||||
This file provides guidance to programming agents when working with code in this repository.
|
||||
|
||||
## Development Commands
|
||||
|
||||
All commands use `uv` (package manager) and `poe` (task runner):
|
||||
|
||||
```bash
|
||||
# Install all dependencies (dev + extras + pre-commit + playwright)
|
||||
uv run poe install-dev
|
||||
|
||||
# Run full check suite (lint + type-check + unit tests)
|
||||
uv run poe check-code
|
||||
|
||||
# Linting (ruff format check + ruff check)
|
||||
uv run poe lint
|
||||
|
||||
# Auto-fix formatting
|
||||
uv run poe format
|
||||
|
||||
# Type checking (ty)
|
||||
uv run poe type-check
|
||||
|
||||
# Run all unit tests
|
||||
uv run poe unit-tests
|
||||
|
||||
# Run a single test file
|
||||
uv run pytest tests/unit/path/to/test_file.py
|
||||
|
||||
# Run a single test by name
|
||||
uv run pytest tests/unit/path/to/test_file.py::test_name -v
|
||||
|
||||
# Run tests with coverage XML report
|
||||
uv run poe unit-tests-cov
|
||||
|
||||
# Build package
|
||||
uv run poe build
|
||||
|
||||
# Clean build artifacts
|
||||
uv run poe clean
|
||||
```
|
||||
|
||||
Note: `uv run poe unit-tests` first runs tests marked `@pytest.mark.run_alone` in isolation, then runs the rest with `-x` (fail-fast) and parallelism via `pytest-xdist`.
|
||||
|
||||
## Code Style
|
||||
|
||||
- **Linter/formatter**: Ruff with `select = ["ALL"]` and specific ignores
|
||||
- **Line length**: 120 characters
|
||||
- **Quotes**: Single quotes (double for docstrings)
|
||||
- **Docstrings**: Google format (enforced by Ruff)
|
||||
- **Type checker**: ty (Astral's type checker), target Python 3.10
|
||||
- **Async mode**: pytest-asyncio in `auto` mode (no need for `@pytest.mark.asyncio`)
|
||||
- **Commits**: [Conventional Commits](https://www.conventionalcommits.org/) format. Choose the type based on *what* changed, not just *why*:
|
||||
- `feat:` / `fix:` / `perf:` / `refactor:` / `style:` — **source code only**; these trigger a release and appear in the changelog
|
||||
- `test:` — test additions or changes (no release triggered)
|
||||
- `docs:` — documentation changes; also triggers a doc release on master
|
||||
- `ci:` — CI/workflow changes
|
||||
- `chore:` — dependency bumps, tooling, and other housekeeping
|
||||
- `build:` — build system changes
|
||||
|
||||
## Architecture
|
||||
|
||||
### Crawler Hierarchy
|
||||
|
||||
```
|
||||
BasicCrawler[TCrawlingContext, TStatisticsState]
|
||||
├── AbstractHttpCrawler → HttpCrawler, BeautifulSoupCrawler, ParselCrawler
|
||||
├── PlaywrightCrawler
|
||||
└── AdaptivePlaywrightCrawler (extends PlaywrightCrawler)
|
||||
```
|
||||
|
||||
- **BasicCrawler** (`src/crawlee/crawlers/_basic/`): Core request lifecycle, autoscaling pool, retries, session management, router dispatch. Generic over `TCrawlingContext`.
|
||||
- **AbstractHttpCrawler** (`src/crawlee/crawlers/_abstract_http/`): Adds HTTP client integration, response parsing, pre-navigation hooks. Generic over parser result type.
|
||||
- **PlaywrightCrawler** (`src/crawlee/crawlers/_playwright/`): Browser-based crawling with Playwright.
|
||||
|
||||
### Context Pipeline (Middleware Pattern)
|
||||
|
||||
Contexts are progressively enhanced through `ContextPipeline` middleware:
|
||||
|
||||
```
|
||||
BasicCrawlingContext → HttpCrawlingContext → ParsedHttpCrawlingContext → BeautifulSoupCrawlingContext
|
||||
```
|
||||
|
||||
Each middleware is an async generator that wraps the next handler, enabling setup/teardown around request processing.
|
||||
|
||||
### Storage Layer
|
||||
|
||||
Three-tier design:
|
||||
- **High-level**: `Dataset`, `KeyValueStore`, `RequestQueue` in `src/crawlee/storages/`
|
||||
- **Storage clients** (`src/crawlee/storage_clients/`): `FileSystemStorageClient` (default), `MemoryStorageClient`, `SqlStorageClient`, `RedisStorageClient`
|
||||
- **Instance caching**: `StorageInstanceManager` is a global singleton that caches storage instances by ID/name
|
||||
|
||||
### Service Locator
|
||||
|
||||
`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageClient`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`.
|
||||
|
||||
### HTTP Clients
|
||||
|
||||
Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`:
|
||||
- `ImpitHttpClient` (default), `HttpxHttpClient`, `CurlImpersonateHttpClient`
|
||||
- Each provides `crawl()` (for crawler pipeline) and `send_request()` (for in-handler use)
|
||||
|
||||
### Request Model
|
||||
|
||||
`Request` (`src/crawlee/_request.py`) uses `unique_key` for deduplication. Lifecycle states: `UNPROCESSED → DONE`. Crawlee-specific metadata stored in `user_data['__crawlee']`.
|
||||
|
||||
### Router
|
||||
|
||||
```python
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BeautifulSoupCrawlingContext): ...
|
||||
|
||||
@crawler.router.handler(label='detail')
|
||||
async def detail(context: BeautifulSoupCrawlingContext): ...
|
||||
```
|
||||
|
||||
Requests are routed by their `label` field; unmatched requests go to the default handler.
|
||||
|
||||
### Key Directories
|
||||
|
||||
- `src/crawlee/crawlers/` - All crawler implementations
|
||||
- `src/crawlee/storages/` - Dataset, KVS, RequestQueue
|
||||
- `src/crawlee/storage_clients/` - Backend implementations
|
||||
- `src/crawlee/http_clients/` - HTTP client implementations
|
||||
- `src/crawlee/browsers/` - Playwright browser pool and plugins
|
||||
- `src/crawlee/sessions/` - Session management with cookie persistence
|
||||
- `src/crawlee/events/` - Event system (persist state, progress, aborting)
|
||||
- `src/crawlee/_autoscaling/` - Autoscaled pool for concurrency control
|
||||
- `src/crawlee/fingerprint_suite/` - Anti-bot fingerprint generation
|
||||
- `src/crawlee/project_template/` - CLI scaffolding template (excluded from linting)
|
||||
- `tests/unit/` - Unit tests
|
||||
- `tests/e2e/` - End-to-end tests (require `apify-cli` + API token)
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
# Development
|
||||
|
||||
Here you'll find a contributing guide to get started with development.
|
||||
|
||||
## Environment
|
||||
|
||||
For local development, it is required to have Python 3.10 (or a later version) installed.
|
||||
|
||||
We use [uv](https://docs.astral.sh/uv/) for project management. Install it and set up your IDE accordingly.
|
||||
|
||||
We use [Poe the Poet](https://poethepoet.natn.io/) as a task runner, similar to npm scripts in `package.json`.
|
||||
All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be run with `uv run poe <task>`.
|
||||
|
||||
### Available tasks
|
||||
|
||||
| Task | Description |
|
||||
| ---- | ----------- |
|
||||
| `install-dev` | Install development dependencies |
|
||||
| `check-code` | Run lint, type-check, and unit-tests |
|
||||
| `lint` | Run linter |
|
||||
| `format` | Fix lint issues and format code |
|
||||
| `type-check` | Run type checker |
|
||||
| `unit-tests` | Run unit tests |
|
||||
| `unit-tests-cov` | Run unit tests with coverage |
|
||||
| `e2e-templates-tests` | Run end-to-end template tests |
|
||||
| `build-docs` | Build documentation website |
|
||||
| `run-docs` | Run documentation website locally |
|
||||
| `build` | Build package |
|
||||
| `clean` | Remove build artifacts and clean caches |
|
||||
|
||||
## Dependencies
|
||||
|
||||
To install this package and its development dependencies, run:
|
||||
|
||||
```sh
|
||||
uv run poe install-dev
|
||||
```
|
||||
|
||||
## Code checking
|
||||
|
||||
To execute all code checking tools together, run:
|
||||
|
||||
```sh
|
||||
uv run poe check-code
|
||||
```
|
||||
|
||||
### Linting
|
||||
|
||||
We utilize [ruff](https://docs.astral.sh/ruff/) for linting, which analyzes code for potential issues and enforces consistent style. Refer to `pyproject.toml` for configuration details.
|
||||
|
||||
To run linting:
|
||||
|
||||
```sh
|
||||
uv run poe lint
|
||||
```
|
||||
|
||||
### Formatting
|
||||
|
||||
Our automated code formatting also leverages [ruff](https://docs.astral.sh/ruff/), ensuring uniform style and addressing fixable linting issues. Configuration specifics are outlined in `pyproject.toml`.
|
||||
|
||||
To run formatting:
|
||||
|
||||
```sh
|
||||
uv run poe format
|
||||
```
|
||||
|
||||
### Type checking
|
||||
|
||||
Type checking is handled by [ty](https://docs.astral.sh/ty/), verifying code against type annotations. Configuration settings can be found in `pyproject.toml`.
|
||||
|
||||
To run type checking:
|
||||
|
||||
```sh
|
||||
uv run poe type-check
|
||||
```
|
||||
|
||||
### Unit tests
|
||||
|
||||
We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. Check `pyproject.toml` for configuration details and installed plugins.
|
||||
|
||||
To run unit tests:
|
||||
|
||||
```sh
|
||||
uv run poe unit-tests
|
||||
```
|
||||
|
||||
To run unit tests with coverage report:
|
||||
|
||||
```sh
|
||||
uv run poe unit-tests-cov
|
||||
```
|
||||
|
||||
## End-to-end tests
|
||||
|
||||
Prerequisites:
|
||||
|
||||
- [apify-cli](https://docs.apify.com/cli/docs/installation) installed and available in `PATH`
|
||||
- Set `APIFY_TEST_USER_API_TOKEN` to your [Apify API token](https://docs.apify.com/platform/integrations/api#api-token)
|
||||
|
||||
To run end-to-end tests:
|
||||
|
||||
```sh
|
||||
uv run poe e2e-templates-tests
|
||||
```
|
||||
|
||||
## Documentation
|
||||
|
||||
We follow the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for code documentation. All user-facing classes and functions must be documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/).
|
||||
|
||||
Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with custom post-processing. Additional content is provided through markdown files in the `docs/` directory. The final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages.
|
||||
|
||||
To run the documentation locally, ensure you have `Node.js` 20+ installed, then run:
|
||||
|
||||
```sh
|
||||
uv run poe run-docs
|
||||
```
|
||||
|
||||
## Commits
|
||||
|
||||
We use [Conventional Commits](https://www.conventionalcommits.org/) format for commit messages. This convention is used to automatically determine version bumps during the release process.
|
||||
|
||||
### Available commit types
|
||||
|
||||
| Type | Description |
|
||||
| ---- | ----------- |
|
||||
| `feat` | A new feature |
|
||||
| `fix` | A bug fix |
|
||||
| `docs` | Documentation only changes |
|
||||
| `style` | Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) |
|
||||
| `refactor` | A code change that neither fixes a bug nor adds a feature |
|
||||
| `perf` | A code change that improves performance |
|
||||
| `test` | Adding missing tests or correcting existing tests |
|
||||
| `build` | Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) |
|
||||
| `ci` | Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) |
|
||||
| `chore` | Other changes that don't modify src or test files |
|
||||
| `revert` | Reverts a previous commit |
|
||||
|
||||
## Release process
|
||||
|
||||
Publishing new versions to [PyPI](https://pypi.org/project/crawlee) is automated through GitHub Actions.
|
||||
|
||||
- **Beta releases**: On each commit to the master branch, a new beta release is automatically published. The version number is determined based on the latest release and conventional commits. The beta version suffix is incremented by 1 from the last beta release on PyPI.
|
||||
- **Stable releases**: A stable version release may be created by triggering the `release` GitHub Actions workflow. The version number is determined based on the latest release and conventional commits (`auto` release type), or it may be overridden using the `custom` release type.
|
||||
|
||||
### Publishing to PyPI manually
|
||||
|
||||
1. **Do not do this unless absolutely necessary.** In all conceivable scenarios, you should use the `release` workflow instead.
|
||||
2. **Make sure you know what you're doing.**
|
||||
|
||||
3. Update the version number:
|
||||
|
||||
- Modify the `version` field under `project` in `pyproject.toml`.
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "crawlee"
|
||||
version = "x.z.y"
|
||||
```
|
||||
|
||||
4. Build the package:
|
||||
|
||||
```sh
|
||||
uv run poe build
|
||||
```
|
||||
|
||||
5. Upload to PyPI:
|
||||
|
||||
```sh
|
||||
uv publish --token YOUR_API_TOKEN
|
||||
```
|
||||
|
|
@ -0,0 +1,133 @@
|
|||
# Coding guidelines
|
||||
|
||||
This file provides guidance to programming agents when working with code in this repository.
|
||||
|
||||
## Development Commands
|
||||
|
||||
All commands use `uv` (package manager) and `poe` (task runner):
|
||||
|
||||
```bash
|
||||
# Install all dependencies (dev + extras + pre-commit + playwright)
|
||||
uv run poe install-dev
|
||||
|
||||
# Run full check suite (lint + type-check + unit tests)
|
||||
uv run poe check-code
|
||||
|
||||
# Linting (ruff format check + ruff check)
|
||||
uv run poe lint
|
||||
|
||||
# Auto-fix formatting
|
||||
uv run poe format
|
||||
|
||||
# Type checking (ty)
|
||||
uv run poe type-check
|
||||
|
||||
# Run all unit tests
|
||||
uv run poe unit-tests
|
||||
|
||||
# Run a single test file
|
||||
uv run pytest tests/unit/path/to/test_file.py
|
||||
|
||||
# Run a single test by name
|
||||
uv run pytest tests/unit/path/to/test_file.py::test_name -v
|
||||
|
||||
# Run tests with coverage XML report
|
||||
uv run poe unit-tests-cov
|
||||
|
||||
# Build package
|
||||
uv run poe build
|
||||
|
||||
# Clean build artifacts
|
||||
uv run poe clean
|
||||
```
|
||||
|
||||
Note: `uv run poe unit-tests` first runs tests marked `@pytest.mark.run_alone` in isolation, then runs the rest with `-x` (fail-fast) and parallelism via `pytest-xdist`.
|
||||
|
||||
## Code Style
|
||||
|
||||
- **Linter/formatter**: Ruff with `select = ["ALL"]` and specific ignores
|
||||
- **Line length**: 120 characters
|
||||
- **Quotes**: Single quotes (double for docstrings)
|
||||
- **Docstrings**: Google format (enforced by Ruff)
|
||||
- **Type checker**: ty (Astral's type checker), target Python 3.10
|
||||
- **Async mode**: pytest-asyncio in `auto` mode (no need for `@pytest.mark.asyncio`)
|
||||
- **Commits**: [Conventional Commits](https://www.conventionalcommits.org/) format. Choose the type based on *what* changed, not just *why*:
|
||||
- `feat:` / `fix:` / `perf:` / `refactor:` / `style:` — **source code only**; these trigger a release and appear in the changelog
|
||||
- `test:` — test additions or changes (no release triggered)
|
||||
- `docs:` — documentation changes; also triggers a doc release on master
|
||||
- `ci:` — CI/workflow changes
|
||||
- `chore:` — dependency bumps, tooling, and other housekeeping
|
||||
- `build:` — build system changes
|
||||
|
||||
## Architecture
|
||||
|
||||
### Crawler Hierarchy
|
||||
|
||||
```
|
||||
BasicCrawler[TCrawlingContext, TStatisticsState]
|
||||
├── AbstractHttpCrawler → HttpCrawler, BeautifulSoupCrawler, ParselCrawler
|
||||
├── PlaywrightCrawler
|
||||
└── AdaptivePlaywrightCrawler (extends PlaywrightCrawler)
|
||||
```
|
||||
|
||||
- **BasicCrawler** (`src/crawlee/crawlers/_basic/`): Core request lifecycle, autoscaling pool, retries, session management, router dispatch. Generic over `TCrawlingContext`.
|
||||
- **AbstractHttpCrawler** (`src/crawlee/crawlers/_abstract_http/`): Adds HTTP client integration, response parsing, pre-navigation hooks. Generic over parser result type.
|
||||
- **PlaywrightCrawler** (`src/crawlee/crawlers/_playwright/`): Browser-based crawling with Playwright.
|
||||
|
||||
### Context Pipeline (Middleware Pattern)
|
||||
|
||||
Contexts are progressively enhanced through `ContextPipeline` middleware:
|
||||
|
||||
```
|
||||
BasicCrawlingContext → HttpCrawlingContext → ParsedHttpCrawlingContext → BeautifulSoupCrawlingContext
|
||||
```
|
||||
|
||||
Each middleware is an async generator that wraps the next handler, enabling setup/teardown around request processing.
|
||||
|
||||
### Storage Layer
|
||||
|
||||
Three-tier design:
|
||||
- **High-level**: `Dataset`, `KeyValueStore`, `RequestQueue` in `src/crawlee/storages/`
|
||||
- **Storage clients** (`src/crawlee/storage_clients/`): `FileSystemStorageClient` (default), `MemoryStorageClient`, `SqlStorageClient`, `RedisStorageClient`
|
||||
- **Instance caching**: `StorageInstanceManager` is a global singleton that caches storage instances by ID/name
|
||||
|
||||
### Service Locator
|
||||
|
||||
`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageClient`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`.
|
||||
|
||||
### HTTP Clients
|
||||
|
||||
Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`:
|
||||
- `ImpitHttpClient` (default), `HttpxHttpClient`, `CurlImpersonateHttpClient`
|
||||
- Each provides `crawl()` (for crawler pipeline) and `send_request()` (for in-handler use)
|
||||
|
||||
### Request Model
|
||||
|
||||
`Request` (`src/crawlee/_request.py`) uses `unique_key` for deduplication. Lifecycle states: `UNPROCESSED → DONE`. Crawlee-specific metadata stored in `user_data['__crawlee']`.
|
||||
|
||||
### Router
|
||||
|
||||
```python
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BeautifulSoupCrawlingContext): ...
|
||||
|
||||
@crawler.router.handler(label='detail')
|
||||
async def detail(context: BeautifulSoupCrawlingContext): ...
|
||||
```
|
||||
|
||||
Requests are routed by their `label` field; unmatched requests go to the default handler.
|
||||
|
||||
### Key Directories
|
||||
|
||||
- `src/crawlee/crawlers/` - All crawler implementations
|
||||
- `src/crawlee/storages/` - Dataset, KVS, RequestQueue
|
||||
- `src/crawlee/storage_clients/` - Backend implementations
|
||||
- `src/crawlee/http_clients/` - HTTP client implementations
|
||||
- `src/crawlee/browsers/` - Playwright browser pool and plugins
|
||||
- `src/crawlee/sessions/` - Session management with cookie persistence
|
||||
- `src/crawlee/events/` - Event system (persist state, progress, aborting)
|
||||
- `src/crawlee/_autoscaling/` - Autoscaled pool for concurrency control
|
||||
- `src/crawlee/fingerprint_suite/` - Anti-bot fingerprint generation
|
||||
- `src/crawlee/project_template/` - CLI scaffolding template (excluded from linting)
|
||||
- `tests/unit/` - Unit tests
|
||||
- `tests/e2e/` - End-to-end tests (require `apify-cli` + API token)
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2023 Apify Technologies s.r.o.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
<h1 align="center">
|
||||
<a href="https://crawlee.dev">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee-python/master/website/static/img/crawlee-dark.svg?sanitize=true">
|
||||
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee-python/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
|
||||
</picture>
|
||||
</a>
|
||||
<br>
|
||||
<small>A web scraping and browser automation library</small>
|
||||
</h1>
|
||||
|
||||
<p align=center>
|
||||
<a href="https://trendshift.io/repositories/11169" target="_blank"><img src="https://trendshift.io/api/badge/repositories/11169" alt="apify%2Fcrawlee-python | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://badge.fury.io/py/crawlee" rel="nofollow"><img src="https://badge.fury.io/py/crawlee.svg" alt="PyPI package version"></a>
|
||||
<a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/dm/crawlee" alt="PyPI package downloads"></a>
|
||||
<a href="https://codecov.io/gh/apify/crawlee-python"><img src="https://codecov.io/gh/apify/crawlee-python/graph/badge.svg?token=cCju61iPQG" alt="Codecov report"></a>
|
||||
<a href="https://pypi.org/project/crawlee/" rel="nofollow"><img src="https://img.shields.io/pypi/pyversions/crawlee" alt="PyPI Python version"></a>
|
||||
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on Discord"></a>
|
||||
</p>
|
||||
|
||||
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
||||
|
||||
Your crawlers will appear almost human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it.
|
||||
|
||||
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev/python/)** 👈
|
||||
|
||||
We also have a TypeScript implementation of the Crawlee, which you can explore and utilize for your projects. Visit our GitHub repository for more information [Crawlee for JS/TS on GitHub](https://github.com/apify/crawlee).
|
||||
|
||||
## Installation
|
||||
|
||||
We recommend visiting the [Introduction tutorial](https://crawlee.dev/python/docs/introduction) in Crawlee documentation for more information.
|
||||
|
||||
Crawlee is available as [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. This package includes the core functionality, while additional features are available as optional extras to keep dependencies and package size minimal.
|
||||
|
||||
To install Crawlee with all features, run the following command:
|
||||
|
||||
```sh
|
||||
python -m pip install 'crawlee[all]'
|
||||
```
|
||||
|
||||
Then, install the [Playwright](https://playwright.dev/) dependencies:
|
||||
|
||||
```sh
|
||||
playwright install
|
||||
```
|
||||
|
||||
Verify that Crawlee is successfully installed:
|
||||
|
||||
```sh
|
||||
python -c 'import crawlee; print(crawlee.__version__)'
|
||||
```
|
||||
|
||||
For detailed installation instructions see the [Setting up](https://crawlee.dev/python/docs/introduction/setting-up) documentation page.
|
||||
|
||||
### With Crawlee CLI
|
||||
|
||||
The quickest way to get started with Crawlee is by using the Crawlee CLI and selecting one of the prepared templates. First, ensure you have [uv](https://pypi.org/project/uv/) installed:
|
||||
|
||||
```sh
|
||||
uv --help
|
||||
```
|
||||
|
||||
If [uv](https://pypi.org/project/uv/) is not installed, follow the official [installation guide](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
|
||||
Then, run the CLI and choose from the available templates:
|
||||
|
||||
```sh
|
||||
uvx 'crawlee[cli]' create my-crawler
|
||||
```
|
||||
|
||||
If you already have `crawlee` installed, you can spin it up by running:
|
||||
|
||||
```sh
|
||||
crawlee create my-crawler
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
Here are some practical examples to help you get started with different types of crawlers in Crawlee. Each example demonstrates how to set up and run a crawler for specific use cases, whether you need to handle simple HTML pages or interact with JavaScript-heavy sites. A crawler run will create a `storage/` directory in your current working directory.
|
||||
|
||||
### BeautifulSoupCrawler
|
||||
|
||||
The [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) downloads web pages using an HTTP library and provides HTML-parsed content to the user. By default it uses [`HttpxHttpClient`](https://crawlee.dev/python/api/class/HttpxHttpClient) for HTTP communication and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) for parsing HTML. It is ideal for projects that require efficient extraction of data from HTML content. This crawler has very good performance since it does not use a browser. However, if you need to execute client-side JavaScript, to get your content, this is not going to be enough and you will need to use [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler). Also if you want to use this crawler, make sure you install `crawlee` with `beautifulsoup` extra.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
### PlaywrightCrawler
|
||||
|
||||
The [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler) uses a headless browser to download web pages and provides an API for data extraction. It is built on [Playwright](https://playwright.dev/), an automation library designed for managing headless browsers. It excels at retrieving web pages that rely on client-side JavaScript for content generation, or tasks requiring interaction with JavaScript-driven content. For scenarios where JavaScript execution is unnecessary or higher performance is required, consider using the [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler). Also if you want to use this crawler, make sure you install `crawlee` with `playwright` extra.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.page.title(),
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
### More examples
|
||||
|
||||
Explore our [Examples](https://crawlee.dev/python/docs/examples) page in the Crawlee documentation for a wide range of additional use cases and demonstrations.
|
||||
|
||||
## Features
|
||||
|
||||
Why Crawlee is the preferred choice for web scraping and crawling?
|
||||
|
||||
### Why use Crawlee instead of just a random HTTP library with an HTML parser?
|
||||
|
||||
- Unified interface for **HTTP & headless browser** crawling.
|
||||
- Automatic **parallel crawling** based on available system resources.
|
||||
- Written in Python with **type hints** - enhances DX (IDE autocompletion) and reduces bugs (static type checking).
|
||||
- Automatic **retries** on errors or when you’re getting blocked.
|
||||
- Integrated **proxy rotation** and session management.
|
||||
- Configurable **request routing** - direct URLs to the appropriate handlers.
|
||||
- Persistent **queue for URLs** to crawl.
|
||||
- Pluggable **storage** of both tabular data and files.
|
||||
- Robust **error handling**.
|
||||
|
||||
### Why to use Crawlee rather than Scrapy?
|
||||
|
||||
- **Asyncio-based** – Leveraging the standard [Asyncio](https://docs.python.org/3/library/asyncio.html) library, Crawlee delivers better performance and seamless compatibility with other modern asynchronous libraries.
|
||||
- **Type hints** – Newer project built with modern Python, and complete type hint coverage for a better developer experience.
|
||||
- **Simple integration** – Crawlee crawlers are regular Python scripts, requiring no additional launcher executor. This flexibility allows to integrate a crawler directly into other applications.
|
||||
- **State persistence** – Supports state persistence during interruptions, saving time and costs by avoiding the need to restart scraping pipelines from scratch after an issue.
|
||||
- **Organized data storages** – Allows saving of multiple types of results in a single scraping run. Offers several storing options (see [datasets](https://crawlee.dev/python/api/class/Dataset) & [key-value stores](https://crawlee.dev/python/api/class/KeyValueStore)).
|
||||
|
||||
## Running on the Apify platform
|
||||
|
||||
Crawlee is open-source and runs anywhere, but since it's developed by [Apify](https://apify.com), it's easy to set up on the Apify platform and run in the cloud. Visit the [Apify SDK website](https://docs.apify.com/sdk/python/) to learn more about deploying Crawlee to the Apify platform.
|
||||
|
||||
## Support
|
||||
|
||||
If you find any bug or issue with Crawlee, please [submit an issue on GitHub](https://github.com/apify/crawlee-python/issues). For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/apify), in GitHub Discussions or you can join our [Discord server](https://discord.com/invite/jyEM2PRvMU).
|
||||
|
||||
## Contributing
|
||||
|
||||
Your code contributions are welcome, and you'll be praised for eternity! If you have any ideas for improvements, either submit an issue or create a pull request. For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/crawlee-python/blob/master/CONTRIBUTING.md).
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](https://github.com/apify/crawlee-python/blob/master/LICENSE) file for details.
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
coverage:
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: auto
|
||||
threshold: 0.10% # tolerate up to 0.10% decrease
|
||||
informational: true # CI check reports status but never fails
|
||||
patch:
|
||||
default:
|
||||
target: 50% # error only if patch coverage drops below 50%
|
||||
informational: true # CI check reports status but never fails
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
---
|
||||
id: apify-platform
|
||||
title: Apify platform
|
||||
description: Apify platform - large-scale and high-performance web scraping
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
|
||||
import CodeBlock from '@theme/CodeBlock';
|
||||
|
||||
import LogWithConfigExample from '!!raw-loader!./code_examples/apify/log_with_config_example.py';
|
||||
import CrawlerAsActorExample from '!!raw-loader!./code_examples/apify/crawler_as_actor_example.py';
|
||||
import ProxyExample from '!!raw-loader!./code_examples/apify/proxy_example.py';
|
||||
import ProxyAdvancedExample from '!!raw-loader!./code_examples/apify/proxy_advanced_example.py';
|
||||
|
||||
Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor), convenient request and result storages, [proxies](../guides/proxy-management), scheduling, webhooks and [more](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com) or an [API](https://docs.apify.com/api).
|
||||
|
||||
While we think that the Apify platform is super cool, and it's definitely worth signing up for a [free account](https://console.apify.com/sign-up), **Crawlee is and will always be open source**, runnable locally or on any cloud infrastructure.
|
||||
|
||||
:::note
|
||||
|
||||
We do not test Crawlee in other cloud environments such as Lambda or on specific architectures such as Raspberry PI. We strive to make it work, but there are no guarantees.
|
||||
|
||||
:::
|
||||
|
||||
## Requirements
|
||||
|
||||
To run your Crawlee code on Apify platform, you need an Apify account. If you don't have one yet, you can sign up [here](https://console.apify.com/sign-up).
|
||||
|
||||
Additionally, you must have the [Apify CLI](https://docs.apify.com/cli/) installed on your computer. For installation instructions, refer to the [Installation guide](https://docs.apify.com/cli/docs/installation).
|
||||
|
||||
Finally, ensure that the [Apify SDK] (https://docs.apify.com/sdk/python/) is installed in your project. You can install it using `pip`:
|
||||
|
||||
```bash
|
||||
pip install apify
|
||||
```
|
||||
|
||||
## Logging into Apify platform from Crawlee
|
||||
|
||||
To access your [Apify account](https://console.apify.com/sign-up) from Crawlee, you must provide credentials - your [API token](https://console.apify.com/account?tab=integrations). You can do that either by utilizing [Apify CLI](https://docs.apify.com/cli/) or with environment variables.
|
||||
|
||||
Once you provide credentials to your Apify CLI installation, you will be able to use all the Apify platform features, such as calling Actors, saving to cloud storages, using Apify proxies, setting up webhooks and so on.
|
||||
|
||||
### Log in with CLI
|
||||
|
||||
Apify CLI allows you to log in to your Apify account on your computer. If you then run your crawler using the CLI, your credentials will automatically be added.
|
||||
|
||||
```bash
|
||||
npm install -g apify-cli
|
||||
apify login -t YOUR_API_TOKEN
|
||||
```
|
||||
|
||||
### Log in with environment variables
|
||||
|
||||
Alternatively, you can always provide credentials to your Actor by setting the [`APIFY_TOKEN`](#apify_token) environment variable to your API token.
|
||||
|
||||
> There's also the [`APIFY_PROXY_PASSWORD`](#apify_proxy_password)
|
||||
> environment variable. Actor automatically infers that from your token, but it can be useful
|
||||
> when you need to access proxies from a different account than your token represents.
|
||||
|
||||
### Log in with Configuration
|
||||
|
||||
Another option is to use the [`Configuration`](https://docs.apify.com/sdk/python/reference/class/Configuration) instance and set your api token there.
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{LogWithConfigExample}
|
||||
</CodeBlock>
|
||||
|
||||
## What is an Actor
|
||||
|
||||
When you deploy your script to the Apify platform, it becomes an [Actor](https://apify.com/actors). An Actor is a serverless microservice that accepts an input and produces an output. It can run for a few seconds, hours or even infinitely. An Actor can perform anything from a simple action such as filling out a web form or sending an email, to complex operations such as crawling an entire website and removing duplicates from a large dataset.
|
||||
|
||||
Actors can be shared in the [Apify Store](https://apify.com/store) so that other people can use them. But don't worry, if you share your Actor in the store and somebody uses it, it runs under their account, not yours.
|
||||
|
||||
**Related links**
|
||||
|
||||
- [Store of existing Actors](https://apify.com/store)
|
||||
- [Documentation](https://docs.apify.com/actors)
|
||||
- [View Actors in Apify Console](https://console.apify.com/actors)
|
||||
- [API reference](https://apify.com/docs/api/v2#/reference/actors)
|
||||
|
||||
## Running an Actor locally
|
||||
|
||||
First let's create a boilerplate of the new Actor. You could use Apify CLI and just run:
|
||||
|
||||
```bash
|
||||
apify create my-hello-world
|
||||
```
|
||||
|
||||
The CLI will prompt you to select a project boilerplate template - let's pick "Crawlee + BeautifulSoup". The tool will create a directory called `my-hello-world` with Python project files. You can run the Actor as follows:
|
||||
|
||||
```bash
|
||||
cd my-hello-world
|
||||
apify run
|
||||
```
|
||||
|
||||
## Running Crawlee code as an Actor
|
||||
|
||||
For running Crawlee code as an Actor on [Apify platform](https://apify.com/actors) you need to wrap the body of the main function of your crawler with `async with Actor`.
|
||||
|
||||
:::info NOTE
|
||||
Adding `async with Actor` is the only important thing needed to run it on Apify platform as an Actor. It is needed to initialize your Actor (e.g. to set the correct storage implementation) and to correctly handle exiting the process.
|
||||
:::
|
||||
|
||||
Let's look at the `BeautifulSoupCrawler` example from the [Quick start](../quick-start) guide:
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{CrawlerAsActorExample}
|
||||
</CodeBlock>
|
||||
|
||||
Note that you could also run your Actor (that is using Crawlee) locally with Apify CLI. You could start it via the following command in your project folder:
|
||||
|
||||
```bash
|
||||
apify run
|
||||
```
|
||||
|
||||
## Deploying an Actor to Apify platform
|
||||
|
||||
Now (assuming you are already logged in to your Apify account) you can easily deploy your code to the Apify platform by running:
|
||||
|
||||
```bash
|
||||
apify push
|
||||
```
|
||||
|
||||
Your script will be uploaded to and built on the Apify platform so that it can be run there. For more information, view the
|
||||
[Apify Actor](https://docs.apify.com/cli) documentation.
|
||||
|
||||
## Usage on Apify platform
|
||||
|
||||
You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Examples](../examples) section.
|
||||
|
||||
## Storages
|
||||
|
||||
There are several things worth mentioning here.
|
||||
|
||||
### Helper functions for default Key-Value Store and Dataset
|
||||
|
||||
To simplify access to the _default_ storages, instead of using the helper functions of respective storage classes, you could use:
|
||||
- [`Actor.set_value()`](https://docs.apify.com/sdk/python/reference/class/Actor#set_value), [`Actor.get_value()`](https://docs.apify.com/sdk/python/reference/class/Actor#get_value), [`Actor.get_input()`](https://docs.apify.com/sdk/python/reference/class/Actor#get_input) for [`Key-Value Store`](https://docs.apify.com/sdk/python/reference/class/KeyValueStore)
|
||||
- [`Actor.push_data()`](https://docs.apify.com/sdk/python/reference/class/Actor#push_data) for [`Dataset`](https://docs.apify.com/sdk/python/reference/class/Dataset)
|
||||
|
||||
### Using platform storage in a local Actor
|
||||
|
||||
When you plan to use the platform storage while developing and running your Actor locally, you should use [`Actor.open_key_value_store()`](https://docs.apify.com/sdk/python/reference/class/Actor#open_key_value_store), [`Actor.open_dataset()`](https://docs.apify.com/sdk/python/reference/class/Actor#open_dataset) and [`Actor.open_request_queue()`](https://docs.apify.com/sdk/python/reference/class/Actor#open_request_queue) to open the respective storage.
|
||||
|
||||
Using each of these methods allows to pass the `force_cloud` keyword argument. If set to `True`, cloud storage will be used instead of the folder on the local disk.
|
||||
|
||||
:::note
|
||||
If you don't plan to force usage of the platform storages when running the Actor locally, there is no need to use the [`Actor`](https://docs.apify.com/sdk/python/reference/class/Actor) class for it. The Crawlee variants <ApiLink to="class/KeyValueStore#open">`KeyValueStore.open()`</ApiLink>, <ApiLink to="class/Dataset#open">`Dataset.open()`</ApiLink> and <ApiLink to="class/RequestQueue#open">`RequestQueue.open()`</ApiLink> will work the same.
|
||||
:::
|
||||
|
||||
{/*
|
||||
### Getting public url of an item in the platform storage
|
||||
|
||||
If you need to share a link to some file stored in a [Key-Value](https://docs.apify.com/sdk/python/reference/class/KeyValueStore) Store on Apify platform, you can use [`get_public_url()`](https://docs.apify.com/sdk/python/reference/class/KeyValueStore#get_public_url) method. It accepts only one parameter: `key` - the key of the item you want to share.
|
||||
|
||||
<CodeBlock language="python">
|
||||
{GetPublicUrlSource}
|
||||
</CodeBlock>
|
||||
|
||||
*/}
|
||||
|
||||
### Exporting dataset data
|
||||
|
||||
When the <ApiLink to="class/Dataset">`Dataset`</ApiLink> is stored on the [Apify platform](https://apify.com/actors), you can export its data to the following formats: HTML, JSON, CSV, Excel, XML and RSS. The datasets are displayed on the Actor run details page and in the [Storage](https://console.apify.com/storage) section in the Apify Console. The actual data is exported using the [Get dataset items](https://apify.com/docs/api/v2#/reference/datasets/item-collection/get-items) Apify API endpoint. This way you can easily share the crawling results.
|
||||
|
||||
**Related links**
|
||||
|
||||
- [Apify platform storage documentation](https://docs.apify.com/storage)
|
||||
- [View storage in Apify Console](https://console.apify.com/storage)
|
||||
- [Key-value stores API reference](https://apify.com/docs/api/v2#/reference/key-value-stores)
|
||||
- [Datasets API reference](https://docs.apify.com/api/v2#/reference/datasets)
|
||||
- [Request queues API reference](https://docs.apify.com/api/v2#/reference/request-queues)
|
||||
|
||||
## Environment variables
|
||||
|
||||
The following describes select environment variables set by the Apify platform. For a complete list, see the [Environment variables](https://docs.apify.com/platform/actors/development/programming-interface/environment-variables) section in the Apify platform documentation.
|
||||
|
||||
:::note
|
||||
|
||||
It's important to notice that `CRAWLEE_` environment variables don't need to be replaced with equivalent `APIFY_` ones. Likewise, Crawlee understands `APIFY_` environment variables.
|
||||
|
||||
:::
|
||||
|
||||
### `APIFY_TOKEN`
|
||||
|
||||
The API token for your Apify account. It is used to access the Apify API, e.g. to access cloud storage
|
||||
or to run an Actor on the Apify platform. You can find your API token on the
|
||||
[Account Settings / Integrations](https://console.apify.com/account?tab=integrations) page.
|
||||
|
||||
### Combinations of `APIFY_TOKEN` and `CRAWLEE_STORAGE_DIR`
|
||||
|
||||
By combining the env vars in various ways, you can greatly influence the Actor's behavior.
|
||||
|
||||
| Env Vars | API | Storages |
|
||||
| --------------------------------------- | --- | ---------------- |
|
||||
| none OR `CRAWLEE_STORAGE_DIR` | no | local |
|
||||
| `APIFY_TOKEN` | yes | Apify platform |
|
||||
| `APIFY_TOKEN` AND `CRAWLEE_STORAGE_DIR` | yes | local + platform |
|
||||
|
||||
When using both `APIFY_TOKEN` and `CRAWLEE_STORAGE_DIR`, you can use all the Apify platform
|
||||
features and your data will be stored locally by default. If you want to access platform storages,
|
||||
you can use the `force_cloud=true` option in their respective functions.
|
||||
|
||||
### `APIFY_PROXY_PASSWORD`
|
||||
|
||||
Optional password to [Apify Proxy](https://docs.apify.com/proxy) for IP address rotation.
|
||||
Assuming Apify Account was already created, you can find the password on the [Proxy page](https://console.apify.com/proxy)
|
||||
in the Apify Console. The password is automatically inferred using the `APIFY_TOKEN` env var,
|
||||
so in most cases, you don't need to touch it. You should use it when, for some reason,
|
||||
you need access to Apify Proxy, but not access to Apify API, or when you need access to
|
||||
proxy from a different account than your token represents.
|
||||
|
||||
## Proxy management
|
||||
|
||||
In addition to your own proxy servers and proxy servers acquired from
|
||||
third-party providers used together with Crawlee, you can also rely on [Apify Proxy](https://apify.com/proxy)
|
||||
for your scraping needs.
|
||||
|
||||
### Apify proxy
|
||||
|
||||
If you are already subscribed to Apify Proxy, you can start using them immediately in only a few lines of code (for local usage you first should be [logged in](#logging-into-apify-platform-from-crawlee) to your Apify account.
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{ProxyExample}
|
||||
</CodeBlock>
|
||||
|
||||
Note that unlike using your own proxies in Crawlee, you shouldn't use the constructor to create <ApiLink to="class/ProxyConfiguration">`ProxyConfiguration`</ApiLink> instances. For using the Apify Proxy you should create an instance using the [`Actor.create_proxy_configuration()`](https://docs.apify.com/sdk/python/reference/class/Actor#create_proxy_configuration) function instead.
|
||||
|
||||
### Advanced Apify proxy configuration
|
||||
|
||||
With Apify Proxy, you can select specific proxy groups to use, or countries to connect from.
|
||||
This allows you to get better proxy performance after some initial research.
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{ProxyAdvancedExample}
|
||||
</CodeBlock>
|
||||
|
||||
Now your crawlers will use only Residential proxies from the US. Note that you must first get access
|
||||
to a proxy group before you are able to use it. You can check proxy groups available to you
|
||||
in the [proxy dashboard](https://console.apify.com/proxy).
|
||||
|
||||
### Apify proxy vs. own proxies
|
||||
|
||||
The [`ProxyConfiguration`](https://docs.apify.com/sdk/python/reference/class/ProxyConfiguration) class covers both Apify Proxy and custom proxy URLs so that you can easily switch between proxy providers. However, some features of the class are available only to Apify Proxy users, mainly because Apify Proxy is what one would call a super-proxy. It's not a single proxy server, but an API endpoint that allows connection through millions of different IP addresses. So the class essentially has two modes: Apify Proxy or Own (third party) proxy.
|
||||
|
||||
The difference is easy to remember.
|
||||
- If you're using your own proxies - you should create a <ApiLink to="class/ProxyConfiguration">`ProxyConfiguration`</ApiLink> instance directly.
|
||||
- If you are planning to use Apify Proxy - you should create an instance using the [`Actor.create_proxy_configuration()`](https://docs.apify.com/sdk/python/reference/class/Actor#create_proxy_configuration) function. The `new_url_function` parameter enables the use of your custom proxy URLs, whereas all the other options are there to configure Apify Proxy.
|
||||
|
||||
**Related links**
|
||||
|
||||
- [Apify Proxy docs](https://docs.apify.com/proxy)
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
---
|
||||
id: aws-lambda
|
||||
title: Deploy on AWS Lambda
|
||||
description: Prepare your crawler to run on AWS Lambda.
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
|
||||
import CodeBlock from '@theme/CodeBlock';
|
||||
|
||||
import BeautifulSoupCrawlerLambda from '!!raw-loader!./code_examples/aws/beautifulsoup_crawler_lambda.py';
|
||||
import PlaywrightCrawlerLambda from '!!raw-loader!./code_examples/aws/playwright_crawler_lambda.py';
|
||||
import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/aws/playwright_dockerfile';
|
||||
|
||||
[AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) is a serverless compute service that lets you run code without provisioning or managing servers. This guide covers deploying <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> and <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>.
|
||||
|
||||
The code examples are based on the [BeautifulSoupCrawler example](../examples/beautifulsoup-crawler).
|
||||
|
||||
## BeautifulSoupCrawler on AWS Lambda
|
||||
|
||||
For simple crawlers that don't require browser rendering, you can deploy using a ZIP archive.
|
||||
|
||||
### Updating the code
|
||||
|
||||
When instantiating a crawler, use <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink>. By default, Crawlee uses file-based storage, but the Lambda filesystem is read-only (except for `/tmp`). Using `MemoryStorageClient` tells Crawlee to use in-memory storage instead.
|
||||
|
||||
Wrap the crawler logic in a `lambda_handler` function. This is the entry point that AWS will execute.
|
||||
|
||||
:::important
|
||||
|
||||
Make sure to always instantiate a new crawler for every Lambda invocation. AWS keeps the environment running for some time after the first execution (to reduce cold-start times), so subsequent calls may access an already-used crawler instance.
|
||||
|
||||
**TL;DR: Keep your Lambda stateless.**
|
||||
|
||||
:::
|
||||
|
||||
Finally, return the scraped data from the Lambda when the crawler run ends.
|
||||
|
||||
<CodeBlock language="python" title="lambda_function.py">
|
||||
{BeautifulSoupCrawlerLambda}
|
||||
</CodeBlock>
|
||||
|
||||
### Preparing the environment
|
||||
|
||||
Lambda requires all dependencies to be included in the deployment package. Create a virtual environment and install dependencies:
|
||||
|
||||
```bash
|
||||
python3.14 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install 'crawlee[beautifulsoup]' 'boto3' 'aws-lambda-powertools'
|
||||
```
|
||||
|
||||
[`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) is the AWS SDK for Python. Including it in your dependencies is recommended to avoid version misalignment issues with the Lambda runtime.
|
||||
|
||||
### Creating the ZIP archive
|
||||
|
||||
Create a ZIP archive from your project, including dependencies from the virtual environment:
|
||||
|
||||
```bash
|
||||
cd .venv/lib/python3.14/site-packages
|
||||
zip -r ../../../../package.zip .
|
||||
cd ../../../../
|
||||
zip package.zip lambda_function.py
|
||||
```
|
||||
|
||||
:::note Large dependencies?
|
||||
|
||||
AWS has a limit of 50 MB for direct upload and 250 MB for unzipped deployment package size.
|
||||
|
||||
A better way to manage dependencies is by using Lambda Layers. With Layers, you can share files between multiple Lambda functions and keep the actual code as slim as possible.
|
||||
|
||||
To create a Lambda Layer:
|
||||
|
||||
1. Create a `python/` folder and copy dependencies from `site-packages` into it
|
||||
2. Create a zip archive: `zip -r layer.zip python/`
|
||||
3. Create a new Lambda Layer from the archive (you may need to upload it to S3 first)
|
||||
4. Attach the Layer to your Lambda function
|
||||
|
||||
:::
|
||||
|
||||
### Creating the Lambda function
|
||||
|
||||
Create the Lambda function in the AWS Lambda Console:
|
||||
|
||||
1. Navigate to `Lambda` in [AWS Management Console](https://aws.amazon.com/console/).
|
||||
2. Click **Create function**.
|
||||
3. Select **Author from scratch**.
|
||||
4. Enter a **Function name**, for example `BeautifulSoupTest`.
|
||||
5. Choose a **Python runtime** that matches the version used in your virtual environment (for example, Python 3.14).
|
||||
6. Click **Create function** to finish.
|
||||
|
||||
Once created, upload `package.zip` as the code source in the AWS Lambda Console using the "Upload from" button.
|
||||
|
||||
In Lambda Runtime Settings, set the handler. Since the file is named `lambda_function.py` and the function is `lambda_handler`, you can use the default value `lambda_function.lambda_handler`.
|
||||
|
||||
:::tip Configuration
|
||||
|
||||
In the Configuration tab, you can adjust:
|
||||
|
||||
- **Memory**: Memory size can greatly affect execution speed. A minimum of 256-512 MB is recommended.
|
||||
- **Timeout**: Set according to the size of the website you are scraping (1 minute for the example code).
|
||||
- **Ephemeral storage**: Size of the `/tmp` directory.
|
||||
|
||||
See the [official documentation](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html) to learn how performance and cost scale with memory.
|
||||
|
||||
:::
|
||||
|
||||
After the Lambda deploys, you can test it by clicking the "Test" button. The event contents don't matter for a basic test, but you can parameterize your crawler by parsing the event object that AWS passes as the first argument to the handler.
|
||||
|
||||
## PlaywrightCrawler on AWS Lambda
|
||||
|
||||
For crawlers that require browser rendering, you need to deploy using Docker container images because Playwright and browser binaries exceed Lambda's ZIP deployment size limits.
|
||||
|
||||
### Updating the code
|
||||
|
||||
As with <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>, use <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> and wrap the logic in a `lambda_handler` function. Additionally, configure `browser_launch_options` with flags optimized for serverless environments. These flags disable sandboxing and GPU features that aren't available in Lambda's containerized runtime.
|
||||
|
||||
<CodeBlock language="python" title="main.py">
|
||||
{PlaywrightCrawlerLambda}
|
||||
</CodeBlock>
|
||||
|
||||
### Installing and configuring AWS CLI
|
||||
|
||||
Install AWS CLI following the [official documentation](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) according to your operating system.
|
||||
|
||||
Authenticate by running:
|
||||
|
||||
```bash
|
||||
aws login
|
||||
```
|
||||
|
||||
### Preparing the project
|
||||
|
||||
Initialize the project by running `uvx 'crawlee[cli]' create`.
|
||||
|
||||
Or use a single command if you don't need interactive mode:
|
||||
|
||||
```bash
|
||||
uvx 'crawlee[cli]' create aws_playwright --crawler-type playwright --http-client impit --package-manager uv --no-apify --start-url 'https://crawlee.dev' --install
|
||||
```
|
||||
|
||||
Add the following dependencies:
|
||||
|
||||
```bash
|
||||
uv add awslambdaric aws-lambda-powertools boto3
|
||||
```
|
||||
|
||||
[`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) is the AWS SDK for Python. Use it if your function integrates with any other AWS services.
|
||||
|
||||
The project is created with a Dockerfile that needs to be modified for AWS Lambda by adding `ENTRYPOINT` and updating `CMD`:
|
||||
|
||||
<CodeBlock language="dockerfile" title="Dockerfile">
|
||||
{PlaywrightCrawlerDockerfile}
|
||||
</CodeBlock>
|
||||
|
||||
### Building and pushing the Docker image
|
||||
|
||||
Create a repository `lambda/aws-playwright` in [Amazon Elastic Container Registry](https://docs.aws.amazon.com/AmazonECR/latest/userguide/what-is-ecr.html) in the same region where your Lambda functions will run. To learn more, refer to the [official documentation](https://docs.aws.amazon.com/AmazonECR/latest/userguide/getting-started-cli.html).
|
||||
|
||||
Navigate to the created repository and click the "View push commands" button. This will open a window with console commands for uploading the Docker image to your repository. Execute them.
|
||||
|
||||
Example:
|
||||
```bash
|
||||
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin {user-specific-data}
|
||||
docker build --platform linux/amd64 --provenance=false -t lambda/aws-playwright .
|
||||
docker tag lambda/aws-playwright:latest {user-specific-data}/lambda/aws-playwright:latest
|
||||
docker push {user-specific-data}/lambda/aws-playwright:latest
|
||||
```
|
||||
|
||||
### Creating the Lambda function
|
||||
|
||||
1. Navigate to `Lambda` in [AWS Management Console](https://aws.amazon.com/console/).
|
||||
2. Click **Create function**.
|
||||
3. Select **Container image**.
|
||||
4. Browse and select your ECR image.
|
||||
5. Click **Create function** to finish.
|
||||
|
||||
:::tip Configuration
|
||||
|
||||
In the Configuration tab, you can adjust resources. Playwright crawlers require more resources than BeautifulSoup crawlers:
|
||||
|
||||
- **Memory**: Minimum 1024 MB recommended. Browser operations are memory-intensive, so 2048 MB or more may be needed for complex pages.
|
||||
- **Timeout**: Set according to crawl size. Browser startup adds overhead, so allow at least 5 minutes even for simple crawls.
|
||||
- **Ephemeral storage**: Default 512 MB is usually sufficient unless downloading large files.
|
||||
|
||||
See the [official documentation](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html) to learn how performance and cost scale with memory.
|
||||
|
||||
:::
|
||||
|
||||
After the Lambda deploys, click the "Test" button to invoke it. The event contents don't matter for a basic test, but you can parameterize your crawler by parsing the event object that AWS passes as the first argument to the handler.
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Wrap the crawler code in an Actor context manager.
|
||||
async with Actor:
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
store = await Actor.open_key_value_store()
|
||||
await store.set_value('your-file', {'foo': 'bar'})
|
||||
url = store.get_public_url('your-file')
|
||||
Actor.log.info(f'KVS public URL: {url}')
|
||||
# https://api.apify.com/v2/key-value-stores/<your-store-id>/records/your-file
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import asyncio
|
||||
|
||||
from apify import Actor, Configuration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a new configuration with your API key. You can find it at
|
||||
# https://console.apify.com/settings/integrations. It can be provided either
|
||||
# as a parameter "token" or as an environment variable "APIFY_TOKEN".
|
||||
config = Configuration(
|
||||
token='apify_api_YOUR_TOKEN',
|
||||
)
|
||||
|
||||
async with Actor(config):
|
||||
Actor.log.info('Hello from Apify platform!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
proxy_configuration = await Actor.create_proxy_configuration(
|
||||
password='apify_proxy_YOUR_PASSWORD',
|
||||
# Specify the proxy group to use.
|
||||
groups=['RESIDENTIAL'],
|
||||
# Set the country code for the proxy.
|
||||
country_code='US',
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
# Create a new Apify Proxy configuration. The password can be found at
|
||||
# https://console.apify.com/proxy/http-settings and should be provided either
|
||||
# as a parameter "password" or as an environment variable "APIFY_PROXY_PASSWORD".
|
||||
proxy_configuration = await Actor.create_proxy_configuration(
|
||||
password='apify_proxy_YOUR_PASSWORD',
|
||||
)
|
||||
|
||||
if not proxy_configuration:
|
||||
Actor.log.warning('Failed to create proxy configuration.')
|
||||
return
|
||||
|
||||
proxy_url = await proxy_configuration.new_url()
|
||||
Actor.log.info(f'Proxy URL: {proxy_url}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from aws_lambda_powertools.utilities.typing import LambdaContext
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
from crawlee.storages import Dataset, RequestQueue
|
||||
|
||||
|
||||
async def main() -> str:
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
# Initialize storages
|
||||
dataset = await Dataset.open(storage_client=storage_client)
|
||||
request_queue = await RequestQueue.open(storage_client=storage_client)
|
||||
|
||||
crawler = BeautifulSoupCrawler(
|
||||
storage_client=storage_client,
|
||||
max_request_retries=1,
|
||||
request_handler_timeout=timedelta(seconds=30),
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
'h1s': [h1.text for h1 in context.soup.find_all('h1')],
|
||||
'h2s': [h2.text for h2 in context.soup.find_all('h2')],
|
||||
'h3s': [h3.text for h3 in context.soup.find_all('h3')],
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# Extract data saved in `Dataset`
|
||||
data = await crawler.get_data()
|
||||
|
||||
# Clean up storages after the crawl
|
||||
await dataset.drop()
|
||||
await request_queue.drop()
|
||||
|
||||
# Serialize the list of scraped items to JSON string
|
||||
return json.dumps(data.items)
|
||||
|
||||
|
||||
def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> dict[str, Any]:
|
||||
result = asyncio.run(main())
|
||||
# Return the response with results
|
||||
return {'statusCode': 200, 'body': result}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from aws_lambda_powertools.utilities.typing import LambdaContext
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
from crawlee.storages import Dataset, RequestQueue
|
||||
|
||||
|
||||
async def main() -> str:
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
# Initialize storages
|
||||
dataset = await Dataset.open(storage_client=storage_client)
|
||||
request_queue = await RequestQueue.open(storage_client=storage_client)
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
storage_client=storage_client,
|
||||
max_request_retries=1,
|
||||
request_handler_timeout=timedelta(seconds=30),
|
||||
max_requests_per_crawl=10,
|
||||
# highlight-start
|
||||
# Configure Playwright to run in AWS Lambda environment
|
||||
browser_launch_options={
|
||||
'args': [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-gpu',
|
||||
'--single-process',
|
||||
]
|
||||
},
|
||||
# highlight-end
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.page.title(),
|
||||
'h1s': await context.page.locator('h1').all_text_contents(),
|
||||
'h2s': await context.page.locator('h2').all_text_contents(),
|
||||
'h3s': await context.page.locator('h3').all_text_contents(),
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# Extract data saved in `Dataset`
|
||||
data = await crawler.get_data()
|
||||
|
||||
# Clean up storages after the crawl
|
||||
await dataset.drop()
|
||||
await request_queue.drop()
|
||||
|
||||
# Serialize the list of scraped items to JSON string
|
||||
return json.dumps(data.items)
|
||||
|
||||
|
||||
def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> dict[str, Any]:
|
||||
result = asyncio.run(main())
|
||||
# Return the response with results
|
||||
return {'statusCode': 200, 'body': result}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
FROM apify/actor-python-playwright:3.14
|
||||
|
||||
RUN apt update && apt install -yq git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN pip install -U pip setuptools \
|
||||
&& pip install 'uv<1'
|
||||
|
||||
ENV UV_PROJECT_ENVIRONMENT="/usr/local"
|
||||
|
||||
COPY pyproject.toml uv.lock ./
|
||||
|
||||
RUN echo "Python version:" \
|
||||
&& python --version \
|
||||
&& echo "Installing dependencies:" \
|
||||
&& PLAYWRIGHT_INSTALLED=$(pip freeze | grep -q playwright && echo "true" || echo "false") \
|
||||
&& if [ "$PLAYWRIGHT_INSTALLED" = "true" ]; then \
|
||||
echo "Playwright already installed, excluding from uv sync" \
|
||||
&& uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact --no-install-package playwright; \
|
||||
else \
|
||||
echo "Playwright not found, installing all dependencies" \
|
||||
&& uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact; \
|
||||
fi \
|
||||
&& echo "All installed Python packages:" \
|
||||
&& pip freeze
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN python -m compileall -q .
|
||||
|
||||
# highlight-start
|
||||
# AWS Lambda entrypoint
|
||||
ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ]
|
||||
|
||||
# Lambda handler function
|
||||
CMD [ "aws_playwright.main.lambda_handler" ]
|
||||
# highlight-end
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import json
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
from litestar import Litestar, get
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
@get('/')
|
||||
async def main() -> str:
|
||||
"""The crawler entry point that will be called when the HTTP endpoint is accessed."""
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
headless=True,
|
||||
max_requests_per_crawl=10,
|
||||
browser_type='firefox',
|
||||
storage_client=storage_client,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
"""Default request handler that processes each page during crawling."""
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
title = await context.page.query_selector('title')
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.loaded_url,
|
||||
'title': await title.inner_text() if title else None,
|
||||
}
|
||||
)
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
data = await crawler.get_data()
|
||||
|
||||
# Return the results as JSON to the client
|
||||
return json.dumps(data.items)
|
||||
|
||||
|
||||
# Initialize the Litestar app with our route handler
|
||||
app = Litestar(route_handlers=[main])
|
||||
|
||||
# Start the Uvicorn server using the `PORT` environment variable provided by GCP
|
||||
# This is crucial - Cloud Run expects your app to listen on this specific port
|
||||
uvicorn.run(app, host='0.0.0.0', port=int(os.environ.get('PORT', '8080'))) # noqa: S104 # Use all interfaces in a container, safely
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
|
||||
import functions_framework
|
||||
from flask import Request, Response
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
async def main() -> str:
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
crawler = BeautifulSoupCrawler(
|
||||
storage_client=storage_client,
|
||||
max_request_retries=1,
|
||||
request_handler_timeout=timedelta(seconds=30),
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
'h1s': [h1.text for h1 in context.soup.find_all('h1')],
|
||||
'h2s': [h2.text for h2 in context.soup.find_all('h2')],
|
||||
'h3s': [h3.text for h3 in context.soup.find_all('h3')],
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# highlight-start
|
||||
# Extract data saved in `Dataset`
|
||||
data = await crawler.get_data()
|
||||
# Serialize to json string and return
|
||||
return json.dumps(data.items)
|
||||
# highlight-end
|
||||
|
||||
|
||||
@functions_framework.http
|
||||
def crawlee_run(request: Request) -> Response:
|
||||
# You can pass data to your crawler using `request`
|
||||
function_id = request.headers['Function-Execution-Id']
|
||||
response_str = asyncio.run(main())
|
||||
|
||||
# Return a response with the crawling results
|
||||
return Response(response=response_str, status=200)
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
id: gcp-cloud-run-functions
|
||||
title: Cloud Run functions
|
||||
description: Prepare your crawler to run in Cloud Run functions on Google Cloud Platform.
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
|
||||
import CodeBlock from '@theme/CodeBlock';
|
||||
|
||||
import GoogleFunctions from '!!raw-loader!./code_examples/google/google_example.py';
|
||||
|
||||
[Google Cloud Run Functions](https://cloud.google.com/functions) is a serverless execution environment for running simple HTTP-based web scrapers. This service is best suited for lightweight crawlers that don't require browser rendering capabilities and can be executed via HTTP requests.
|
||||
|
||||
## Updating the project
|
||||
|
||||
For the project foundation, use <ApiLink to="class/BeautifulSoupCrawler">BeautifulSoupCrawler</ApiLink> as described in this [example](../examples/beautifulsoup-crawler).
|
||||
|
||||
Add [`functions-framework`](https://pypi.org/project/functions-framework/) to your dependencies file `requirements.txt`. If you're using a project manager like `poetry` or `uv`, export your dependencies to `requirements.txt`.
|
||||
|
||||
Update the project code to make it compatible with Cloud Functions and return data in JSON format. Also add an entry point that Cloud Functions will use to run the project.
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{GoogleFunctions.replace(/^.*?\n/, '')}
|
||||
</CodeBlock>
|
||||
|
||||
You can test your project locally. Start the server by running:
|
||||
|
||||
```bash
|
||||
functions-framework --target=crawlee_run
|
||||
```
|
||||
|
||||
Then make a GET request to `http://127.0.0.1:8080/`, for example in your browser.
|
||||
|
||||
## Deploying to Google Cloud Platform
|
||||
|
||||
In the Google Cloud dashboard, create a new function, allocate memory and CPUs to it, set region and function timeout.
|
||||
|
||||
When deploying, select **"Use an inline editor to create a function"**. This allows you to configure the project using only the Google Cloud Console dashboard.
|
||||
|
||||
Using the `inline editor`, update the function files according to your project. **Make sure** to update the `requirements.txt` file to match your project's dependencies.
|
||||
|
||||
Also, make sure to set the **Function entry point** to the name of the function decorated with `@functions_framework.http`, which in our case is `crawlee_run`.
|
||||
|
||||
After the Function deploys, you can test it by clicking the "Test" button. This button opens a popup with a `curl` script that calls your new Cloud Function. To avoid having to install the `gcloud` CLI application locally, you can also run this script in the Cloud Shell by clicking the link above the code block.
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
---
|
||||
id: gcp-cloud-run
|
||||
title: Cloud Run
|
||||
description: Prepare your crawler to run in Cloud Run on Google Cloud Platform.
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
|
||||
import CodeBlock from '@theme/CodeBlock';
|
||||
|
||||
import GoogleCloudRun from '!!raw-loader!./code_examples/google/cloud_run_example.py';
|
||||
|
||||
|
||||
[Google Cloud Run](https://cloud.google.com/run) is a container-based serverless platform that allows you to run web crawlers with headless browsers. This service is recommended when your Crawlee applications need browser rendering capabilities, require more granular control, or have complex dependencies that aren't supported by [Cloud Functions](./gcp-cloud-run-functions).
|
||||
|
||||
GCP Cloud Run allows you to deploy using Docker containers, giving you full control over your environment and the flexibility to use any web server framework of your choice, unlike Cloud Functions which are limited to [Flask](https://flask.palletsprojects.com/en/stable/).
|
||||
|
||||
## Preparing the project
|
||||
|
||||
We'll prepare our project using [Litestar](https://litestar.dev/) and the [Uvicorn](https://www.uvicorn.org/) web server. The HTTP server handler will wrap the crawler to communicate with clients. Because the Cloud Run platform sees only an opaque Docker container, we have to take care of this bit ourselves.
|
||||
|
||||
:::info
|
||||
|
||||
GCP passes you an environment variable called `PORT` - your HTTP server is expected to be listening on this port (GCP exposes this one to the outer world).
|
||||
|
||||
:::
|
||||
|
||||
<CodeBlock className="language-python">
|
||||
{GoogleCloudRun.replace(/^.*?\n/, '')}
|
||||
</CodeBlock>
|
||||
|
||||
|
||||
:::tip
|
||||
|
||||
Always make sure to keep all the logic in the request handler - as with other FaaS services, your request handlers have to be **stateless.**
|
||||
|
||||
:::
|
||||
|
||||
## Deploying to Google Cloud Platform
|
||||
|
||||
Now, we’re ready to deploy! If you have initialized your project using `uvx crawlee create`, the initialization script has prepared a Dockerfile for you.
|
||||
|
||||
All you have to do now is run `gcloud run deploy` in your project folder (the one with your Dockerfile in it). The gcloud CLI application will ask you a few questions, such as what region you want to deploy your application in, or whether you want to make your application public or private.
|
||||
|
||||
After answering those questions, you should be able to see your application in the GCP dashboard and run it using the link you find there.
|
||||
|
||||
:::tip
|
||||
|
||||
In case your first execution of your newly created Cloud Run fails, try editing the Run configuration - mainly setting the available memory to 1GiB or more and updating the request timeout according to the size of the website you are scraping.
|
||||
|
||||
:::
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
---
|
||||
id: add-data-to-dataset
|
||||
title: Add data to dataset
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_bs.py';
|
||||
import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_pw.py';
|
||||
import DatasetExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_dataset.py';
|
||||
|
||||
This example demonstrates how to store extracted data into datasets using the <ApiLink to="class/PushDataFunction#open">`context.push_data`</ApiLink> helper function. If the specified dataset does not already exist, it will be created automatically. Additionally, you can save data to custom datasets by providing `dataset_id` or `dataset_name` parameters to the <ApiLink to="class/PushDataFunction#open">`push_data`</ApiLink> function.
|
||||
|
||||
<Tabs groupId="main">
|
||||
<TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
Each item in the dataset will be stored in its own file within the following directory:
|
||||
|
||||
```text
|
||||
{PROJECT_FOLDER}/storage/datasets/default/
|
||||
```
|
||||
|
||||
For more control, you can also open a dataset manually using the asynchronous constructor <ApiLink to="class/Dataset#open">`Dataset.open`</ApiLink>
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{DatasetExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
id: beautifulsoup-crawler
|
||||
title: BeautifulSoup crawler
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler.py';
|
||||
|
||||
This example demonstrates how to use <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> to crawl a list of URLs, load each URL using a plain HTTP request, parse the HTML using the [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library and extract some data from it - the page title and all `<h1>`, `<h2>` and `<h3>` tags. This setup is perfect for scraping specific elements from web pages. Thanks to the well-known BeautifulSoup, you can easily navigate the HTML structure and retrieve the data you need with minimal code. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
---
|
||||
id: capture-screenshots-using-playwright
|
||||
title: Capture screenshots using Playwright
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/capture_screenshot_using_playwright.py';
|
||||
|
||||
This example demonstrates how to capture screenshots of web pages using <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and store them in the key-value store.
|
||||
|
||||
The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is configured to automate the browsing and interaction with web pages. It uses headless Chromium as the browser type to perform these tasks. Each web page specified in the initial list of URLs is visited sequentially, and a screenshot of the page is captured using Playwright's `page.screenshot()` method.
|
||||
|
||||
The captured screenshots are stored in the key-value store, which is suitable for managing and storing files in various formats. In this case, screenshots are stored as PNG images with a unique key generated from the URL of the page.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{CaptureScreenshotExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
id: capturing-page-snapshots-with-error-snapshotter
|
||||
title: Capturing page snapshots with ErrorSnapshotter
|
||||
description: How to capture page snapshots on errors.
|
||||
---
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import ParselCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_with_error_snapshotter.py';
|
||||
import PlaywrightCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_error_snapshotter.py';
|
||||
|
||||
|
||||
This example demonstrates how to capture page snapshots on first occurrence of each unique error. The capturing happens automatically if you set `save_error_snapshots=True` in the crawler's <ApiLink to="class/Statistics">`Statistics`</ApiLink>. The error snapshot can contain `html` file and `jpeg` file that are created from the page where the unhandled exception was raised. Captured error snapshot files are saved to the default key-value store. Both <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and [HTTP crawlers](../guides/http-crawlers) are capable of capturing the html file, but only <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> is able to capture page screenshot as well.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="ParselCrawler" label="ParselCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{ ParselCrawlerWithErrorSnapshotter }
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{ PlaywrightCrawlerWithErrorSnapshotter }
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from playwright.async_api import Route
|
||||
|
||||
from crawlee.crawlers import (
|
||||
AdaptivePlaywrightCrawler,
|
||||
AdaptivePlaywrightCrawlingContext,
|
||||
AdaptivePlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Crawler created by following factory method will use `beautifulsoup`
|
||||
# for parsing static content.
|
||||
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
playwright_crawler_specific_kwargs={'headless': False},
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler_for_label(
|
||||
context: AdaptivePlaywrightCrawlingContext,
|
||||
) -> None:
|
||||
# Do some processing using `parsed_content`
|
||||
context.log.info(context.parsed_content.title)
|
||||
|
||||
# Locate element h2 within 5 seconds
|
||||
h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
|
||||
# Do stuff with element found by the selector
|
||||
context.log.info(h2)
|
||||
|
||||
# Find more links and enqueue them.
|
||||
await context.enqueue_links()
|
||||
# Save some data.
|
||||
await context.push_data({'Visited url': context.request.url})
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
"""Hook executed both in static sub crawler and playwright sub crawler.
|
||||
|
||||
Trying to access `context.page` in this hook would raise `AdaptiveContextError`
|
||||
for pages crawled without playwright."""
|
||||
context.log.info(f'pre navigation hook for: {context.request.url} ...')
|
||||
|
||||
@crawler.pre_navigation_hook(playwright_only=True)
|
||||
async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
"""Hook executed only in playwright sub crawler.
|
||||
|
||||
It is safe to access `page` object.
|
||||
"""
|
||||
|
||||
async def some_routing_function(route: Route) -> None:
|
||||
await route.continue_()
|
||||
|
||||
await context.page.route('*/**', some_routing_function)
|
||||
context.log.info(
|
||||
f'Playwright only pre navigation hook for: {context.request.url} ...'
|
||||
)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
'html': str(context.soup)[:1000],
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(
|
||||
[
|
||||
'https://crawlee.dev',
|
||||
'https://apify.com',
|
||||
'https://example.com',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.storages import Dataset
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Open dataset manually using asynchronous constructor open().
|
||||
dataset = await Dataset.open()
|
||||
|
||||
# Interact with dataset directly.
|
||||
await dataset.push_data({'key': 'value'})
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.page.title(),
|
||||
'html': str(await context.page.content())[:1000],
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(
|
||||
[
|
||||
'https://crawlee.dev',
|
||||
'https://apify.com',
|
||||
'https://example.com',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee.crawlers import (
|
||||
BasicCrawlingContext,
|
||||
BeautifulSoupCrawler,
|
||||
BeautifulSoupCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an instance of the BeautifulSoupCrawler class, a crawler that automatically
|
||||
# loads the URLs and parses their HTML using the BeautifulSoup library.
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# On error, retry each page at most once.
|
||||
max_request_retries=1,
|
||||
# Increase the timeout for processing each page to 30 seconds.
|
||||
request_handler_timeout=timedelta(seconds=30),
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
# The handler receives a context parameter, providing various properties and
|
||||
# helper methods. Here are a few key ones we use for demonstration:
|
||||
# - request: an instance of the Request class containing details such as the URL
|
||||
# being crawled and the HTTP method used.
|
||||
# - soup: the BeautifulSoup object containing the parsed HTML of the response.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
'h1s': [h1.text for h1 in context.soup.find_all('h1')],
|
||||
'h2s': [h2.text for h2 in context.soup.find_all('h2')],
|
||||
'h3s': [h3.text for h3 in context.soup.find_all('h3')],
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset. In local configuration,
|
||||
# the data will be stored as JSON files in ./storage/datasets/default.
|
||||
await context.push_data(data)
|
||||
|
||||
# Register pre navigation hook which will be called before each request.
|
||||
# This hook is optional and does not need to be defined at all.
|
||||
@crawler.pre_navigation_hook
|
||||
async def some_hook(context: BasicCrawlingContext) -> None:
|
||||
pass
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee._types import BasicCrawlingContext
|
||||
from crawlee.crawlers import BeautifulSoupCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Keep the crawler alive even when there are no requests to be processed now.
|
||||
keep_alive=True,
|
||||
)
|
||||
|
||||
def stop_crawler_if_url_visited(context: BasicCrawlingContext) -> None:
|
||||
"""Stop crawler once specific url is visited.
|
||||
|
||||
Example of guard condition to stop the crawler."""
|
||||
if context.request.url == 'https://crawlee.dev/docs/examples':
|
||||
crawler.stop(
|
||||
'Stop crawler that was in keep_alive state after specific url was visite'
|
||||
)
|
||||
else:
|
||||
context.log.info('keep_alive=True, waiting for more requests to come.')
|
||||
|
||||
async def add_request_later(url: str, after_s: int) -> None:
|
||||
"""Add requests to the queue after some time. Can be done by external code."""
|
||||
# Just an example of request being added to the crawler later,
|
||||
# when it is waiting due to `keep_alive=True`.
|
||||
await asyncio.sleep(after_s)
|
||||
await crawler.add_requests([url])
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BasicCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Stop crawler if some guard condition has been met.
|
||||
stop_crawler_if_url_visited(context)
|
||||
|
||||
# Start some tasks that will add some requests later to simulate real situation,
|
||||
# where requests are added later by external code.
|
||||
add_request_later_task1 = asyncio.create_task(
|
||||
add_request_later(url='https://crawlee.dev', after_s=1)
|
||||
)
|
||||
add_request_later_task2 = asyncio.create_task(
|
||||
add_request_later(url='https://crawlee.dev/docs/examples', after_s=5)
|
||||
)
|
||||
|
||||
# Run the crawler without the initial list of requests.
|
||||
# Wait for more requests to be added to the queue later due to `keep_alive=True`.
|
||||
await crawler.run()
|
||||
|
||||
await asyncio.gather(add_request_later_task1, add_request_later_task2)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an instance of the BeautifulSoupCrawler class, a crawler that automatically
|
||||
# loads the URLs and parses their HTML using the BeautifulSoup library.
|
||||
crawler = BeautifulSoupCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
# The handler receives a context parameter, providing various properties and
|
||||
# helper methods. Here are a few key ones we use for demonstration:
|
||||
# - request: an instance of the Request class containing details such as the URL
|
||||
# being crawled and the HTTP method used.
|
||||
# - soup: the BeautifulSoup object containing the parsed HTML of the response.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Create custom condition to stop crawler once it finds what it is looking for.
|
||||
if 'crawlee' in context.request.url:
|
||||
crawler.stop(
|
||||
reason='Manual stop of crawler after finding `crawlee` in the url.'
|
||||
)
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset. In local configuration,
|
||||
# the data will be stored as JSON files in ./storage/datasets/default.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storages import KeyValueStore
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
# Headless mode, set to False to see the browser in action.
|
||||
headless=False,
|
||||
# Browser types supported by Playwright.
|
||||
browser_type='chromium',
|
||||
)
|
||||
|
||||
# Open the default key-value store.
|
||||
kvs = await KeyValueStore.open()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Capture the screenshot of the page using Playwright's API.
|
||||
screenshot = await context.page.screenshot()
|
||||
name = context.request.url.split('/')[-1]
|
||||
|
||||
# Store the screenshot in the key-value store.
|
||||
await kvs.set_value(
|
||||
key=f'screenshot-{name}',
|
||||
value=screenshot,
|
||||
content_type='image/png',
|
||||
)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(
|
||||
[
|
||||
'https://crawlee.dev',
|
||||
'https://apify.com',
|
||||
'https://example.com',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from loguru import Record
|
||||
|
||||
|
||||
# Configure loguru interceptor to capture standard logging output
|
||||
class InterceptHandler(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
# Get corresponding Loguru level if it exists
|
||||
try:
|
||||
level: str | int = logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
|
||||
# Find caller from where originated the logged message
|
||||
frame, depth = inspect.currentframe(), 0
|
||||
while frame:
|
||||
filename = frame.f_code.co_filename
|
||||
is_logging = filename == logging.__file__
|
||||
is_frozen = 'importlib' in filename and '_bootstrap' in filename
|
||||
if depth > 0 and not (is_logging | is_frozen):
|
||||
break
|
||||
frame = frame.f_back
|
||||
depth += 1
|
||||
|
||||
dummy_record = logging.LogRecord('dummy', 0, 'dummy', 0, 'dummy', None, None)
|
||||
standard_attrs = set(dummy_record.__dict__.keys())
|
||||
extra_dict = {
|
||||
key: value
|
||||
for key, value in record.__dict__.items()
|
||||
if key not in standard_attrs
|
||||
}
|
||||
|
||||
(
|
||||
logger.bind(**extra_dict)
|
||||
.opt(depth=depth, exception=record.exc_info)
|
||||
.patch(lambda loguru_record: loguru_record.update({'name': record.name}))
|
||||
.log(level, record.getMessage())
|
||||
)
|
||||
|
||||
|
||||
# Configure loguru formatter
|
||||
def formatter(record: Record) -> str:
|
||||
basic_format = '[{name}] | <level>{level: ^8}</level> | - {message}'
|
||||
if record['extra']:
|
||||
basic_format = basic_format + ' {extra}'
|
||||
return f'{basic_format}\n'
|
||||
|
||||
|
||||
# Remove default loguru logger
|
||||
logger.remove()
|
||||
|
||||
# Set up loguru with JSONL serialization in file `crawler.log`
|
||||
logger.add('crawler.log', format=formatter, serialize=True, level='INFO')
|
||||
|
||||
# Set up loguru logger for console
|
||||
logger.add(sys.stderr, format=formatter, colorize=True, level='INFO')
|
||||
|
||||
# Configure standard logging to use our interceptor
|
||||
logging.basicConfig(handlers=[InterceptHandler()], level=logging.INFO, force=True)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Initialize crawler with disabled table logs
|
||||
crawler = HttpCrawler(
|
||||
configure_logging=False, # Disable default logging configuration
|
||||
statistics_log_format='inline', # Set inline formatting for statistics logs
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Run the crawler
|
||||
await crawler.run(['https://www.crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(
|
||||
[
|
||||
'https://crawlee.dev',
|
||||
'https://apify.com',
|
||||
'https://example.com',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(
|
||||
[
|
||||
'https://crawlee.dev',
|
||||
'https://apify.com',
|
||||
'https://example.com',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import Glob
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all the documentation links found on the page, except for the examples.
|
||||
await context.enqueue_links(
|
||||
include=[Glob('https://crawlee.dev/docs/**')],
|
||||
exclude=[Glob('https://crawlee.dev/docs/examples')],
|
||||
)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import Glob
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all the documentation links found on the page, except for the examples.
|
||||
await context.enqueue_links(
|
||||
include=[Glob('https://crawlee.dev/docs/**')],
|
||||
exclude=[Glob('https://crawlee.dev/docs/examples')],
|
||||
)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links found on the page. Any URLs found will be matched by
|
||||
# this strategy, even if they go off the site you are currently crawling.
|
||||
await context.enqueue_links(strategy='all')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Setting the strategy to same domain will enqueue all links found that
|
||||
# are on the same hostname as request.loaded_url or request.url.
|
||||
await context.enqueue_links(strategy='same-domain')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Setting the strategy to same hostname will enqueue all links found that are on
|
||||
# the same hostname (including subdomains) as request.loaded_url or request.url.
|
||||
await context.enqueue_links(strategy='same-hostname')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Setting the strategy to same origin will enqueue all links found that are on
|
||||
# the same origin as request.loaded_url or request.url.
|
||||
await context.enqueue_links(strategy='same-origin')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import asyncio
|
||||
import csv
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# Export the entire dataset to a CSV file.
|
||||
# Use semicolon as delimiter and always quote strings.
|
||||
await crawler.export_data(path='results.csv', delimiter=';', quoting=csv.QUOTE_ALL)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# Export the entire dataset to a JSON file.
|
||||
# Set ensure_ascii=False to allow Unicode characters in the output.
|
||||
await crawler.export_data(path='results.json', ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import Glob
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract all the documentation links found on the page, except for the examples.
|
||||
extracted_links = await context.extract_links(
|
||||
include=[Glob('https://crawlee.dev/docs/**')],
|
||||
exclude=[Glob('https://crawlee.dev/docs/examples')],
|
||||
)
|
||||
# Some very custom filtering which can't be achieved by `extract_links` arguments.
|
||||
max_link_length = 30
|
||||
filtered_links = [
|
||||
link for link in extracted_links if len(link.url) < max_link_length
|
||||
]
|
||||
# Add filtered links to the request queue.
|
||||
await context.add_requests(filtered_links)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import Glob
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract all the documentation links found on the page, except for the examples.
|
||||
extracted_links = await context.extract_links(
|
||||
include=[Glob('https://crawlee.dev/docs/**')],
|
||||
exclude=[Glob('https://crawlee.dev/docs/examples')],
|
||||
)
|
||||
# Some very custom filtering which can't be achieved by `extract_links` arguments.
|
||||
max_link_length = 30
|
||||
filtered_links = [
|
||||
link for link in extracted_links if len(link.url) < max_link_length
|
||||
]
|
||||
# Add filtered links to the request queue.
|
||||
await context.add_requests(filtered_links)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
import asyncio
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
response = (await context.http_response.read()).decode('utf-8')
|
||||
context.log.info(f'Response: {response}') # To see the response in the logs.
|
||||
|
||||
# Prepare a POST request to the form endpoint.
|
||||
request = Request.from_url(
|
||||
url='https://httpbin.org/post',
|
||||
method='POST',
|
||||
headers={'content-type': 'application/x-www-form-urlencoded'},
|
||||
payload=urlencode(
|
||||
{
|
||||
'custname': 'John Doe',
|
||||
'custtel': '1234567890',
|
||||
'custemail': 'johndoe@example.com',
|
||||
'size': 'large',
|
||||
'topping': ['bacon', 'cheese', 'mushroom'],
|
||||
'delivery': '13:00',
|
||||
'comments': 'Please ring the doorbell upon arrival.',
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run([request])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import asyncio
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from crawlee import Request
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Prepare a POST request to the form endpoint.
|
||||
request = Request.from_url(
|
||||
url='https://httpbin.org/post',
|
||||
method='POST',
|
||||
headers={'content-type': 'application/x-www-form-urlencoded'},
|
||||
payload=urlencode(
|
||||
{
|
||||
'custname': 'John Doe',
|
||||
'custtel': '1234567890',
|
||||
'custemail': 'johndoe@example.com',
|
||||
'size': 'large',
|
||||
'topping': ['bacon', 'cheese', 'mushroom'],
|
||||
'delivery': '13:00',
|
||||
'comments': 'Please ring the doorbell upon arrival.',
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
|
||||
|
||||
# Regex for identifying email addresses on a webpage.
|
||||
EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = ParselCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
'email_address_list': context.selector.re(EMAIL_REGEX),
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Enqueue all links found on the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Register pre navigation hook which will be called before each request.
|
||||
# This hook is optional and does not need to be defined at all.
|
||||
@crawler.pre_navigation_hook
|
||||
async def some_hook(context: BasicCrawlingContext) -> None:
|
||||
pass
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://github.com'])
|
||||
|
||||
# Export the entire dataset to a JSON file.
|
||||
await crawler.export_data(path='results.json')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import asyncio
|
||||
from random import choice
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.statistics import Statistics
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = ParselCrawler(
|
||||
statistics=Statistics.with_default_state(save_error_snapshots=True)
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
# Simulate various errors to demonstrate `ErrorSnapshotter`
|
||||
# saving only the first occurrence of unique error.
|
||||
await context.enqueue_links()
|
||||
random_number = choice(range(10))
|
||||
if random_number == 1:
|
||||
raise KeyError('Some KeyError')
|
||||
if random_number == 2:
|
||||
raise ValueError('Some ValueError')
|
||||
if random_number == 3:
|
||||
raise RuntimeError('Some RuntimeError')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Define the hook, which will be called before every request.
|
||||
@crawler.pre_navigation_hook
|
||||
async def navigation_hook(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
context.log.info(f'Navigating to {context.request.url} ...')
|
||||
|
||||
# Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
|
||||
await context.block_requests(extra_url_patterns=['adsbygoogle.js'])
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
# Headless mode, set to False to see the browser in action.
|
||||
headless=False,
|
||||
# Browser types supported by Playwright.
|
||||
browser_type='chromium',
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
# The handler receives a context parameter, providing various properties and
|
||||
# helper methods. Here are a few key ones we use for demonstration:
|
||||
# - request: an instance of the Request class containing details such as the URL
|
||||
# being crawled and the HTTP method used.
|
||||
# - page: Playwright's Page object, which allows interaction with the web page
|
||||
# (see https://playwright.dev/python/docs/api/class-page for more details).
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract data from the page using Playwright's API.
|
||||
posts = await context.page.query_selector_all('.athing')
|
||||
data = []
|
||||
|
||||
for post in posts:
|
||||
# Get the HTML elements for the title and rank within each post.
|
||||
title_element = await post.query_selector('.title a')
|
||||
rank_element = await post.query_selector('.rank')
|
||||
|
||||
# Extract the data we want from the elements.
|
||||
title = await title_element.inner_text() if title_element else None
|
||||
rank = await rank_element.inner_text() if rank_element else None
|
||||
href = await title_element.get_attribute('href') if title_element else None
|
||||
|
||||
data.append({'title': title, 'rank': rank, 'href': href})
|
||||
|
||||
# Push the extracted data to the default dataset. In local configuration,
|
||||
# the data will be stored as JSON files in ./storage/datasets/default.
|
||||
await context.push_data(data)
|
||||
|
||||
# Find a link to the next page and enqueue it if it exists.
|
||||
await context.enqueue_links(selector='.morelink')
|
||||
|
||||
# Define a hook that will be called each time before navigating to a new URL.
|
||||
# The hook receives a context parameter, providing access to the request and
|
||||
# browser page among other things. In this example, we log the URL being
|
||||
# navigated to.
|
||||
@crawler.pre_navigation_hook
|
||||
async def log_navigation_url(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
context.log.info(f'Navigating to {context.request.url} ...')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://news.ycombinator.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import asyncio
|
||||
|
||||
# Camoufox is external package and needs to be installed. It is not included in crawlee.
|
||||
from camoufox import AsyncNewBrowser
|
||||
from typing_extensions import override
|
||||
|
||||
from crawlee.browsers import (
|
||||
BrowserPool,
|
||||
PlaywrightBrowserController,
|
||||
PlaywrightBrowserPlugin,
|
||||
)
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
class CamoufoxPlugin(PlaywrightBrowserPlugin):
|
||||
"""Example browser plugin that uses Camoufox browser,
|
||||
but otherwise keeps the functionality of PlaywrightBrowserPlugin.
|
||||
"""
|
||||
|
||||
@override
|
||||
async def new_browser(self) -> PlaywrightBrowserController:
|
||||
if not self._playwright:
|
||||
raise RuntimeError('Playwright browser plugin is not initialized.')
|
||||
|
||||
return PlaywrightBrowserController(
|
||||
browser=await AsyncNewBrowser(
|
||||
self._playwright, **self._browser_launch_options
|
||||
),
|
||||
# Increase, if camoufox can handle it in your use case.
|
||||
max_open_pages_per_browser=1,
|
||||
# This turns off the crawlee header_generation. Camoufox has its own.
|
||||
header_generator=None,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
# Custom browser pool. Gives users full control over browsers used by the crawler.
|
||||
browser_pool=BrowserPool(plugins=[CamoufoxPlugin()]),
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract some data from the page using Playwright's API.
|
||||
posts = await context.page.query_selector_all('.athing')
|
||||
for post in posts:
|
||||
# Get the HTML elements for the title and rank within each post.
|
||||
title_element = await post.query_selector('.title a')
|
||||
|
||||
# Extract the data we want from the elements.
|
||||
title = await title_element.inner_text() if title_element else None
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data({'title': title})
|
||||
|
||||
# Find a link to the next page and enqueue it if it exists.
|
||||
await context.enqueue_links(selector='.morelink')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://news.ycombinator.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import asyncio
|
||||
from random import choice
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.statistics import Statistics
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
statistics=Statistics.with_default_state(save_error_snapshots=True)
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
# Simulate various errors to demonstrate `ErrorSnapshotter`
|
||||
# saving only the first occurrence of unique error.
|
||||
await context.enqueue_links()
|
||||
random_number = choice(range(10))
|
||||
if random_number == 1:
|
||||
raise KeyError('Some KeyError')
|
||||
if random_number == 2:
|
||||
raise ValueError('Some ValueError')
|
||||
if random_number == 3:
|
||||
raise RuntimeError('Some RuntimeError')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.fingerprint_suite import (
|
||||
DefaultFingerprintGenerator,
|
||||
HeaderGeneratorOptions,
|
||||
ScreenOptions,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Use default fingerprint generator with desired fingerprint options.
|
||||
# Generator will generate real looking browser fingerprint based on the options.
|
||||
# Unspecified fingerprint options will be automatically selected by the generator.
|
||||
fingerprint_generator = DefaultFingerprintGenerator(
|
||||
header_options=HeaderGeneratorOptions(browsers=['chrome']),
|
||||
screen_options=ScreenOptions(min_width=400),
|
||||
)
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
# Headless mode, set to False to see the browser in action.
|
||||
headless=False,
|
||||
# Browser types supported by Playwright.
|
||||
browser_type='chromium',
|
||||
# Fingerprint generator to be used. By default no fingerprint generation is done.
|
||||
fingerprint_generator=fingerprint_generator,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Find a link to the next page and enqueue it if it exists.
|
||||
await context.enqueue_links(selector='.morelink')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://news.ycombinator.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import SkippedReason
|
||||
from crawlee.crawlers import (
|
||||
BeautifulSoupCrawler,
|
||||
BeautifulSoupCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Initialize the crawler with robots.txt compliance enabled
|
||||
crawler = BeautifulSoupCrawler(respect_robots_txt_file=True)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# highlight-start
|
||||
# This handler is called when a request is skipped
|
||||
@crawler.on_skipped_request
|
||||
async def skipped_request_handler(url: str, reason: SkippedReason) -> None:
|
||||
# Check if the request was skipped due to robots.txt rules
|
||||
if reason == 'robots_txt':
|
||||
crawler.log.info(f'Skipped {url} due to robots.txt rules.')
|
||||
|
||||
# highlight-end
|
||||
|
||||
# Start the crawler with the specified URLs
|
||||
# The login URL will be skipped and handled by the skipped_request_handler
|
||||
await crawler.run(
|
||||
['https://news.ycombinator.com/', 'https://news.ycombinator.com/login']
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
BeautifulSoupCrawler,
|
||||
BeautifulSoupCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Initialize the crawler with robots.txt compliance enabled
|
||||
crawler = BeautifulSoupCrawler(respect_robots_txt_file=True)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Start the crawler with the specified URLs
|
||||
# The crawler will check the robots.txt file before making requests
|
||||
# In this example, 'https://news.ycombinator.com/login' will be skipped
|
||||
# because it's disallowed in the site's robots.txt file
|
||||
await crawler.run(
|
||||
['https://news.ycombinator.com/', 'https://news.ycombinator.com/login']
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import ConcurrencySettings, service_locator
|
||||
from crawlee.crawlers import (
|
||||
BeautifulSoupCrawler,
|
||||
BeautifulSoupCrawlingContext,
|
||||
)
|
||||
|
||||
# Disable clearing the `RequestQueue`, `KeyValueStore` and `Dataset` on each run.
|
||||
# This makes the scraper continue from where it left off in the previous run.
|
||||
# The recommended way to achieve this behavior is setting the environment variable
|
||||
# `CRAWLEE_PURGE_ON_START=0`
|
||||
configuration = service_locator.get_configuration()
|
||||
configuration.purge_on_start = False
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Let's slow down the crawler for a demonstration
|
||||
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=20)
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# List of links for crawl
|
||||
requests = [
|
||||
'https://crawlee.dev',
|
||||
'https://crawlee.dev/python/docs',
|
||||
'https://crawlee.dev/python/docs/examples',
|
||||
'https://crawlee.dev/python/docs/guides',
|
||||
'https://crawlee.dev/python/docs/quick-start',
|
||||
]
|
||||
|
||||
await crawler.run(requests)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import asyncio
|
||||
|
||||
from crawlee import ConcurrencySettings
|
||||
from crawlee.crawlers import (
|
||||
ParselCrawler,
|
||||
ParselCrawlingContext,
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
)
|
||||
from crawlee.sessions import SessionPool
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Open request queues for both crawlers with different aliases
|
||||
playwright_rq = await RequestQueue.open(alias='playwright-requests')
|
||||
parsel_rq = await RequestQueue.open(alias='parsel-requests')
|
||||
|
||||
# Use a shared session pool between both crawlers
|
||||
async with SessionPool() as session_pool:
|
||||
playwright_crawler = PlaywrightCrawler(
|
||||
# Set the request queue for Playwright crawler
|
||||
request_manager=playwright_rq,
|
||||
session_pool=session_pool,
|
||||
# Configure concurrency settings for Playwright crawler
|
||||
concurrency_settings=ConcurrencySettings(
|
||||
max_concurrency=5, desired_concurrency=5
|
||||
),
|
||||
# Set `keep_alive`` so that the crawler does not stop working when there are
|
||||
# no requests in the queue.
|
||||
keep_alive=True,
|
||||
)
|
||||
|
||||
parsel_crawler = ParselCrawler(
|
||||
# Set the request queue for Parsel crawler
|
||||
request_manager=parsel_rq,
|
||||
session_pool=session_pool,
|
||||
# Configure concurrency settings for Parsel crawler
|
||||
concurrency_settings=ConcurrencySettings(
|
||||
max_concurrency=10, desired_concurrency=10
|
||||
),
|
||||
# Set maximum requests per crawl for Parsel crawler
|
||||
max_requests_per_crawl=50,
|
||||
)
|
||||
|
||||
@playwright_crawler.router.default_handler
|
||||
async def handle_playwright(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Playwright Processing {context.request.url}...')
|
||||
|
||||
title = await context.page.title()
|
||||
# Push the extracted data to the dataset for Playwright crawler
|
||||
await context.push_data(
|
||||
{'title': title, 'url': context.request.url, 'source': 'playwright'},
|
||||
dataset_name='playwright-data',
|
||||
)
|
||||
|
||||
@parsel_crawler.router.default_handler
|
||||
async def handle_parsel(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Parsel Processing {context.request.url}...')
|
||||
|
||||
title = context.parsed_content.css('title::text').get()
|
||||
# Push the extracted data to the dataset for Parsel crawler
|
||||
await context.push_data(
|
||||
{'title': title, 'url': context.request.url, 'source': 'parsel'},
|
||||
dataset_name='parsel-data',
|
||||
)
|
||||
|
||||
# Enqueue links to the Playwright request queue for blog pages
|
||||
await context.enqueue_links(
|
||||
selector='a[href*="/blog/"]', rq_alias='playwright-requests'
|
||||
)
|
||||
# Enqueue other links to the Parsel request queue
|
||||
await context.enqueue_links(selector='a:not([href*="/blog/"])')
|
||||
|
||||
# Start the Playwright crawler in the background
|
||||
background_crawler_task = asyncio.create_task(playwright_crawler.run([]))
|
||||
|
||||
# Run the Parsel crawler with the initial URL and wait for it to finish
|
||||
await parsel_crawler.run(['https://crawlee.dev/blog'])
|
||||
|
||||
# Wait for the Playwright crawler to finish processing all requests
|
||||
while not await playwright_rq.is_empty():
|
||||
playwright_crawler.log.info('Waiting for Playwright crawler to finish...')
|
||||
await asyncio.sleep(5)
|
||||
|
||||
# Stop the Playwright crawler after all requests are processed
|
||||
playwright_crawler.stop()
|
||||
|
||||
# Wait for the background Playwright crawler task to complete
|
||||
await background_crawler_task
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
import asyncio
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
# Profile name to use (usually 'Default' for single profile setups)
|
||||
PROFILE_NAME = 'Default'
|
||||
|
||||
# Paths to Chrome profiles in your system (example for Windows)
|
||||
# Use `chrome://version/` to find your profile path
|
||||
PROFILE_PATH = Path(Path.home(), 'AppData', 'Local', 'Google', 'Chrome', 'User Data')
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a temporary folder to copy the profile to
|
||||
with TemporaryDirectory(prefix='crawlee-') as tmpdirname:
|
||||
tmp_profile_dir = Path(tmpdirname)
|
||||
|
||||
# Copy the profile to a temporary folder
|
||||
shutil.copytree(
|
||||
PROFILE_PATH / PROFILE_NAME,
|
||||
tmp_profile_dir / PROFILE_NAME,
|
||||
dirs_exist_ok=True,
|
||||
)
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
headless=False,
|
||||
# Use the installed Chrome browser
|
||||
browser_type='chrome',
|
||||
# Disable fingerprints to preserve profile identity
|
||||
fingerprint_generator=None,
|
||||
# Set user data directory to temp folder
|
||||
user_data_dir=tmp_profile_dir,
|
||||
browser_launch_options={
|
||||
# Slow down actions to mimic human behavior
|
||||
'slow_mo': 200,
|
||||
'args': [
|
||||
# Use the specified profile
|
||||
f'--profile-directory={PROFILE_NAME}',
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Visiting {context.request.url}')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
# Replace this with your actual Firefox profile name
|
||||
# Find it at about:profiles in Firefox
|
||||
PROFILE_NAME = 'your-profile-name-here'
|
||||
|
||||
# Paths to Firefox profiles in your system (example for Windows)
|
||||
# Use `about:profiles` to find your profile path
|
||||
PROFILE_PATH = Path(
|
||||
Path.home(), 'AppData', 'Roaming', 'Mozilla', 'Firefox', 'Profiles', PROFILE_NAME
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Use Firefox browser type
|
||||
browser_type='firefox',
|
||||
# Disable fingerprints to use the profile as is
|
||||
fingerprint_generator=None,
|
||||
headless=False,
|
||||
# Path to your Firefox profile
|
||||
user_data_dir=PROFILE_PATH,
|
||||
browser_launch_options={
|
||||
'args': [
|
||||
# Required to avoid version conflicts
|
||||
'--allow-downgrade'
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Visiting {context.request.url}')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
import asyncio
|
||||
from collections.abc import Callable
|
||||
|
||||
from yarl import URL
|
||||
|
||||
from crawlee import RequestOptions, RequestTransformAction
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
from crawlee.request_loaders import SitemapRequestLoader
|
||||
|
||||
|
||||
# Create a transform_request_function that maps request options based on the host in
|
||||
# the URL
|
||||
def create_transform_request(
|
||||
data_mapper: dict[str, dict],
|
||||
) -> Callable[[RequestOptions], RequestOptions | RequestTransformAction]:
|
||||
def transform_request(
|
||||
request_options: RequestOptions,
|
||||
) -> RequestOptions | RequestTransformAction:
|
||||
# According to the Sitemap protocol, all URLs in a Sitemap must be from a single
|
||||
# host.
|
||||
request_host = URL(request_options['url']).host
|
||||
|
||||
if request_host and (mapping_data := data_mapper.get(request_host)):
|
||||
# Set properties from the mapping data
|
||||
if 'label' in mapping_data:
|
||||
request_options['label'] = mapping_data['label']
|
||||
if 'user_data' in mapping_data:
|
||||
request_options['user_data'] = mapping_data['user_data']
|
||||
|
||||
return request_options
|
||||
|
||||
return 'unchanged'
|
||||
|
||||
return transform_request
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Prepare data mapping for hosts
|
||||
apify_host = URL('https://apify.com/sitemap.xml').host
|
||||
crawlee_host = URL('https://crawlee.dev/sitemap.xml').host
|
||||
|
||||
if not apify_host or not crawlee_host:
|
||||
raise ValueError('Unable to extract host from URLs')
|
||||
|
||||
data_map = {
|
||||
apify_host: {
|
||||
'label': 'apify',
|
||||
'user_data': {'source': 'apify'},
|
||||
},
|
||||
crawlee_host: {
|
||||
'label': 'crawlee',
|
||||
'user_data': {'source': 'crawlee'},
|
||||
},
|
||||
}
|
||||
|
||||
# Initialize the SitemapRequestLoader with the transform function
|
||||
async with SitemapRequestLoader(
|
||||
# Set the sitemap URLs and the HTTP client
|
||||
sitemap_urls=['https://crawlee.dev/sitemap.xml', 'https://apify.com/sitemap.xml'],
|
||||
http_client=ImpitHttpClient(),
|
||||
transform_request_function=create_transform_request(data_map),
|
||||
) as sitemap_loader:
|
||||
# Convert the sitemap loader to a request manager
|
||||
request_manager = await sitemap_loader.to_tandem()
|
||||
|
||||
# Create and configure the crawler
|
||||
crawler = BeautifulSoupCrawler(
|
||||
request_manager=request_manager,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Create default handler for requests without a specific label
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
source = context.request.user_data.get('source', 'unknown')
|
||||
context.log.info(
|
||||
f'Processing request: {context.request.url} from source: {source}'
|
||||
)
|
||||
|
||||
# Create handler for requests labeled 'apify'
|
||||
@crawler.router.handler('apify')
|
||||
async def apify_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
source = context.request.user_data.get('source', 'unknown')
|
||||
context.log.info(
|
||||
f'Apify handler processing: {context.request.url} from source: {source}'
|
||||
)
|
||||
|
||||
# Create handler for requests labeled 'crawlee'
|
||||
@crawler.router.handler('crawlee')
|
||||
async def crawlee_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
source = context.request.user_data.get('source', 'unknown')
|
||||
context.log.info(
|
||||
f'Crawlee handler processing: {context.request.url} from source: {source}'
|
||||
)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
id: crawl-all-links-on-website
|
||||
title: Crawl all links on website
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_bs.py';
|
||||
import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_pw.py';
|
||||
|
||||
This example uses the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper to add new links to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink> as the crawler navigates from page to page. By automatically discovering and enqueuing all links on a given page, the crawler can systematically scrape an entire website. This approach is ideal for web scraping tasks where you need to collect data from multiple interconnected pages.
|
||||
|
||||
:::tip
|
||||
|
||||
If no options are given, by default the method will only add links that are under the same subdomain. This behavior can be controlled with the `strategy` option, which is an instance of the `EnqueueStrategy` type alias. You can find more info about this option in the [Crawl website with relative links](./crawl-website-with-relative-links) example.
|
||||
|
||||
:::
|
||||
|
||||
<Tabs groupId="main">
|
||||
<TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
id: crawl-multiple-urls
|
||||
title: Crawl multiple URLs
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_bs.py';
|
||||
import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_pw.py';
|
||||
|
||||
This example demonstrates how to crawl a specified list of URLs using different crawlers. You'll learn how to set up the crawler, define a request handler, and run the crawler with multiple URLs. This setup is useful for scraping data from multiple pages or websites concurrently.
|
||||
|
||||
<Tabs groupId="main">
|
||||
<TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
---
|
||||
id: crawl-specific-links-on-website
|
||||
title: Crawl specific links on website
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_bs.py';
|
||||
import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_pw.py';
|
||||
|
||||
import BeautifulSoupExampleExtractAndAdd from '!!raw-loader!roa-loader!./code_examples/extract_and_add_specific_links_on_website_bs.py';
|
||||
import PlaywrightExampleExtractAndAdd from '!!raw-loader!roa-loader!./code_examples/extract_and_add_specific_links_on_website_pw.py';
|
||||
|
||||
This example demonstrates how to crawl a website while targeting specific patterns of links. By utilizing the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> helper, you can pass `include` or `exclude` parameters to improve your crawling strategy. This approach ensures that only the links matching the specified patterns are added to the <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>. Both `include` and `exclude` support lists of globs or regular expressions. This functionality is great for focusing on relevant sections of a website and avoiding scraping unnecessary or irrelevant content.
|
||||
|
||||
<Tabs groupId="first-example">
|
||||
<TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Even more control over the enqueued links
|
||||
|
||||
<ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> is a convenience helper and internally it calls <ApiLink to="class/ExtractLinksFunction">`extract_links`</ApiLink> to find the links and <ApiLink to="class/AddRequestsFunction">`add_requests`</ApiLink> to add them to the queue. If you need some additional custom filtering of the extracted links before enqueuing them, then consider using <ApiLink to="class/ExtractLinksFunction">`extract_links`</ApiLink> and <ApiLink to="class/AddRequestsFunction">`add_requests`</ApiLink> instead of the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink>
|
||||
|
||||
<Tabs groupId="second-example">
|
||||
<TabItem value="BeautifulSoupCrawler" label="BeautifulSoupCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExampleExtractAndAdd}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="PlaywrightCrawler" label="PlaywrightCrawler">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightExampleExtractAndAdd}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
id: crawl-website-with-relative-links
|
||||
title: Crawl website with relative links
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import AllLinksExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_all_links.py';
|
||||
import SameDomainExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_domain.py';
|
||||
import SameHostnameExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py';
|
||||
import SameOriginExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_origin.py';
|
||||
|
||||
When crawling a website, you may encounter various types of links that you wish to include in your crawl. To facilitate this, we provide the <ApiLink to="class/EnqueueLinksFunction">`enqueue_links`</ApiLink> method on the crawler context, which will automatically find and add these links to the crawler's <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>. This method simplifies the process of handling different types of links, including relative links, by automatically resolving them based on the page's context.
|
||||
|
||||
:::note
|
||||
|
||||
For these examples, we are using the <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. However, the same method is available for other crawlers as well. You can use it in exactly the same way.
|
||||
|
||||
:::
|
||||
|
||||
`EnqueueStrategy` type alias provides four distinct strategies for crawling relative links:
|
||||
|
||||
- `all` - Enqueues all links found, regardless of the domain they point to. This strategy is useful when you want to follow every link, including those that navigate to external websites.
|
||||
- `same-domain` - Enqueues all links found that share the same domain name, including any possible subdomains. This strategy ensures that all links within the same top-level and base domain are included.
|
||||
- `same-hostname` - Enqueues all links found for the exact same hostname. This is the **default** strategy, and it restricts the crawl to links that have the same hostname as the current page, excluding subdomains.
|
||||
- `same-origin` - Enqueues all links found that share the same origin. The same origin refers to URLs that share the same protocol, domain, and port, ensuring a strict scope for the crawl.
|
||||
|
||||
<Tabs groupId="main">
|
||||
<TabItem value="all_links" label="All links">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{AllLinksExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="same-domain" label="Same domain">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{SameDomainExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="same-hostname" label="Same hostname">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{SameHostnameExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="same-origin" label="Same origin">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{SameOriginExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
id: crawler-keep-alive
|
||||
title: Keep a Crawler alive waiting for more requests
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_keep_alive.py';
|
||||
|
||||
This example demonstrates how to keep crawler alive even when there are no requests at the moment by using `keep_alive=True` argument of <ApiLink to="class/BasicCrawler#__init__">`BasicCrawler.__init__`</ApiLink>. This is available to all crawlers that inherit from <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> and in the example below it is shown on <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. To stop the crawler that was started with `keep_alive=True` you can call `crawler.stop()`.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
id: crawler-stop
|
||||
title: Stopping a Crawler with stop method
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_stop.py';
|
||||
|
||||
This example demonstrates how to use `stop` method of <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> to stop crawler once the crawler finds what it is looking for. This method is available to all crawlers that inherit from <ApiLink to="class/BasicCrawler">`BasicCrawler`</ApiLink> and in the example below it is shown on <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. Simply call `crawler.stop()` to stop the crawler. It will not continue to crawl through new requests. Requests that are already being concurrently processed are going to get finished. It is possible to call `stop` method with optional argument `reason` that is a string that will be used in logs and it can improve logs readability especially if you have multiple different conditions for triggering `stop`.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{BeautifulSoupExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
id: export-entire-dataset-to-file
|
||||
title: Export entire dataset to file
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import JsonExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_json.py';
|
||||
import CsvExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_csv.py';
|
||||
|
||||
This example demonstrates how to use the <ApiLink to="class/BasicCrawler#export_data">`BasicCrawler.export_data`</ApiLink> method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format and also accepts additional keyword arguments so you can fine-tune the underlying `json.dump` or `csv.writer` behavior.
|
||||
|
||||
:::note
|
||||
|
||||
For these examples, we are using the <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>. However, the same method is available for other crawlers as well. You can use it in exactly the same way.
|
||||
|
||||
:::
|
||||
|
||||
<Tabs groupId="main">
|
||||
<TabItem value="json" label="JSON">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{JsonExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
<TabItem value="csv" label="CSV">
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{CsvExample}
|
||||
</RunnableCodeBlock>
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
---
|
||||
id: fill-and-submit-web-form
|
||||
title: Fill and submit web form
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_request.py';
|
||||
import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_crawler.py';
|
||||
|
||||
This example demonstrates how to fill and submit a web form using the <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink> crawler. The same approach applies to any crawler that inherits from it, such as the <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink> or <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink>.
|
||||
|
||||
We are going to use the [httpbin.org](https://httpbin.org) website to demonstrate how it works.
|
||||
|
||||
## Investigate the form fields
|
||||
|
||||
First, we need to examine the form fields and the form's action URL. You can do this by opening the [httpbin.org/forms/post](https://httpbin.org/forms/post) page in a browser and inspecting the form fields.
|
||||
|
||||
In Chrome, right-click on the page and select "Inspect" or press `Ctrl+Shift+I`.
|
||||
Use the element selector (`Ctrl+Shift+C`) to click on the form element you want to inspect.
|
||||
|
||||
|
||||
|
||||
Identify the field names. For example, the customer name field is `custname`, the email field is `custemail`, and the phone field is `custtel`.
|
||||
|
||||
Now navigate to the "Network" tab in developer tools and submit the form by clicking the "Submit order" button.
|
||||
|
||||
|
||||
|
||||
Find the form submission request and examine its details. The "Headers" tab will show the submission URL, in this case, it is `https://httpbin.org/post`.
|
||||
|
||||
|
||||
|
||||
The "Payload" tab will display the form fields and their submitted values. This method could be an alternative to inspecting the HTML source code directly.
|
||||
|
||||
|
||||
|
||||
## Preparing a POST request
|
||||
|
||||
Now, let's create a POST request with the form fields and their values using the <ApiLink to="class/Request">`Request`</ApiLink> class, specifically its <ApiLink to="class/Request#from_url">`Request.from_url`</ApiLink> constructor:
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{RequestExample}
|
||||
</RunnableCodeBlock>
|
||||
|
||||
Alternatively, you can send form data as URL parameters using the `url` argument. It depends on the form and how it is implemented. However, sending the data as a POST request body using the `payload` is generally a better approach.
|
||||
|
||||
## Implementing the crawler
|
||||
|
||||
Finally, let's implement the crawler and run it with the prepared request. Although we are using the <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink>, the process is the same for any crawler that inherits from it.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{CrawlerExample}
|
||||
</RunnableCodeBlock>
|
||||
|
||||
## Running the crawler
|
||||
|
||||
Finally, run your crawler. Your logs should show something like this:
|
||||
|
||||
```plaintext
|
||||
...
|
||||
[crawlee.http_crawler._http_crawler] INFO Processing https://httpbin.org/post ...
|
||||
[crawlee.http_crawler._http_crawler] INFO Response: {
|
||||
"args": {},
|
||||
"data": "",
|
||||
"files": {},
|
||||
"form": {
|
||||
"comments": "Please ring the doorbell upon arrival.",
|
||||
"custemail": "johndoe@example.com",
|
||||
"custname": "John Doe",
|
||||
"custtel": "1234567890",
|
||||
"delivery": "13:00",
|
||||
"size": "large",
|
||||
"topping": [
|
||||
"bacon",
|
||||
"cheese",
|
||||
"mushroom"
|
||||
]
|
||||
},
|
||||
"headers": {
|
||||
"Accept": "*/*",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Length": "190",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Host": "httpbin.org",
|
||||
"User-Agent": "python-httpx/0.27.0",
|
||||
"X-Amzn-Trace-Id": "Root=1-66c849d6-1ae432fb7b4156e6149ff37f"
|
||||
},
|
||||
"json": null,
|
||||
"origin": "78.80.81.196",
|
||||
"url": "https://httpbin.org/post"
|
||||
}
|
||||
|
||||
[crawlee._autoscaling.autoscaled_pool] INFO Waiting for remaining tasks to finish
|
||||
[crawlee.http_crawler._http_crawler] INFO Final request statistics:
|
||||
┌───────────────────────────────┬──────────┐
|
||||
│ requests_finished │ 1 │
|
||||
│ requests_failed │ 0 │
|
||||
│ retry_histogram │ [1] │
|
||||
│ request_avg_failed_duration │ None │
|
||||
│ request_avg_finished_duration │ 0.678442 │
|
||||
│ requests_finished_per_minute │ 85 │
|
||||
│ requests_failed_per_minute │ 0 │
|
||||
│ request_total_duration │ 0.678442 │
|
||||
│ requests_total │ 1 │
|
||||
│ crawler_runtime │ 0.707666 │
|
||||
└───────────────────────────────┴──────────┘
|
||||
```
|
||||
|
||||
This log output confirms that the crawler successfully submitted the form and processed the response. Congratulations! You have successfully filled and submitted a web form using the <ApiLink to="class/HttpCrawler">`HttpCrawler`</ApiLink>.
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
---
|
||||
id: configure-json-logging
|
||||
title: Сonfigure JSON logging
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/configure_json_logging.py';
|
||||
|
||||
This example demonstrates how to configure JSON line (JSONL) logging with Crawlee. By using the `use_table_logs=False` parameter, you can disable table-formatted statistics logs, which makes it easier to parse logs with external tools or to serialize them as JSON.
|
||||
|
||||
The example shows how to integrate with the popular [`loguru`](https://github.com/delgan/loguru) library to capture Crawlee logs and format them as JSONL (one JSON object per line). This approach works well when you need to collect logs for analysis, monitoring, or when integrating with logging platforms like ELK Stack, Grafana Loki, or similar systems.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{JsonLoggingExample}
|
||||
</RunnableCodeBlock>
|
||||
|
||||
Here's an example of what a crawler statistics log entry in JSONL format.
|
||||
|
||||
```json
|
||||
{
|
||||
"text": "[HttpCrawler] | INFO | - Final request statistics: {'requests_finished': 1, 'requests_failed': 0, 'retry_histogram': [1], 'request_avg_failed_duration': None, 'request_avg_finished_duration': 3.57098, 'requests_finished_per_minute': 17, 'requests_failed_per_minute': 0, 'request_total_duration': 3.57098, 'requests_total': 1, 'crawler_runtime': 3.59165}\n",
|
||||
"record": {
|
||||
"elapsed": { "repr": "0:00:05.604568", "seconds": 5.604568 },
|
||||
"exception": null,
|
||||
"extra": {
|
||||
"requests_finished": 1,
|
||||
"requests_failed": 0,
|
||||
"retry_histogram": [1],
|
||||
"request_avg_failed_duration": null,
|
||||
"request_avg_finished_duration": 3.57098,
|
||||
"requests_finished_per_minute": 17,
|
||||
"requests_failed_per_minute": 0,
|
||||
"request_total_duration": 3.57098,
|
||||
"requests_total": 1,
|
||||
"crawler_runtime": 3.59165
|
||||
},
|
||||
"file": {
|
||||
"name": "_basic_crawler.py",
|
||||
"path": "/crawlers/_basic/_basic_crawler.py"
|
||||
},
|
||||
"function": "run",
|
||||
"level": { "icon": "ℹ️", "name": "INFO", "no": 20 },
|
||||
"line": 583,
|
||||
"message": "Final request statistics:",
|
||||
"module": "_basic_crawler",
|
||||
"name": "HttpCrawler",
|
||||
"process": { "id": 198383, "name": "MainProcess" },
|
||||
"thread": { "id": 135312814966592, "name": "MainThread" },
|
||||
"time": {
|
||||
"repr": "2025-03-17 17:14:45.339150+00:00",
|
||||
"timestamp": 1742231685.33915
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
id: parsel-crawler
|
||||
title: Parsel crawler
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler.py';
|
||||
|
||||
This example shows how to use <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> to crawl a website or a list of URLs. Each URL is loaded using a plain HTTP request and the response is parsed using [Parsel](https://pypi.org/project/parsel/) library which supports CSS and XPath selectors for HTML responses and JMESPath for JSON responses. We can extract data from all kinds of complex HTML structures using XPath. In this example, we will use Parsel to crawl github.com and extract page title, URL and emails found in the webpage. The default handler will scrape data from the current webpage and enqueue all the links found in the webpage for continuous scraping. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{ParselCrawlerExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
---
|
||||
id: playwright-crawler
|
||||
title: Playwright crawler
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler.py';
|
||||
|
||||
This example demonstrates how to use <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> to recursively scrape the Hacker news website using headless Chromium and Playwright.
|
||||
|
||||
The <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> manages the browser and page instances, simplifying the process of interacting with web pages. In the request handler, Playwright's API is used to extract data from each post on the page. Specifically, it retrieves the title, rank, and URL of each post. Additionally, the handler enqueues links to the next pages to ensure continuous scraping. This setup is ideal for scraping dynamic web pages where JavaScript execution is required to render the content.
|
||||
|
||||
A **pre-navigation hook** can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightCrawlerExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
---
|
||||
id: adaptive-playwright-crawler
|
||||
title: Adaptive Playwright crawler
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import AdaptivePlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/adaptive_playwright_crawler.py';
|
||||
|
||||
This example demonstrates how to use <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink>. An <ApiLink to="class/AdaptivePlaywrightCrawler">`AdaptivePlaywrightCrawler`</ApiLink> is a combination of <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> and some implementation of HTTP-based crawler such as <ApiLink to="class/ParselCrawler">`ParselCrawler`</ApiLink> or <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>.
|
||||
It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit.
|
||||
|
||||
A [pre-navigation hook](/python/docs/guides/adaptive-playwright-crawler#page-configuration-with-pre-navigation-hooks) can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation. Hooks will be executed both for the pages crawled by HTTP-bases sub crawler and playwright based sub crawler. Use `playwright_only=True` to mark hooks that should be executed only for playwright sub crawler.
|
||||
|
||||
For more detailed description please see [Adaptive Playwright crawler guide](/python/docs/guides/adaptive-playwright-crawler)
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{AdaptivePlaywrightCrawlerExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
---
|
||||
id: playwright-crawler-with-block-requests
|
||||
title: Playwright crawler with block requests
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import PlaywrightBlockRequests from '!!raw-loader!roa-loader!./code_examples/playwright_block_requests.py';
|
||||
|
||||
This example demonstrates how to optimize your <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> performance by blocking unnecessary network requests.
|
||||
|
||||
The primary use case is when you need to scrape or interact with web pages without loading non-essential resources like images, styles, or analytics scripts. This can significantly reduce bandwidth usage and improve crawling speed.
|
||||
|
||||
The <ApiLink to="class/BlockRequestsFunction">`block_requests`</ApiLink> helper provides the most efficient way to block requests as it operates directly in the browser.
|
||||
|
||||
By default, <ApiLink to="class/BlockRequestsFunction">`block_requests`</ApiLink> will block all URLs including the following patterns:
|
||||
|
||||
```python
|
||||
['.css', '.webp', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip']
|
||||
```
|
||||
|
||||
You can also replace the default patterns list with your own by providing `url_patterns`, or extend it by passing additional patterns in `extra_url_patterns`.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightBlockRequests}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
id: playwright-crawler-with-camoufox
|
||||
title: Playwright crawler with Camoufox
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_camoufox.py';
|
||||
|
||||
This example demonstrates how to integrate Camoufox into <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> using <ApiLink to="class/BrowserPool">`BrowserPool`</ApiLink> with custom <ApiLink to="class/PlaywrightBrowserPlugin">`PlaywrightBrowserPlugin`</ApiLink>.
|
||||
|
||||
Camoufox is a stealthy minimalistic build of Firefox. For details please visit its homepage https://camoufox.com/ .
|
||||
To be able to run this example you will need to install camoufox, as it is external tool, and it is not part of the crawlee. For installation please see https://pypi.org/project/camoufox/.
|
||||
|
||||
**Warning!** Camoufox is using custom build of firefox. This build can be hundreds of MB large.
|
||||
You can either pre-download this file using following command `python3 -m camoufox fetch` or camoufox will download it automatically once you try to run it, and it does not find existing binary.
|
||||
For more details please refer to: https://github.com/daijro/camoufox/tree/main/pythonlib#camoufox-python-interface
|
||||
|
||||
**Project template -** It is possible to generate project with Python code which includes Camoufox integration into crawlee through crawlee cli. Call `crawlee create` and pick `Playwright-camoufox` when asked for Crawler type.
|
||||
|
||||
The example code after PlayWrightCrawler instantiation is similar to example describing the use of Playwright Crawler. The main difference is that in this example Camoufox will be used as the browser through BrowserPool.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightCrawlerExampleWithCamoufox}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
id: playwright-crawler-with-fingerprint-generator
|
||||
title: Playwright crawler with fingerprint generator
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py';
|
||||
|
||||
This example demonstrates how to use <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> together with <ApiLink to="class/FingerprintGenerator">`FingerprintGenerator`</ApiLink> that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting.
|
||||
|
||||
You can implement your own fingerprint generator or use <ApiLink to="class/BrowserforgeFingerprintGenerator">`DefaultFingerprintGenerator`</ApiLink>. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it.
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{PlaywrightCrawlerExample}
|
||||
</RunnableCodeBlock>
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
---
|
||||
id: respect-robots-txt-file
|
||||
title: Respect robots.txt file
|
||||
---
|
||||
|
||||
import ApiLink from '@site/src/components/ApiLink';
|
||||
import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
|
||||
|
||||
import RespectRobotsTxt from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file.py';
|
||||
import OnSkippedRequest from '!!raw-loader!roa-loader!./code_examples/respect_robots_on_skipped_request.py';
|
||||
|
||||
This example demonstrates how to configure your crawler to respect the rules established by websites for crawlers as described in the [robots.txt](https://www.robotstxt.org/robotstxt.html) file.
|
||||
|
||||
To configure `Crawlee` to follow the `robots.txt` file, set the parameter `respect_robots_txt_file=True` in <ApiLink to="class/BasicCrawlerOptions">`BasicCrawlerOptions`</ApiLink>. In this case, `Crawlee` will skip any URLs forbidden in the website's robots.txt file.
|
||||
|
||||
As an example, let's look at the website `https://news.ycombinator.com/` and its corresponding [robots.txt](https://news.ycombinator.com/robots.txt) file. Since the file has a rule `Disallow: /login`, the URL `https://news.ycombinator.com/login` will be automatically skipped.
|
||||
|
||||
The code below demonstrates this behavior using the <ApiLink to="class/BeautifulSoupCrawler">`BeautifulSoupCrawler`</ApiLink>:
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{RespectRobotsTxt}
|
||||
</RunnableCodeBlock>
|
||||
|
||||
## Handle with `on_skipped_request`
|
||||
|
||||
If you want to process URLs skipped according to the `robots.txt` rules, for example for further analysis, you should use the `on_skipped_request` handler from <ApiLink to="class/BasicCrawler#on_skipped_request">`BasicCrawler`</ApiLink>.
|
||||
|
||||
Let's update the code by adding the `on_skipped_request` handler:
|
||||
|
||||
<RunnableCodeBlock className="language-python" language="python">
|
||||
{OnSkippedRequest}
|
||||
</RunnableCodeBlock>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue