firecrawl main branch snapshot (tar.gz, no history)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
FIRECRAWL_API_KEY=<your api key>
|
||||
FIRECRAWL_API_URL=https://api.firecrawl.dev
|
||||
|
||||
# TESTING PURPOSE:
|
||||
IDMUX_URL=
|
||||
@@ -0,0 +1,6 @@
|
||||
bin
|
||||
build
|
||||
include
|
||||
lib
|
||||
# local venv
|
||||
apps/python-sdk/.venv/
|
||||
@@ -0,0 +1,2 @@
|
||||
[FORMAT]
|
||||
max-line-length = 120
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 Sideguide Technologies Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,295 @@
|
||||
# Firecrawl Python SDK
|
||||
|
||||
The Firecrawl Python SDK is a library that lets you easily search, scrape, and interact with the web for AI agents — returning clean Markdown or structured data your agents can ship with. It provides a simple and intuitive interface for the Firecrawl API.
|
||||
|
||||
## Installation
|
||||
|
||||
To install the Firecrawl Python SDK, you can use pip:
|
||||
|
||||
```bash
|
||||
pip install firecrawl-py
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
|
||||
2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `Firecrawl` class.
|
||||
|
||||
Here's an example of how to use the SDK:
|
||||
|
||||
```python
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.types import ScrapeOptions
|
||||
|
||||
firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY")
|
||||
|
||||
# Scrape a website (v2):
|
||||
data = firecrawl.scrape(
|
||||
'https://firecrawl.dev',
|
||||
formats=['markdown', 'html']
|
||||
)
|
||||
print(data)
|
||||
|
||||
# Crawl a website (v2 waiter):
|
||||
crawl_status = firecrawl.crawl(
|
||||
'https://firecrawl.dev',
|
||||
limit=100,
|
||||
scrape_options=ScrapeOptions(formats=['markdown', 'html'])
|
||||
)
|
||||
print(crawl_status)
|
||||
```
|
||||
|
||||
### Scraping a URL
|
||||
|
||||
To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.
|
||||
|
||||
```python
|
||||
# Scrape a website (v2):
|
||||
scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'html'])
|
||||
print(scrape_result)
|
||||
```
|
||||
|
||||
### Video extraction
|
||||
|
||||
Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
|
||||
|
||||
```python
|
||||
doc = firecrawl.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', formats=['video'])
|
||||
print(doc.video)
|
||||
```
|
||||
|
||||
### Product extraction
|
||||
|
||||
Use the `product` format on product pages to deterministically pull structured product data (title, price, availability, variants). It is the deterministic counterpart to the LLM-based `json` format.
|
||||
|
||||
```python
|
||||
doc = firecrawl.scrape('https://firecrawl.dev', formats=['product'])
|
||||
print(doc.product)
|
||||
```
|
||||
|
||||
### Menu extraction
|
||||
|
||||
Use the `menu` format on menu pages to deterministically pull structured menu data (merchant, sections, items, prices, availability). It is the deterministic counterpart to the LLM-based `json` format.
|
||||
|
||||
```python
|
||||
doc = firecrawl.scrape('https://example.com/restaurant/menu', formats=['menu'])
|
||||
print(doc.menu)
|
||||
```
|
||||
|
||||
### Parsing uploaded files
|
||||
|
||||
Use `parse` to upload local bytes/files (`html`, `pdf`, `docx`, etc.) as multipart form data and return the parsed document.
|
||||
`parse` does not support change tracking or browser-only options (actions, wait_for, location, mobile, screenshot, branding, audio, video).
|
||||
|
||||
```python
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import ParseOptions
|
||||
|
||||
firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY")
|
||||
|
||||
doc = firecrawl.parse(
|
||||
b"<!DOCTYPE html><html><body><h1>Python Parse</h1></body></html>",
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
options=ParseOptions(formats=["markdown"]),
|
||||
)
|
||||
|
||||
print(doc.markdown)
|
||||
```
|
||||
|
||||
### Crawling a Website
|
||||
|
||||
To crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more.
|
||||
|
||||
```python
|
||||
crawl_status = firecrawl.crawl(
|
||||
'https://firecrawl.dev',
|
||||
limit=100,
|
||||
scrape_options=ScrapeOptions(formats=['markdown', 'html']),
|
||||
poll_interval=30
|
||||
)
|
||||
print(crawl_status)
|
||||
```
|
||||
|
||||
### Asynchronous Crawling
|
||||
|
||||
<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>
|
||||
|
||||
To enqueue a crawl asynchronously, use `start_crawl`. It returns the crawl `ID` which you can use to check the status of the crawl job.
|
||||
|
||||
```python
|
||||
crawl_job = firecrawl.start_crawl(
|
||||
'https://firecrawl.dev',
|
||||
limit=100,
|
||||
scrape_options=ScrapeOptions(formats=['markdown', 'html']),
|
||||
)
|
||||
print(crawl_job)
|
||||
```
|
||||
|
||||
### Checking Crawl Status
|
||||
|
||||
To check the status of a crawl job, use the `get_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
|
||||
|
||||
```python
|
||||
crawl_status = firecrawl.get_crawl_status("<crawl_id>")
|
||||
print(crawl_status)
|
||||
```
|
||||
|
||||
### Manual Pagination (v2)
|
||||
|
||||
Crawl and batch scrape status responses may include a `next` URL when more data is available. The SDK auto-paginates by default; to page manually, disable auto-pagination and pass the opaque `next` URL back to the SDK.
|
||||
|
||||
```python
|
||||
from firecrawl.v2.types import PaginationConfig
|
||||
|
||||
# Crawl: fetch one page at a time
|
||||
crawl_job = firecrawl.start_crawl("https://firecrawl.dev", limit=100)
|
||||
status = firecrawl.get_crawl_status(
|
||||
crawl_job.id,
|
||||
pagination_config=PaginationConfig(auto_paginate=False),
|
||||
)
|
||||
if status.next:
|
||||
page2 = firecrawl.get_crawl_status_page(status.next)
|
||||
|
||||
# Batch scrape: fetch one page at a time
|
||||
batch_job = firecrawl.start_batch_scrape(["https://firecrawl.dev"])
|
||||
status = firecrawl.get_batch_scrape_status(
|
||||
batch_job.id,
|
||||
pagination_config=PaginationConfig(auto_paginate=False),
|
||||
)
|
||||
if status.next:
|
||||
page2 = firecrawl.get_batch_scrape_status_page(status.next)
|
||||
```
|
||||
|
||||
### Cancelling a Crawl
|
||||
|
||||
To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
|
||||
|
||||
```python
|
||||
cancel_crawl = firecrawl.cancel_crawl(id)
|
||||
print(cancel_crawl)
|
||||
```
|
||||
|
||||
### Map a Website
|
||||
|
||||
Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to use the sitemap or include subdomains.
|
||||
|
||||
```python
|
||||
# Map a website (v2):
|
||||
map_result = firecrawl.map('https://firecrawl.dev')
|
||||
print(map_result)
|
||||
```
|
||||
|
||||
### Scrape-bound interactive browsing (v2)
|
||||
|
||||
Use a scrape job ID to keep interacting with the replayed browser context:
|
||||
|
||||
```python
|
||||
doc = firecrawl.scrape(
|
||||
"https://example.com",
|
||||
actions=[{"type": "click", "selector": "a[href='/pricing']"}],
|
||||
)
|
||||
|
||||
scrape_job_id = doc.metadata_typed.scrape_id
|
||||
if not scrape_job_id:
|
||||
raise RuntimeError("Missing scrape job id")
|
||||
|
||||
run = firecrawl.interact(
|
||||
scrape_job_id,
|
||||
code="print(await page.url())",
|
||||
language="python",
|
||||
timeout=60,
|
||||
)
|
||||
print(run.stdout)
|
||||
|
||||
firecrawl.stop_interaction(scrape_job_id)
|
||||
```
|
||||
|
||||
{/* ### Extracting Structured Data from Websites
|
||||
|
||||
To extract structured data from websites, use the `extract` method. It takes the URLs to extract data from, a prompt, and a schema as arguments. The schema is a Pydantic model that defines the structure of the extracted data.
|
||||
|
||||
<ExtractPythonShort /> */}
|
||||
|
||||
### Crawling a Website with WebSockets
|
||||
|
||||
To crawl a website with WebSockets, use the `crawl_url_and_watch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
|
||||
|
||||
```python
|
||||
# inside an async function...
|
||||
nest_asyncio.apply()
|
||||
|
||||
# Define event handlers
|
||||
def on_document(detail):
|
||||
print("DOC", detail)
|
||||
|
||||
def on_error(detail):
|
||||
print("ERR", detail['error'])
|
||||
|
||||
def on_done(detail):
|
||||
print("DONE", detail['status'])
|
||||
|
||||
# Function to start the crawl and watch process
|
||||
async def start_crawl_and_watch():
|
||||
# Initiate the crawl job and get the watcher
|
||||
watcher = app.crawl_url_and_watch('firecrawl.dev', exclude_paths=['blog/*'], limit=5)
|
||||
|
||||
# Add event listeners
|
||||
watcher.add_event_listener("document", on_document)
|
||||
watcher.add_event_listener("error", on_error)
|
||||
watcher.add_event_listener("done", on_done)
|
||||
|
||||
# Start the watcher
|
||||
await watcher.connect()
|
||||
|
||||
# Run the event loop
|
||||
await start_crawl_and_watch()
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message.
|
||||
|
||||
## Async Class
|
||||
|
||||
For async operations, you can use the `AsyncFirecrawl` class. Its methods mirror the `Firecrawl` class, but you `await` them.
|
||||
|
||||
```python
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
firecrawl = AsyncFirecrawl(api_key="YOUR_API_KEY")
|
||||
|
||||
# Async Scrape (v2)
|
||||
async def example_scrape():
|
||||
scrape_result = await firecrawl.scrape(url="https://example.com")
|
||||
print(scrape_result)
|
||||
|
||||
# Async Parse (v2)
|
||||
async def example_parse():
|
||||
parse_result = await firecrawl.parse(
|
||||
b"<!DOCTYPE html><html><body><h1>Async Parse</h1></body></html>",
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
print(parse_result)
|
||||
|
||||
# Async Crawl (v2)
|
||||
async def example_crawl():
|
||||
crawl_result = await firecrawl.crawl(url="https://example.com")
|
||||
print(crawl_result)
|
||||
```
|
||||
|
||||
## v1 compatibility
|
||||
|
||||
For legacy code paths, v1 remains available under `firecrawl.v1` with the original method names.
|
||||
|
||||
```python
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
firecrawl = Firecrawl(api_key="YOUR_API_KEY")
|
||||
|
||||
# v1 methods (feature‑frozen)
|
||||
doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])
|
||||
crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)
|
||||
map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal examples for Firecrawl v2.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
def main():
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError("FIRECRAWL_API_KEY is not set")
|
||||
|
||||
api_url = os.getenv("FIRECRAWL_API_URL")
|
||||
if not api_url:
|
||||
raise ValueError("FIRECRAWL_API_URL is not set")
|
||||
|
||||
firecrawl = Firecrawl(api_key=api_key, api_url=api_url)
|
||||
|
||||
# Scrape
|
||||
doc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])
|
||||
print("scrape:", doc.markdown)
|
||||
# doc.metadata_dict is a dict, doc.metadata_typed is a DocumentMetadata object
|
||||
print(doc.metadata_dict.get("source_url"))
|
||||
print('metadata_dict.get("title"):', doc.metadata_dict.get("title"))
|
||||
print("metadata_typed.title:", doc.metadata_typed.title)
|
||||
print("metadata.title", doc.metadata.title if doc.metadata else None)
|
||||
|
||||
|
||||
# Crawl (waits until terminal state)
|
||||
crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)
|
||||
print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)
|
||||
|
||||
# Batch scrape
|
||||
batch = firecrawl.batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], poll_interval=1, wait_timeout=120)
|
||||
print("batch:", batch.status, batch.completed, "/", batch.total)
|
||||
|
||||
# Search
|
||||
search_response = firecrawl.search(query="What is the capital of France?", limit=5)
|
||||
print("search web results:", len(getattr(search_response, "web", []) or []))
|
||||
|
||||
# Map
|
||||
map_response = firecrawl.map("https://firecrawl.dev")
|
||||
print("map links:", len(getattr(map_response, "links", []) or []))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal async examples for Firecrawl v2.
|
||||
|
||||
Env variables required:
|
||||
- FIRECRAWL_API_KEY
|
||||
- FIRECRAWL_API_URL
|
||||
"""
|
||||
|
||||
import os
|
||||
import asyncio
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
load_dotenv()
|
||||
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError("FIRECRAWL_API_KEY is not set")
|
||||
|
||||
api_url = os.getenv("FIRECRAWL_API_URL")
|
||||
if not api_url:
|
||||
raise ValueError("FIRECRAWL_API_URL is not set")
|
||||
|
||||
client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
|
||||
|
||||
# Scrape (minimal)
|
||||
doc = await client.v2.scrape("https://docs.firecrawl.dev")
|
||||
print("scrape markdown:", doc.markdown)
|
||||
|
||||
# Crawl (waiter)
|
||||
crawl_job = await client.v2.crawl(url="https://docs.firecrawl.dev", limit=2, poll_interval=1, timeout=120)
|
||||
print("crawl:", crawl_job.status, crawl_job.completed, "/", crawl_job.total)
|
||||
|
||||
# Batch scrape (minimal waiter)
|
||||
batch = await client.v2.batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], poll_interval=1, timeout=180)
|
||||
print("batch:", batch.status, batch.completed, "/", batch.total)
|
||||
|
||||
# Search (minimal)
|
||||
search = await client.v2.search("What is the capital of France?", limit=5)
|
||||
web_count = len(getattr(search, "web", []) or []) if hasattr(search, "web") else len(getattr(search, "results", []) or [])
|
||||
print("search web results:", web_count)
|
||||
|
||||
# Map (minimal)
|
||||
mp = await client.v2.map("https://firecrawl.dev")
|
||||
links = getattr(mp, "links", []) or []
|
||||
print("map links:", len(links))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
Example demonstrating pagination functionality in Firecrawl v2 SDK.
|
||||
|
||||
This example shows how to use the new pagination features for both crawl and batch scrape operations.
|
||||
"""
|
||||
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import PaginationConfig
|
||||
|
||||
# Initialize the client
|
||||
firecrawl = Firecrawl(api_key="your-api-key")
|
||||
|
||||
# Example 1: Simple crawl - automatically waits for completion and returns all results
|
||||
# Use this when you want all the data and don't need to control pagination
|
||||
print("=== Example 1: Simple crawl (default) ===")
|
||||
crawl_result = firecrawl.crawl("https://example.com", limit=100)
|
||||
print(f"Total documents fetched: {len(crawl_result.data)}")
|
||||
print(f"Next URL: {crawl_result.next}") # Should be None since auto-pagination is enabled
|
||||
|
||||
# Example 2: Manual crawl with pagination control
|
||||
# Use this when you need to control how many pages to fetch or want to process results incrementally.
|
||||
# The next URL is opaque; pass it back to the SDK to fetch the next page.
|
||||
print("\n=== Example 2: Manual crawl with pagination control ===")
|
||||
crawl_job = firecrawl.start_crawl("https://example.com", limit=100)
|
||||
|
||||
# Get just the first page of results (useful for large crawls where you want to process incrementally)
|
||||
pagination_config = PaginationConfig(auto_paginate=False)
|
||||
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
|
||||
print(f"Documents from first page: {len(status.data)}")
|
||||
print(f"Next URL: {status.next}") # Will show the next page URL
|
||||
if status.next:
|
||||
next_page = firecrawl.get_crawl_status_page(status.next)
|
||||
print(f"Documents from next page: {len(next_page.data)}")
|
||||
|
||||
# Example 3: Limited pagination - fetch only 3 pages
|
||||
# Useful for controlling memory usage or processing time
|
||||
print("\n=== Example 3: Limited pagination ===")
|
||||
pagination_config = PaginationConfig(max_pages=3)
|
||||
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
|
||||
print(f"Documents from first 3 pages: {len(status.data)}")
|
||||
|
||||
# Example 4: Result-limited pagination - stop after 50 results
|
||||
# Useful when you only need a specific number of results
|
||||
print("\n=== Example 4: Result-limited pagination ===")
|
||||
pagination_config = PaginationConfig(max_results=50)
|
||||
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
|
||||
print(f"Documents (max 50): {len(status.data)}")
|
||||
|
||||
# Example 5: Time-limited pagination - stop after 30 seconds
|
||||
# Useful for controlling processing time in time-sensitive applications
|
||||
print("\n=== Example 5: Time-limited pagination ===")
|
||||
pagination_config = PaginationConfig(max_wait_time=30)
|
||||
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
|
||||
print(f"Documents fetched within 30 seconds: {len(status.data)}")
|
||||
|
||||
# Example 6: Combined pagination limits
|
||||
# Useful for fine-grained control over resource usage
|
||||
print("\n=== Example 6: Combined limits ===")
|
||||
pagination_config = PaginationConfig(
|
||||
max_pages=5, # Only fetch 5 pages maximum
|
||||
max_results=100, # Only fetch 100 results maximum
|
||||
max_wait_time=60 # Spend at most 60 seconds fetching additional pages
|
||||
)
|
||||
status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)
|
||||
print(f"Documents with combined limits: {len(status.data)}")
|
||||
|
||||
# Example 7: Simple batch scrape - automatically waits for completion and returns all results
|
||||
# Use this when you want all the data from multiple URLs and don't need to control pagination
|
||||
print("\n=== Example 7: Simple batch scrape (default) ===")
|
||||
urls = ["https://example1.com", "https://example2.com", "https://example3.com"]
|
||||
batch_result = firecrawl.batch_scrape(urls)
|
||||
print(f"Batch scrape documents: {len(batch_result.data)}")
|
||||
|
||||
# Example 8: Manual batch scrape with pagination control
|
||||
# Use this when you need to control how many pages to fetch or want to process results incrementally.
|
||||
# The next URL is opaque; pass it back to the SDK to fetch the next page.
|
||||
print("\n=== Example 8: Manual batch scrape with pagination control ===")
|
||||
batch_job = firecrawl.start_batch_scrape(urls)
|
||||
status = firecrawl.get_batch_scrape_status(batch_job.id, pagination_config=PaginationConfig(auto_paginate=False))
|
||||
print(f"Batch scrape documents: {len(status.data)}")
|
||||
print(f"Next URL: {status.next}")
|
||||
if status.next:
|
||||
next_page = firecrawl.get_batch_scrape_status_page(status.next)
|
||||
print(f"Batch scrape next page documents: {len(next_page.data)}")
|
||||
|
||||
# Example 9: Async usage
|
||||
print("\n=== Example 9: Async pagination ===")
|
||||
import asyncio
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
async def async_example():
|
||||
async_client = AsyncFirecrawl(api_key="your-api-key")
|
||||
|
||||
# Simple async crawl - automatically waits for completion
|
||||
crawl_result = await async_client.crawl("https://example.com", limit=50)
|
||||
print(f"Async crawl documents: {len(crawl_result.data)}")
|
||||
|
||||
# Manual async crawl with pagination
|
||||
crawl_job = await async_client.start_crawl("https://example.com", limit=50)
|
||||
pagination_config = PaginationConfig(max_pages=2)
|
||||
status = await async_client.get_crawl_status(
|
||||
crawl_job.id,
|
||||
pagination_config=pagination_config
|
||||
)
|
||||
print(f"Async crawl with pagination: {len(status.data)}")
|
||||
if status.next:
|
||||
next_page = await async_client.get_crawl_status_page(status.next)
|
||||
print(f"Async crawl next page documents: {len(next_page.data)}")
|
||||
|
||||
# Run async example
|
||||
# asyncio.run(async_example())
|
||||
@@ -0,0 +1,286 @@
|
||||
from firecrawl import V1JsonConfig, FirecrawlApp, AsyncFirecrawlApp, V1ScrapeOptions, V1JsonConfig
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List
|
||||
import time
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# -- Async example imports
|
||||
import nest_asyncio
|
||||
import asyncio
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# Get API key from environment
|
||||
api_key = os.getenv("TEST_API_KEY")
|
||||
api_url = os.getenv("TEST_API_URL", "https://api.firecrawl.dev")
|
||||
|
||||
app = FirecrawlApp(api_key=api_key, api_url=api_url)
|
||||
|
||||
# Scrape a website:
|
||||
scrape_result = app.v1.scrape_url('example.com', formats=["markdown", "html"])
|
||||
print(scrape_result.markdown)
|
||||
|
||||
# # # Test batch scrape
|
||||
urls = ['https://example.com', 'https://docs.firecrawl.dev']
|
||||
# # Synchronous batch scrape
|
||||
batch_result = app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])
|
||||
print("Synchronous Batch Scrape Result:")
|
||||
print(batch_result.data[0].markdown)
|
||||
|
||||
# # # Asynchronous batch scrape
|
||||
async_batch_result = app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])
|
||||
print("\nAsynchronous Batch Scrape Result:")
|
||||
print(async_batch_result)
|
||||
|
||||
# # Crawl a website:
|
||||
crawl_result = app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
|
||||
print(crawl_result.data[0].markdown)
|
||||
|
||||
# # Asynchronous Crawl a website:
|
||||
async_result = app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
|
||||
print(async_result)
|
||||
|
||||
crawl_status = app.v1.check_crawl_status(async_result.id)
|
||||
print(crawl_status)
|
||||
|
||||
attempts = 15
|
||||
while attempts > 0 and crawl_status.status != 'completed':
|
||||
print(crawl_status)
|
||||
crawl_status = app.v1.check_crawl_status(async_result.id)
|
||||
attempts -= 1
|
||||
time.sleep(1)
|
||||
|
||||
crawl_status = app.v1.check_crawl_status(async_result.id)
|
||||
print(crawl_status)
|
||||
|
||||
# JSON format:
|
||||
# Define schema to extract contents into using json schema
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"top": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"points": {"type": "number"},
|
||||
"by": {"type": "string"},
|
||||
"commentsURL": {"type": "string"}
|
||||
},
|
||||
"required": ["title", "points", "by", "commentsURL"]
|
||||
},
|
||||
"description": "Top 5 stories on Hacker News"
|
||||
}
|
||||
},
|
||||
"required": ["top"]
|
||||
}
|
||||
|
||||
extract_config = V1JsonConfig(schema=json_schema)
|
||||
llm_extraction_result = app.v1.scrape_url('https://news.ycombinator.com', formats=["json"], json_options=extract_config)
|
||||
|
||||
print(llm_extraction_result.json)
|
||||
|
||||
# Map a website:
|
||||
map_result = app.v1.map_url('https://firecrawl.dev', search="blog")
|
||||
print(map_result)
|
||||
|
||||
# Extract URLs:
|
||||
class ExtractSchema(BaseModel):
|
||||
title: str
|
||||
description: str
|
||||
links: List[str]
|
||||
|
||||
# Define the schema using Pydantic
|
||||
extract_schema = ExtractSchema.schema()
|
||||
|
||||
# Perform the extraction
|
||||
extract_result = app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
|
||||
print(extract_result)
|
||||
|
||||
|
||||
# Deep research example
|
||||
research_result = app.v1.deep_research(
|
||||
"What are the latest developments in large language models?",
|
||||
max_urls=4
|
||||
)
|
||||
print("Research Results:", research_result)
|
||||
|
||||
# Generate LLMs.txt example
|
||||
llms_result = app.v1.generate_llms_text(
|
||||
"https://firecrawl.dev")
|
||||
print("LLMs.txt Results:", llms_result)
|
||||
|
||||
|
||||
# Crawl a website with WebSockets:
|
||||
# inside an async function...
|
||||
import nest_asyncio
|
||||
nest_asyncio.apply()
|
||||
|
||||
# Define event handlers
|
||||
def on_document(detail):
|
||||
print("DOC", detail)
|
||||
|
||||
def on_error(detail):
|
||||
print("ERR", detail['error'])
|
||||
|
||||
def on_done(detail):
|
||||
print("DONE", detail['status'])
|
||||
|
||||
# Function to start the crawl and watch process
|
||||
async def start_crawl_and_watch():
|
||||
# Initiate the crawl job and get the watcher
|
||||
watcher = app.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
|
||||
|
||||
# Add event listeners
|
||||
watcher.add_event_listener("document", on_document)
|
||||
watcher.add_event_listener("error", on_error)
|
||||
watcher.add_event_listener("done", on_done)
|
||||
|
||||
# Start the watcher
|
||||
await watcher.connect()
|
||||
|
||||
|
||||
class ExtractSchema(BaseModel):
|
||||
company_mission: str
|
||||
supports_sso: bool
|
||||
is_open_source: bool
|
||||
is_in_yc: bool
|
||||
|
||||
extract_config = V1JsonConfig(schema=ExtractSchema.model_json_schema())
|
||||
data = app.scrape_url('https://docs.firecrawl.dev/', formats=['json'], json_options=extract_config)
|
||||
print(data.json)
|
||||
|
||||
# --- Async
|
||||
|
||||
async_app = AsyncFirecrawlApp(api_url="https://api.firecrawl.dev")
|
||||
|
||||
async def example_scrape():
|
||||
# Scrape a website:
|
||||
scrape_result = await async_app.v1.scrape_url('example.com', formats=["markdown", "html"])
|
||||
print(scrape_result.markdown)
|
||||
|
||||
async def example_batch_scrape():
|
||||
# Batch scrape
|
||||
urls = ['https://example.com', 'https://docs.firecrawl.dev']
|
||||
|
||||
# Synchronous batch scrape
|
||||
batch_result = await async_app.v1.batch_scrape_urls(urls, formats=["markdown", "html"])
|
||||
print("Synchronous Batch Scrape Result:")
|
||||
print(batch_result.data[0].markdown)
|
||||
|
||||
# Asynchronous batch scrape
|
||||
async_batch_result = await async_app.v1.async_batch_scrape_urls(urls, formats=["markdown", "html"])
|
||||
print("\nAsynchronous Batch Scrape Result:")
|
||||
print(async_batch_result)
|
||||
|
||||
async def example_crawl():
|
||||
# Crawl a website:
|
||||
crawl_result = await async_app.v1.crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
|
||||
print(crawl_result.data[0].markdown)
|
||||
|
||||
# Asynchronous Crawl a website:
|
||||
async_result = await async_app.v1.async_crawl_url('firecrawl.dev', exclude_paths=['blog/*'])
|
||||
print(async_result)
|
||||
|
||||
crawl_status = await async_app.v1.check_crawl_status(async_result.id)
|
||||
print(crawl_status)
|
||||
|
||||
attempts = 15
|
||||
while attempts > 0 and crawl_status.status != 'completed':
|
||||
print(crawl_status)
|
||||
crawl_status = await async_app.v1.check_crawl_status(async_result.id)
|
||||
attempts -= 1
|
||||
await asyncio.sleep(1) # Use async sleep instead of time.sleep
|
||||
|
||||
crawl_status = await async_app.v1.check_crawl_status(async_result.id)
|
||||
print(crawl_status)
|
||||
|
||||
async def example_llm_extraction():
|
||||
# Define schema to extract contents into using pydantic
|
||||
class ArticleSchema(BaseModel):
|
||||
title: str
|
||||
points: int
|
||||
by: str
|
||||
commentsURL: str
|
||||
|
||||
class TopArticlesSchema(BaseModel):
|
||||
top: List[ArticleSchema] = Field(..., description="Top 5 stories")
|
||||
|
||||
extract_config = V1JsonConfig(schema=TopArticlesSchema.model_json_schema())
|
||||
|
||||
llm_extraction_result = await async_app.v1.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)
|
||||
|
||||
print(llm_extraction_result.extract)
|
||||
|
||||
async def example_map_and_extract():
|
||||
# Map a website:
|
||||
map_result = await async_app.v1.map_url('https://firecrawl.dev', search="blog")
|
||||
print(map_result)
|
||||
|
||||
# Extract URLs:
|
||||
class ExtractSchema(BaseModel):
|
||||
title: str
|
||||
description: str
|
||||
links: List[str]
|
||||
|
||||
# Define the schema using Pydantic
|
||||
extract_schema = ExtractSchema.schema()
|
||||
|
||||
# Perform the extraction
|
||||
extract_result = await async_app.v1.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)
|
||||
print(extract_result)
|
||||
|
||||
async def example_deep_research():
|
||||
# Deep research example
|
||||
research_result = await async_app.v1.deep_research(
|
||||
"What are the latest developments in large language models?",
|
||||
max_urls=4
|
||||
)
|
||||
print("Research Results:", research_result)
|
||||
|
||||
async def example_generate_llms_text():
|
||||
# Generate LLMs.txt example
|
||||
llms_result = await async_app.v1.generate_llms_text(
|
||||
"https://firecrawl.dev")
|
||||
print("LLMs.txt Results:", llms_result)
|
||||
|
||||
# Define event handlers for websocket
|
||||
def on_document(detail):
|
||||
print("DOC", detail)
|
||||
|
||||
def on_error(detail):
|
||||
print("ERR", detail['error'])
|
||||
|
||||
def on_done(detail):
|
||||
print("DONE", detail['status'])
|
||||
|
||||
async def example_websocket_crawl():
|
||||
# Initiate the crawl job and get the watcher
|
||||
watcher = await async_app.v1.crawl_url_and_watch('firecrawl.dev', { 'excludePaths': ['blog/*'], 'limit': 5 })
|
||||
|
||||
# Add event listeners
|
||||
watcher.add_event_listener("document", on_document)
|
||||
watcher.add_event_listener("error", on_error)
|
||||
watcher.add_event_listener("done", on_done)
|
||||
|
||||
# Start the watcher
|
||||
await watcher.connect()
|
||||
|
||||
async def main():
|
||||
nest_asyncio.apply()
|
||||
|
||||
await example_scrape()
|
||||
await example_batch_scrape()
|
||||
await example_crawl()
|
||||
await example_llm_extraction()
|
||||
await example_map_and_extract()
|
||||
await example_websocket_crawl()
|
||||
await example_deep_research()
|
||||
await example_generate_llms_text()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
from firecrawl.client import Firecrawl
|
||||
|
||||
# Initialize Firecrawl client
|
||||
firecrawl = Firecrawl(api_key="YOUR_API_KEY")
|
||||
|
||||
# =============================================================================
|
||||
# SCRAPE EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Basic scraping - Get markdown content
|
||||
print("=== Basic Scrape (Markdown) ===")
|
||||
# doc = firecrawl.scrape("https://firecrawl.dev", formats=["markdown"])
|
||||
# print(doc.markdown)
|
||||
|
||||
# Scraping with location settings
|
||||
print("\n=== Scrape with Location Settings ===")
|
||||
# doc = firecrawl.scrape('https://docs.firecrawl.dev',
|
||||
# formats=['markdown'],
|
||||
# location={
|
||||
# 'country': 'US',
|
||||
# 'languages': ['en']
|
||||
# }
|
||||
# )
|
||||
# print(doc)
|
||||
|
||||
# Scraping with JSON extraction using Pydantic schema
|
||||
print("\n=== Scrape with JSON Schema (Pydantic) ===")
|
||||
# from pydantic import BaseModel
|
||||
#
|
||||
# class JsonSchema(BaseModel):
|
||||
# company_mission: str
|
||||
# supports_sso: bool
|
||||
# is_open_source: bool
|
||||
# is_in_yc: bool
|
||||
#
|
||||
# result = firecrawl.scrape(
|
||||
# 'https://firecrawl.dev',
|
||||
# formats=[{
|
||||
# "type": "json",
|
||||
# "schema": JsonSchema
|
||||
# }],
|
||||
# only_main_content=False,
|
||||
# max_age=120000000
|
||||
# )
|
||||
# print(result)
|
||||
|
||||
# Scraping with JSON extraction using prompt
|
||||
print("\n=== Scrape with JSON Prompt ===")
|
||||
# result = firecrawl.scrape(
|
||||
# 'https://firecrawl.dev',
|
||||
# formats=[{
|
||||
# "type": "json",
|
||||
# "prompt": "Extract the company mission from the page."
|
||||
# }],
|
||||
# only_main_content=False,
|
||||
# timeout=120000
|
||||
# )
|
||||
# print(result.json["companyMission"])
|
||||
|
||||
# Advanced scraping with multiple formats and options
|
||||
print("\n=== Advanced Scrape (Multiple Formats + Options) ===")
|
||||
# response = firecrawl.scrape('https://docs.firecrawl.dev',
|
||||
# formats=[
|
||||
# 'markdown',
|
||||
# { 'type': 'json', 'schema': { 'type': 'object', 'properties': { 'title': { 'type': 'string' } } } }
|
||||
# ],
|
||||
# proxy='auto',
|
||||
# max_age=600000,
|
||||
# only_main_content=True
|
||||
# )
|
||||
# print(response)
|
||||
|
||||
# =============================================================================
|
||||
# CRAWL EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Basic crawling
|
||||
print("\n=== Basic Crawl ===")
|
||||
# docs = firecrawl.crawl(url="https://docs.firecrawl.dev", limit=10)
|
||||
# print(docs)
|
||||
|
||||
# =============================================================================
|
||||
# BATCH SCRAPE EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Batch scraping multiple URLs
|
||||
print("\n=== Batch Scrape ===")
|
||||
# from firecrawl import Firecrawl
|
||||
#
|
||||
# firecrawl = Firecrawl(api_key="fc-YOUR-API-KEY")
|
||||
#
|
||||
# job = firecrawl.batch_scrape([
|
||||
# "https://firecrawl.dev",
|
||||
# "https://docs.firecrawl.dev",
|
||||
# ], formats=["markdown"], poll_interval=2, wait_timeout=120)
|
||||
# print(job)
|
||||
|
||||
# =============================================================================
|
||||
# SEARCH EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Search functionality
|
||||
print("\n=== Search ===")
|
||||
# search = firecrawl.search(query="firecrawl", sources=[{"type": "web"}], limit=5)
|
||||
# print(search.web[0].title)
|
||||
|
||||
# =============================================================================
|
||||
# MAP EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Website mapping
|
||||
print("\n=== Map Website ===")
|
||||
# res = firecrawl.map(url="https://firecrawl.dev", limit=50, sitemap="include", search="price")
|
||||
# print(res.links[0].url)
|
||||
# print(res.links[1].url)
|
||||
# print(res.links[2].url)
|
||||
# print(res.links[3].url)
|
||||
|
||||
# =============================================================================
|
||||
# EXTRACT EXAMPLES
|
||||
# =============================================================================
|
||||
|
||||
# Data extraction with schema
|
||||
print("\n=== Extract with Schema ===")
|
||||
# schema = {
|
||||
# "type": "object",
|
||||
# "properties": {"title": {"type": "string"}},
|
||||
# "required": ["title"],
|
||||
# }
|
||||
#
|
||||
# res = firecrawl.extract(
|
||||
# urls=["https://docs.firecrawl.dev"],
|
||||
# prompt="Extract the page title",
|
||||
# schema=schema,
|
||||
# )
|
||||
# print(res.data["title"])
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal websocket watcher examples (sync and async).
|
||||
|
||||
Env:
|
||||
FIRECRAWL_API_KEY
|
||||
FIRECRAWL_API_URL
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import asyncio
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from firecrawl import Firecrawl, Watcher, AsyncWatcher
|
||||
|
||||
|
||||
def example_watcher() -> None:
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY")
|
||||
api_url = os.getenv("FIRECRAWL_API_URL")
|
||||
if not api_key or not api_url:
|
||||
raise ValueError("FIRECRAWL_API_KEY and FIRECRAWL_API_URL must be set")
|
||||
|
||||
client = Firecrawl(api_key=api_key, api_url=api_url)
|
||||
|
||||
# Start a small crawl job
|
||||
job = client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
|
||||
events = {"document": 0, "done": 0}
|
||||
statuses = []
|
||||
|
||||
w: Watcher = client.watcher(job.id, kind="crawl", poll_interval=1, timeout=180)
|
||||
w.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
|
||||
w.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
|
||||
w.add_listener(lambda s: statuses.append(s.status))
|
||||
w.start()
|
||||
|
||||
# Wait until terminal
|
||||
deadline = time.time() + 180
|
||||
while time.time() < deadline:
|
||||
if statuses and statuses[-1] in ("completed", "failed"):
|
||||
break
|
||||
time.sleep(1)
|
||||
w.stop()
|
||||
|
||||
print("sync watcher:", {"last_status": statuses[-1] if statuses else None, **events})
|
||||
|
||||
|
||||
async def example_async_watcher() -> None:
|
||||
"""
|
||||
Example of using the async watcher.
|
||||
"""
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY")
|
||||
api_url = os.getenv("FIRECRAWL_API_URL")
|
||||
if not api_key or not api_url:
|
||||
raise ValueError("FIRECRAWL_API_KEY and FIRECRAWL_API_URL must be set")
|
||||
|
||||
client = Firecrawl(api_key=api_key, api_url=api_url)
|
||||
|
||||
# Start a small crawl job
|
||||
job = client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
|
||||
async for snapshot in AsyncWatcher(client, job.id, kind="crawl"):
|
||||
print("async watcher:", snapshot.status, f"docs={len(snapshot.data)}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
load_dotenv()
|
||||
example_watcher()
|
||||
asyncio.run(example_async_watcher())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
- [ ] improve error handling for 500s
|
||||
|
||||
|
||||
============================================================================ FAILURES ============================================================================
|
||||
_____________________________________________________ TestCrawlE2E.test_get_active_crawls_with_running_crawl _____________________________________________________
|
||||
|
||||
self = <test_crawl.TestCrawlE2E object at 0x1039a5810>
|
||||
|
||||
def test_get_active_crawls_with_running_crawl(self):
|
||||
"""Test getting active crawls when there's a running crawl."""
|
||||
# Start a crawl
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5)
|
||||
assert start_job.id is not None
|
||||
|
||||
# Get active crawls
|
||||
> active_crawls_response = self.client.active_crawls()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
firecrawl/__tests__/e2e/v2/test_crawl.py:149:
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
firecrawl/v2/client.py:386: in active_crawls
|
||||
return self.get_active_crawls()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
firecrawl/v2/client.py:377: in get_active_crawls
|
||||
return crawl_module.get_active_crawls(self.http_client)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
firecrawl/v2/methods/crawl.py:446: in get_active_crawls
|
||||
handle_response_error(response, "get active crawls")
|
||||
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
|
||||
|
||||
response = <Response [500]>, action = 'get active crawls'
|
||||
|
||||
def handle_response_error(response: requests.Response, action: str) -> None:
|
||||
"""
|
||||
Handle API response errors and raise appropriate exceptions.
|
||||
|
||||
Args:
|
||||
response: The HTTP response object
|
||||
action: Description of the action being performed
|
||||
|
||||
Raises:
|
||||
FirecrawlError: Appropriate error based on status code
|
||||
"""
|
||||
try:
|
||||
response_json = response.json()
|
||||
error_message = response_json.get('error', 'No error message provided.')
|
||||
error_details = response_json.get('details', 'No additional error details provided.')
|
||||
except:
|
||||
# If we can't parse JSON, provide a helpful error message
|
||||
try:
|
||||
response_text = response.text[:500] # Limit to first 500 chars
|
||||
if response_text.strip():
|
||||
error_message = f"Server returned non-JSON response: {response_text}"
|
||||
error_details = f"Full response status: {response.status_code}"
|
||||
else:
|
||||
error_message = f"Server returned empty response with status {response.status_code}"
|
||||
error_details = "No additional details available"
|
||||
except:
|
||||
error_message = f"Server returned unreadable response with status {response.status_code}"
|
||||
error_details = "No additional details available"
|
||||
|
||||
# Create appropriate error message
|
||||
if response.status_code == 400:
|
||||
message = f"Bad Request: Failed to {action}. {error_message} - {error_details}"
|
||||
raise BadRequestError(message, response.status_code, response)
|
||||
elif response.status_code == 401:
|
||||
message = f"Unauthorized: Failed to {action}. {error_message} - {error_details}"
|
||||
raise UnauthorizedError(message, response.status_code, response)
|
||||
elif response.status_code == 402:
|
||||
message = f"Payment Required: Failed to {action}. {error_message} - {error_details}"
|
||||
raise PaymentRequiredError(message, response.status_code, response)
|
||||
elif response.status_code == 403:
|
||||
message = f"Website Not Supported: Failed to {action}. {error_message} - {error_details}"
|
||||
raise WebsiteNotSupportedError(message, response.status_code, response)
|
||||
elif response.status_code == 408:
|
||||
message = f"Request Timeout: Failed to {action} as the request timed out. {error_message} - {error_details}"
|
||||
raise RequestTimeoutError(message, response.status_code, response)
|
||||
elif response.status_code == 429:
|
||||
message = f"Rate Limit Exceeded: Failed to {action}. {error_message} - {error_details}"
|
||||
raise RateLimitError(message, response.status_code, response)
|
||||
elif response.status_code == 500:
|
||||
message = f"Internal Server Error: Failed to {action}. {error_message} - {error_details}"
|
||||
> raise InternalServerError(message, response.status_code, response)
|
||||
E firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@firecrawl.com for help. Your exception ID is c775af539e3a44f286664e55c488638e - No additional error details provided.
|
||||
|
||||
firecrawl/v2/utils/error_handler.py:104: InternalServerError
|
||||
==================================================================== short test summary info =====================================================================
|
||||
FAILED firecrawl/__tests__/e2e/v2/test_crawl.py::TestCrawlE2E::test_get_active_crawls_with_running_crawl - firecrawl.v2.utils.error_handler.InternalServerError: Internal Server Error: Failed to get active crawls. An unexpected error occurred. Please contact help@f...
|
||||
======================================================================= 1 failed in 1.17s ========================================================================
|
||||
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Firecrawl Python SDK
|
||||
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from .client import Firecrawl, AsyncFirecrawl, FirecrawlApp, AsyncFirecrawlApp
|
||||
from .v2.watcher import Watcher
|
||||
from .v2.watcher_async import AsyncWatcher
|
||||
from .v2.types import AuditMetadata
|
||||
from .v1 import (
|
||||
V1FirecrawlApp,
|
||||
AsyncV1FirecrawlApp,
|
||||
V1JsonConfig,
|
||||
V1ScrapeOptions,
|
||||
V1ChangeTrackingOptions,
|
||||
)
|
||||
|
||||
__version__ = "4.34.0"
|
||||
|
||||
# Define the logger for the Firecrawl project
|
||||
logger: logging.Logger = logging.getLogger("firecrawl")
|
||||
|
||||
|
||||
def _configure_logger() -> None:
|
||||
"""
|
||||
Configure the firecrawl logger for console output.
|
||||
|
||||
The function attaches a handler for console output with a specific format and date
|
||||
format to the firecrawl logger.
|
||||
"""
|
||||
try:
|
||||
formatter = logging.Formatter(
|
||||
"[%(asctime)s - %(name)s:%(lineno)d - %(levelname)s] %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(console_handler)
|
||||
except Exception as e:
|
||||
logger.error("Failed to configure logging: %s", e)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
"""Set up logging based on the FIRECRAWL_LOGGING_LEVEL environment variable."""
|
||||
if logger.hasHandlers():
|
||||
return
|
||||
|
||||
if not (env := os.getenv("FIRECRAWL_LOGGING_LEVEL", "").upper()):
|
||||
logger.addHandler(logging.NullHandler())
|
||||
return
|
||||
|
||||
_configure_logger()
|
||||
|
||||
if env == "DEBUG":
|
||||
logger.setLevel(logging.DEBUG)
|
||||
elif env == "INFO":
|
||||
logger.setLevel(logging.INFO)
|
||||
elif env == "WARNING":
|
||||
logger.setLevel(logging.WARNING)
|
||||
elif env == "ERROR":
|
||||
logger.setLevel(logging.ERROR)
|
||||
elif env == "CRITICAL":
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
else:
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.warning("Unknown logging level: %s, defaulting to INFO", env)
|
||||
|
||||
setup_logging()
|
||||
logger.debug("Debugging logger setup")
|
||||
|
||||
__all__ = [
|
||||
'Firecrawl',
|
||||
'AsyncFirecrawl',
|
||||
'FirecrawlApp',
|
||||
'AsyncFirecrawlApp',
|
||||
'Watcher',
|
||||
'AsyncWatcher',
|
||||
'AuditMetadata',
|
||||
'V1FirecrawlApp',
|
||||
'AsyncV1FirecrawlApp',
|
||||
'V1JsonConfig',
|
||||
'V1ScrapeOptions',
|
||||
'V1ChangeTrackingOptions',
|
||||
]
|
||||
@@ -0,0 +1,5 @@
|
||||
FIRECRAWL_API_KEY=<your api key>
|
||||
FIRECRAWL_API_URL=https://api.firecrawl.dev
|
||||
|
||||
# TESTING PURPOSE:
|
||||
IDMUX_URL=
|
||||
@@ -0,0 +1,62 @@
|
||||
"""
|
||||
Pytest configuration for async E2E tests.
|
||||
Ensures environment variables are loaded before any test runs.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
|
||||
# Load environment IMMEDIATELY at module import time (before pytest collects tests)
|
||||
# This ensures env vars are loaded before the first test runs
|
||||
_env_loaded = False
|
||||
|
||||
def _ensure_env_loaded():
|
||||
"""Ensure environment is loaded exactly once, synchronously."""
|
||||
global _env_loaded
|
||||
if not _env_loaded:
|
||||
# Find and load .env from multiple possible locations
|
||||
env_file = find_dotenv(usecwd=True)
|
||||
if env_file:
|
||||
load_dotenv(env_file, override=True)
|
||||
else:
|
||||
# Try loading from current directory
|
||||
load_dotenv(override=True)
|
||||
_env_loaded = True
|
||||
|
||||
# Load env immediately when this module is imported
|
||||
_ensure_env_loaded()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def load_environment():
|
||||
"""Ensure environment variables are loaded before running any tests."""
|
||||
# Double-check env is loaded (should already be done at import time)
|
||||
_ensure_env_loaded()
|
||||
|
||||
# Validate required environment variables
|
||||
required_vars = ["API_KEY", "API_URL"]
|
||||
missing_vars = [var for var in required_vars if not os.getenv(var)]
|
||||
|
||||
if missing_vars:
|
||||
pytest.skip(f"Skipping E2E tests: Missing required environment variables: {', '.join(missing_vars)}")
|
||||
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def api_key():
|
||||
"""Provide API key for tests."""
|
||||
key = os.getenv("API_KEY")
|
||||
if not key:
|
||||
pytest.skip("API_KEY not set")
|
||||
return key
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def api_url():
|
||||
"""Provide API URL for tests."""
|
||||
url = os.getenv("API_URL")
|
||||
if not url:
|
||||
pytest.skip("API_URL not set")
|
||||
return url
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import asyncio
|
||||
import pytest
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_batch_start_and_status(api_key, api_url):
|
||||
client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
|
||||
start = await client.start_batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], max_concurrency=1)
|
||||
job_id = start.id
|
||||
|
||||
deadline = asyncio.get_event_loop().time() + 240
|
||||
status = await client.get_batch_scrape_status(job_id)
|
||||
while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
|
||||
await asyncio.sleep(2)
|
||||
status = await client.get_batch_scrape_status(job_id)
|
||||
|
||||
assert status.status in ("completed", "failed", "cancelled")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_batch_wait_minimal(api_key, api_url):
|
||||
client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
|
||||
job = await client.batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], poll_interval=1, timeout=120)
|
||||
assert job.status in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_batch_wait_with_all_params(api_key, api_url):
|
||||
client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
|
||||
json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
|
||||
job = await client.batch_scrape(
|
||||
[
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
],
|
||||
formats=[
|
||||
"markdown",
|
||||
{"type": "json", "prompt": "Extract page title", "schema": json_schema},
|
||||
{"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
|
||||
],
|
||||
only_main_content=True,
|
||||
mobile=False,
|
||||
ignore_invalid_urls=True,
|
||||
max_concurrency=2,
|
||||
zero_data_retention=False,
|
||||
poll_interval=1,
|
||||
timeout=180,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
assert job.status in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_cancel_batch(api_key, api_url):
|
||||
client = AsyncFirecrawl(api_key=api_key, api_url=api_url)
|
||||
start = await client.start_batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], max_concurrency=1)
|
||||
ok = await client.cancel_batch_scrape(start.id)
|
||||
assert ok is True
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import os
|
||||
import asyncio
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_start_and_status():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
job_id = start.id
|
||||
|
||||
deadline = asyncio.get_event_loop().time() + 180
|
||||
status = await client.get_crawl_status(job_id)
|
||||
while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
|
||||
await asyncio.sleep(2)
|
||||
status = await client.get_crawl_status(job_id)
|
||||
|
||||
assert status.status in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_with_all_params():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
# rich scrape options including json format
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}},
|
||||
"required": ["title"],
|
||||
}
|
||||
status = await client.crawl(
|
||||
url="https://docs.firecrawl.dev",
|
||||
prompt="Extract docs and blog",
|
||||
include_paths=["/docs/*", "/blog/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=2,
|
||||
ignore_sitemap=False,
|
||||
ignore_query_parameters=True,
|
||||
limit=5,
|
||||
crawl_entire_domain=False,
|
||||
allow_external_links=True,
|
||||
allow_subdomains=True,
|
||||
delay=1,
|
||||
max_concurrency=2,
|
||||
integration="_e2e-test",
|
||||
webhook="https://example.com/hook",
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=[
|
||||
"markdown",
|
||||
"rawHtml",
|
||||
{"type": "json", "prompt": "Extract title", "schema": json_schema},
|
||||
],
|
||||
only_main_content=True,
|
||||
mobile=False,
|
||||
timeout=20000,
|
||||
wait_for=500,
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=False,
|
||||
),
|
||||
zero_data_retention=False,
|
||||
poll_interval=2,
|
||||
timeout=180,
|
||||
)
|
||||
assert status.status in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_start_crawl_with_options():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=5, max_discovery_depth=2)
|
||||
assert start.id is not None and start.url is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_start_crawl_with_prompt():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://firecrawl.dev", prompt="Extract all blog posts", limit=3)
|
||||
assert start.id is not None and start.url is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_crawl_status_shape():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
status = await client.get_crawl_status(start.id)
|
||||
assert status.status in ("scraping", "completed", "failed")
|
||||
assert status.completed >= 0
|
||||
assert status.expires_at is not None
|
||||
assert isinstance(status.data, list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_with_wait():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
job = await client.crawl(url="https://docs.firecrawl.dev", limit=3, max_discovery_depth=2, poll_interval=1, timeout=120)
|
||||
assert job.status in ("completed", "failed")
|
||||
assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_with_prompt_and_wait():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
job = await client.crawl(url="https://docs.firecrawl.dev", prompt="Extract all blog posts", limit=3, poll_interval=1, timeout=120)
|
||||
assert job.status in ("completed", "failed")
|
||||
assert job.completed >= 0 and job.total >= 0 and isinstance(job.data, list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_with_scrape_options():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
scrape_opts = ScrapeOptions(formats=["markdown", "links"], only_main_content=False, mobile=True)
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
|
||||
assert start.id is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_with_json_format_object():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
|
||||
scrape_opts = ScrapeOptions(formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}])
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2, scrape_options=scrape_opts)
|
||||
assert start.id is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_cancel_crawl():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
cancelled = await client.cancel_crawl(start.id)
|
||||
assert cancelled is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_crawl_errors_and_invalid_job():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
errs = await client.get_crawl_errors(start.id)
|
||||
assert hasattr(errs, "errors") and hasattr(errs, "robots_blocked")
|
||||
with pytest.raises(Exception):
|
||||
await client.get_crawl_errors("invalid-job-id-12345")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_active_crawls():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
resp = await client.active_crawls()
|
||||
assert hasattr(resp, "success") and hasattr(resp, "crawls")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_active_crawls_with_running_crawl():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
# fetch active crawls and assert our ID is listed
|
||||
active = await client.active_crawls()
|
||||
ids = [c.id for c in active.crawls]
|
||||
assert start.id in ids
|
||||
# cleanup
|
||||
await client.cancel_crawl(start.id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_params_preview():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
params = await client.crawl_params_preview(
|
||||
url="https://docs.firecrawl.dev",
|
||||
prompt="Extract all blog posts and documentation",
|
||||
)
|
||||
assert params is not None
|
||||
# basic sanity: at least one field should be suggested
|
||||
has_any = any([
|
||||
getattr(params, "limit", None) is not None,
|
||||
getattr(params, "include_paths", None) is not None,
|
||||
getattr(params, "max_discovery_depth", None) is not None,
|
||||
])
|
||||
assert has_any
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_extract_minimal():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
res = await client.extract(urls=["https://docs.firecrawl.dev"], prompt="Extract title")
|
||||
assert res is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_extract_with_schema_and_options():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
|
||||
res = await client.extract(
|
||||
urls=["https://docs.firecrawl.dev"],
|
||||
prompt="Extract title",
|
||||
schema=schema,
|
||||
system_prompt="You are a helpful extractor",
|
||||
allow_external_links=False,
|
||||
enable_web_search=False,
|
||||
show_sources=False,
|
||||
integration="_e2e-test",
|
||||
# agent={"model": "FIRE-1", "prompt": "Extract title"}, # Skipping agent test in CI
|
||||
)
|
||||
assert res is not None
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_map_minimal():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
resp = await client.map("https://docs.firecrawl.dev")
|
||||
assert hasattr(resp, "links") and isinstance(resp.links, list)
|
||||
if resp.links:
|
||||
first = resp.links[0]
|
||||
assert hasattr(first, "url") and isinstance(first.url, str) and first.url.startswith("http")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sitemap", ["only", "include", "skip"])
|
||||
async def test_async_map_with_all_params(sitemap):
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
resp = await client.map(
|
||||
"https://docs.firecrawl.dev",
|
||||
search="docs",
|
||||
include_subdomains=True,
|
||||
limit=10,
|
||||
sitemap=sitemap,
|
||||
timeout=15000,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
assert hasattr(resp, "links") and isinstance(resp.links, list)
|
||||
assert len(resp.links) <= 10
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.v2.types import Document, ScrapeOptions
|
||||
|
||||
load_dotenv()
|
||||
API_KEY = (os.getenv("API_KEY") or "").strip()
|
||||
API_URL = (os.getenv("API_URL") or "").strip()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not API_KEY or not API_URL,
|
||||
reason="API_KEY and API_URL are required for async parse e2e tests",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_parse_uploaded_html():
|
||||
client = AsyncFirecrawl(api_key=API_KEY, api_url=API_URL)
|
||||
doc = await client.parse(
|
||||
b"<!DOCTYPE html><html><body><h1>Async Python Parse E2E</h1></body></html>",
|
||||
filename="python-parse-aio-e2e.html",
|
||||
content_type="text/html",
|
||||
options=ScrapeOptions(formats=["markdown"]),
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
assert doc.markdown is not None
|
||||
assert "Async Python Parse E2E" in doc.markdown
|
||||
@@ -0,0 +1,138 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.v2.types import Document
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_minimal():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
doc = await client.scrape("https://docs.firecrawl.dev")
|
||||
assert isinstance(doc, Document)
|
||||
assert (
|
||||
(doc.markdown and len(doc.markdown) > 0)
|
||||
or (doc.html and len(doc.html) > 0)
|
||||
or (doc.raw_html and len(doc.raw_html) > 0)
|
||||
or (doc.links is not None)
|
||||
or (doc.screenshot is not None)
|
||||
or (doc.json is not None)
|
||||
or (doc.summary is not None)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_with_all_params():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
# Include multiple formats with configuration
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}},
|
||||
"required": ["title"],
|
||||
}
|
||||
doc = await client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[
|
||||
"markdown",
|
||||
"rawHtml",
|
||||
{"type": "screenshot", "full_page": False, "quality": 70},
|
||||
{"type": "json", "prompt": "Extract title", "schema": json_schema},
|
||||
],
|
||||
headers={"User-Agent": "E2E-AIO"},
|
||||
include_tags=["main"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=True,
|
||||
timeout=20000,
|
||||
wait_for=500,
|
||||
mobile=False,
|
||||
parsers=["pdf"],
|
||||
actions=[],
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=False,
|
||||
fast_mode=False,
|
||||
use_mock=None,
|
||||
block_ads=False,
|
||||
proxy="basic",
|
||||
max_age=0,
|
||||
store_in_cache=False,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_with_options_markdown():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
doc = await client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=["markdown"],
|
||||
only_main_content=False,
|
||||
mobile=False,
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_with_screenshot_action_viewport():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
doc = await client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[{"type": "screenshot", "full_page": False, "quality": 80, "viewport": {"width": 800, "height": 600}}],
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("fmt,expect_field", [
|
||||
("markdown", "markdown"),
|
||||
("html", "html"),
|
||||
("raw_html", "raw_html"),
|
||||
("links", "links"),
|
||||
("screenshot", "screenshot"),
|
||||
("summary", "summary"),
|
||||
])
|
||||
async def test_async_scrape_basic_formats(fmt, expect_field):
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
doc = await client.scrape("https://docs.firecrawl.dev", formats=[fmt])
|
||||
assert isinstance(doc, Document)
|
||||
if expect_field == "markdown":
|
||||
assert doc.markdown is not None
|
||||
elif expect_field == "html":
|
||||
assert doc.html is not None
|
||||
elif expect_field == "raw_html":
|
||||
assert doc.raw_html is not None
|
||||
elif expect_field == "links":
|
||||
assert isinstance(doc.links, list)
|
||||
elif expect_field == "screenshot":
|
||||
assert doc.screenshot is not None
|
||||
elif expect_field == "summary":
|
||||
assert doc.summary is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_with_json_format_object():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
|
||||
doc = await client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
|
||||
only_main_content=True,
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_invalid_url():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
with pytest.raises(ValueError):
|
||||
await client.scrape("")
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.types import (
|
||||
SearchData,
|
||||
Document,
|
||||
ScrapeOptions,
|
||||
ScrapeFormats,
|
||||
SearchResultWeb,
|
||||
SearchResultNews,
|
||||
SearchResultImages,
|
||||
)
|
||||
|
||||
load_dotenv()
|
||||
|
||||
firecrawl = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def _collect_texts(entries):
|
||||
texts = []
|
||||
for r in entries or []:
|
||||
title = getattr(r, 'title', None) if hasattr(r, 'title') else None
|
||||
desc = getattr(r, 'description', None) if hasattr(r, 'description') else None
|
||||
if title:
|
||||
texts.append(str(title).lower())
|
||||
if desc:
|
||||
texts.append(str(desc).lower())
|
||||
return texts
|
||||
|
||||
def _is_document(entry) -> bool:
|
||||
try:
|
||||
from firecrawl.v2.types import Document
|
||||
return isinstance(entry, Document) or \
|
||||
hasattr(entry, 'markdown') or \
|
||||
hasattr(entry, 'html') or \
|
||||
hasattr(entry, 'raw_html') or \
|
||||
hasattr(entry, 'json') or \
|
||||
hasattr(entry, 'screenshot') or \
|
||||
hasattr(entry, 'change_tracking') or \
|
||||
hasattr(entry, 'summary')
|
||||
except Exception:
|
||||
return hasattr(entry, 'markdown') or \
|
||||
hasattr(entry, 'html') or \
|
||||
hasattr(entry, 'raw_html') or \
|
||||
hasattr(entry, 'json') or \
|
||||
hasattr(entry, 'screenshot') or \
|
||||
hasattr(entry, 'change_tracking') or \
|
||||
hasattr(entry, 'summary')
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_minimal_request():
|
||||
results = await firecrawl.search(
|
||||
query="What is the capital of France?"
|
||||
)
|
||||
assert isinstance(results, SearchData)
|
||||
assert hasattr(results, 'web')
|
||||
assert results.web is not None
|
||||
assert len(results.web) > 0
|
||||
assert hasattr(results, 'news')
|
||||
assert results.news is None
|
||||
assert hasattr(results, 'images')
|
||||
assert results.images is None
|
||||
|
||||
for result in results.web:
|
||||
assert isinstance(result, SearchResultWeb)
|
||||
assert hasattr(result, 'url')
|
||||
assert hasattr(result, 'title')
|
||||
assert hasattr(result, 'description')
|
||||
assert result.url.startswith('http')
|
||||
assert result.title is not None
|
||||
assert result.description is not None
|
||||
|
||||
all_text = ' '.join(_collect_texts(results.web))
|
||||
assert 'paris' in all_text
|
||||
|
||||
assert results.news is None
|
||||
assert results.images is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_with_sources():
|
||||
results = await firecrawl.search(
|
||||
query="firecrawl",
|
||||
sources=["web", "news", "images"],
|
||||
limit=3
|
||||
)
|
||||
assert isinstance(results, SearchData)
|
||||
assert results.web is not None
|
||||
assert len(results.web) <= 3
|
||||
assert isinstance(results.web[0], SearchResultWeb)
|
||||
|
||||
if results.news is not None:
|
||||
assert len(results.news) <= 3
|
||||
assert isinstance(results.news[0], SearchResultNews)
|
||||
|
||||
if results.images is not None:
|
||||
assert len(results.images) <= 3
|
||||
assert isinstance(results.images[0], SearchResultImages)
|
||||
|
||||
web_titles = [result.title.lower() for result in results.web]
|
||||
web_descriptions = [result.description.lower() for result in results.web]
|
||||
all_web_text = ' '.join(web_titles + web_descriptions)
|
||||
assert 'firecrawl' in all_web_text
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_result_structure():
|
||||
results = await firecrawl.search(
|
||||
query="test query",
|
||||
limit=1
|
||||
)
|
||||
if results.web and len(results.web) > 0:
|
||||
result = results.web[0]
|
||||
assert hasattr(result, 'url')
|
||||
assert hasattr(result, 'title')
|
||||
assert hasattr(result, 'description')
|
||||
assert isinstance(result.url, str)
|
||||
assert isinstance(result.title, str) or result.title is None
|
||||
assert isinstance(result.description, str) or result.description is None
|
||||
assert result.url.startswith('http')
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_all_parameters():
|
||||
from firecrawl.types import ScrapeOptions, Location, WaitAction
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"url": {"type": "string"}
|
||||
},
|
||||
"required": ["title", "description"]
|
||||
}
|
||||
results = await firecrawl.search(
|
||||
query="artificial intelligence",
|
||||
sources=[
|
||||
{"type": "web"},
|
||||
{"type": "news"}
|
||||
],
|
||||
limit=3,
|
||||
tbs="qdr:m",
|
||||
location="US",
|
||||
ignore_invalid_urls=True,
|
||||
timeout=60000,
|
||||
integration="_e2e-test",
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=[
|
||||
"markdown",
|
||||
"html",
|
||||
{
|
||||
"type": "json",
|
||||
"prompt": "Extract the title and description from the page",
|
||||
"schema": schema
|
||||
},
|
||||
{"type": "summary"}
|
||||
],
|
||||
headers={"User-Agent": "Firecrawl-Test/1.0"},
|
||||
include_tags=["h1", "h2", "p"],
|
||||
exclude_tags=["nav", "footer"],
|
||||
only_main_content=True,
|
||||
wait_for=2000,
|
||||
mobile=False,
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=True,
|
||||
block_ads=True,
|
||||
proxy="basic",
|
||||
max_age=3600000,
|
||||
store_in_cache=True,
|
||||
location=Location(
|
||||
country="US",
|
||||
languages=["en"]
|
||||
),
|
||||
actions=[
|
||||
WaitAction(milliseconds=1000)
|
||||
]
|
||||
)
|
||||
)
|
||||
assert isinstance(results, SearchData)
|
||||
assert hasattr(results, 'web')
|
||||
assert hasattr(results, 'news')
|
||||
assert hasattr(results, 'images')
|
||||
assert results.web is not None
|
||||
assert len(results.web) <= 3
|
||||
|
||||
non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]
|
||||
if non_doc_entries:
|
||||
all_web_text = ' '.join(_collect_texts(non_doc_entries))
|
||||
ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']
|
||||
assert any(term in all_web_text for term in ai_terms)
|
||||
|
||||
for result in results.web:
|
||||
assert isinstance(result, (SearchResultWeb, Document))
|
||||
if isinstance(result, Document):
|
||||
assert (result.markdown is not None) or (result.html is not None)
|
||||
else:
|
||||
assert hasattr(result, 'url')
|
||||
assert isinstance(result.url, str) and result.url.startswith('http')
|
||||
|
||||
if results.news is not None:
|
||||
assert len(results.news) <= 3
|
||||
for result in results.news:
|
||||
assert isinstance(result, (SearchResultNews, Document))
|
||||
if isinstance(result, Document):
|
||||
assert (result.markdown is not None) or (result.html is not None)
|
||||
else:
|
||||
assert hasattr(result, 'url')
|
||||
assert isinstance(result.url, str) and result.url.startswith('http')
|
||||
|
||||
assert results.images is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_formats_flexibility():
|
||||
# Test with list format
|
||||
results1 = await firecrawl.search(
|
||||
query="python programming",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=["markdown"]
|
||||
)
|
||||
)
|
||||
# Test with ScrapeFormats object
|
||||
results2 = await firecrawl.search(
|
||||
query="python programming",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=ScrapeFormats(markdown=True)
|
||||
)
|
||||
)
|
||||
assert isinstance(results1, SearchData)
|
||||
assert isinstance(results2, SearchData)
|
||||
assert results1.web is not None
|
||||
assert results2.web is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_search_with_json_format_object():
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
results = await firecrawl.search(
|
||||
query="site:docs.firecrawl.dev",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
|
||||
),
|
||||
)
|
||||
assert isinstance(results, SearchData)
|
||||
assert results.web is not None and len(results.web) >= 0
|
||||
@@ -0,0 +1,42 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_concurrency():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
conc = await client.get_concurrency()
|
||||
assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_credit_usage():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
credits = await client.get_credit_usage()
|
||||
assert hasattr(credits, "remaining_credits")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_token_usage():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
tokens = await client.get_token_usage()
|
||||
assert hasattr(tokens, "remaining_tokens")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_queue_status():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
status = await client.get_queue_status()
|
||||
assert hasattr(status, "jobs_in_queue")
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import os
|
||||
import asyncio
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.v2.watcher_async import AsyncWatcher
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_watcher_crawl_progresses():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
statuses = []
|
||||
async for snapshot in AsyncWatcher(client, start.id, kind="crawl", timeout=180):
|
||||
statuses.append(snapshot.status)
|
||||
if snapshot.status in ("completed", "failed"):
|
||||
break
|
||||
assert statuses and statuses[-1] in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_watcher_batch_progresses():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], max_concurrency=1)
|
||||
statuses = []
|
||||
async for snapshot in AsyncWatcher(client, start.id, kind="batch", timeout=240):
|
||||
statuses.append(snapshot.status)
|
||||
if snapshot.status in ("completed", "failed", "cancelled"):
|
||||
break
|
||||
assert statuses and statuses[-1] in ("completed", "failed", "cancelled")
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import os
|
||||
import json
|
||||
import pytest
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
def _idmux(identity_request: dict) -> dict:
|
||||
idmux_url = os.getenv("IDMUX_URL")
|
||||
if not idmux_url:
|
||||
raise EnvironmentError("IDMUX_URL is not set. E2E tests must use idmux for credentials.")
|
||||
run_number = int(os.getenv("GITHUB_RUN_NUMBER") or 0)
|
||||
payload = {
|
||||
"refName": os.getenv("GITHUB_REF_NAME") or "local",
|
||||
"runNumber": run_number,
|
||||
"concurrency": identity_request.get("concurrency", 100),
|
||||
**identity_request,
|
||||
}
|
||||
resp = requests.post(idmux_url + "/", json=payload)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def api_url():
|
||||
# Prefer TEST_URL, then FIRECRAWL_API_URL (for parity with JS), then legacy API_URL
|
||||
return (
|
||||
os.getenv("TEST_URL")
|
||||
or os.getenv("FIRECRAWL_API_URL")
|
||||
or os.getenv("API_URL")
|
||||
or "https://api.firecrawl.dev"
|
||||
)
|
||||
|
||||
# Resolve identity and export environment at import time so tests that read env at module import succeed
|
||||
_IDENTITY = None
|
||||
_API_URL = (
|
||||
os.getenv("TEST_URL")
|
||||
or os.getenv("FIRECRAWL_API_URL")
|
||||
or os.getenv("API_URL")
|
||||
or "https://api.firecrawl.dev"
|
||||
)
|
||||
|
||||
_IDMUX_URL = os.getenv("IDMUX_URL")
|
||||
if _IDMUX_URL:
|
||||
run_name = os.getenv("PYTEST_RUN_NAME") or "py-e2e"
|
||||
# If IDMUX_URL is set, idmux MUST succeed; do not silently fall back
|
||||
_IDENTITY = _idmux({"name": run_name})
|
||||
os.environ["API_KEY"] = _IDENTITY.get("apiKey", "")
|
||||
os.environ["API_URL"] = _API_URL
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def api_identity():
|
||||
return _IDENTITY or {"apiKey": os.getenv("API_KEY") or "", "teamId": os.getenv("TEST_TEAM_ID") or os.getenv("TEAM_ID") or ""}
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inject_client(request, api_identity, api_url):
|
||||
# For class-based tests that rely on self.client, inject a client if missing
|
||||
inst = getattr(request, "instance", None)
|
||||
if inst is not None and not hasattr(inst, "client"):
|
||||
try:
|
||||
from firecrawl import Firecrawl
|
||||
inst.client = Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url)
|
||||
except Exception:
|
||||
pass
|
||||
# For function-based modules that expect a module-level `firecrawl` symbol
|
||||
mod = getattr(request, "module", None)
|
||||
if mod is not None and not hasattr(mod, "firecrawl"):
|
||||
try:
|
||||
from firecrawl import Firecrawl
|
||||
setattr(mod, "firecrawl", Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import os
|
||||
import asyncio
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from firecrawl import AsyncFirecrawl
|
||||
from firecrawl.v2.types import Document
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scrape_minimal():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
doc = await client.scrape("https://docs.firecrawl.dev")
|
||||
assert isinstance(doc, Document)
|
||||
# Accept any primary content or alternate outputs
|
||||
assert doc.markdown is not None and doc.markdown and len(doc.markdown) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_crawl_start_and_status():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
job_id = start.id
|
||||
|
||||
# Poll status until terminal or timeout
|
||||
deadline = asyncio.get_event_loop().time() + 180
|
||||
status = await client.get_crawl_status(job_id)
|
||||
while status.status not in ("completed", "failed") and asyncio.get_event_loop().time() < deadline:
|
||||
await asyncio.sleep(2)
|
||||
status = await client.get_crawl_status(job_id)
|
||||
|
||||
assert status.status in ("completed", "failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_batch_start_and_status():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
start = await client.start_batch_scrape([
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
], formats=["markdown"], max_concurrency=1)
|
||||
job_id = start.id
|
||||
|
||||
deadline = asyncio.get_event_loop().time() + 240
|
||||
status = await client.get_batch_scrape_status(job_id)
|
||||
while status.status not in ("completed", "failed", "cancelled") and asyncio.get_event_loop().time() < deadline:
|
||||
await asyncio.sleep(2)
|
||||
status = await client.get_batch_scrape_status(job_id)
|
||||
|
||||
assert status.status in ("completed", "failed", "cancelled")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_usage_minimal():
|
||||
client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
conc = await client.get_concurrency()
|
||||
assert hasattr(conc, "concurrency") and hasattr(conc, "max_concurrency")
|
||||
|
||||
credits = await client.get_credit_usage()
|
||||
assert hasattr(credits, "remaining_credits")
|
||||
|
||||
tokens = await client.get_token_usage()
|
||||
assert hasattr(tokens, "remaining_tokens")
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import ScrapeOptions, PaginationConfig
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
class TestBatchScrapeE2E:
|
||||
"""End-to-end tests for batch scrape (v2)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_batch_scrape_minimal(self):
|
||||
"""Start a small batch and wait for completion."""
|
||||
urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
]
|
||||
|
||||
job = self.client.batch_scrape(urls, formats=["markdown"], poll_interval=1, wait_timeout=120)
|
||||
|
||||
assert job.status in ["completed", "failed"]
|
||||
assert job.completed >= 0
|
||||
assert job.total >= 0
|
||||
assert isinstance(job.data, list)
|
||||
|
||||
def test_start_batch_minimal_and_status(self):
|
||||
"""Start via start_batch_scrape (minimal), then fetch status once."""
|
||||
urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
]
|
||||
|
||||
start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], ignore_invalid_urls=True)
|
||||
assert start_resp.id is not None
|
||||
assert start_resp.url is not None
|
||||
|
||||
job = self.client.get_batch_scrape_status(start_resp.id)
|
||||
assert job.status in ["scraping", "completed", "failed"]
|
||||
assert job.total >= 0
|
||||
|
||||
def test_get_batch_scrape_status_page(self):
|
||||
"""Fetch a single batch scrape page using the next URL."""
|
||||
urls = [f"https://docs.firecrawl.dev?batch={i}" for i in range(15)]
|
||||
|
||||
start_resp = self.client.start_batch_scrape(
|
||||
urls,
|
||||
formats=["markdown"],
|
||||
ignore_invalid_urls=True,
|
||||
)
|
||||
assert start_resp.id is not None
|
||||
|
||||
pagination_config = PaginationConfig(auto_paginate=False)
|
||||
deadline = time.time() + 120
|
||||
status_job = None
|
||||
while time.time() < deadline:
|
||||
status_job = self.client.get_batch_scrape_status(
|
||||
start_resp.id,
|
||||
pagination_config=pagination_config,
|
||||
)
|
||||
if status_job.next:
|
||||
break
|
||||
if status_job.status in ["completed", "failed", "cancelled"]:
|
||||
break
|
||||
time.sleep(2)
|
||||
|
||||
assert status_job is not None
|
||||
if not status_job.next:
|
||||
pytest.skip("Batch scrape completed without pagination; skipping page fetch.")
|
||||
|
||||
next_page = self.client.get_batch_scrape_status_page(status_job.next)
|
||||
assert isinstance(next_page.data, list)
|
||||
assert next_page.status in ["scraping", "completed", "failed", "cancelled"]
|
||||
|
||||
def test_wait_batch_with_all_params(self):
|
||||
"""Blocking waiter with JSON and changeTracking formats plus many options."""
|
||||
urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
]
|
||||
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
|
||||
opts = ScrapeOptions(
|
||||
formats=[
|
||||
"markdown",
|
||||
{"type": "json", "prompt": "Extract page title", "schema": json_schema},
|
||||
{"type": "changeTracking", "prompt": "Track changes", "modes": ["json"]},
|
||||
],
|
||||
only_main_content=True,
|
||||
mobile=False,
|
||||
)
|
||||
|
||||
job = self.client.batch_scrape(
|
||||
urls,
|
||||
formats=opts.formats,
|
||||
only_main_content=opts.only_main_content,
|
||||
mobile=opts.mobile,
|
||||
ignore_invalid_urls=True,
|
||||
max_concurrency=2,
|
||||
zero_data_retention=False,
|
||||
poll_interval=1,
|
||||
wait_timeout=180,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
|
||||
assert job.status in ["completed", "failed"]
|
||||
assert job.completed >= 0
|
||||
assert job.total >= 0
|
||||
assert isinstance(job.data, list)
|
||||
|
||||
def test_cancel_batch(self):
|
||||
"""Start a batch and cancel it."""
|
||||
urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
]
|
||||
|
||||
start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], max_concurrency=1)
|
||||
assert start_resp.id is not None
|
||||
|
||||
cancelled = self.client.cancel_batch_scrape(start_resp.id)
|
||||
assert cancelled is True
|
||||
@@ -0,0 +1,305 @@
|
||||
import pytest
|
||||
import time
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import ScrapeOptions, PaginationConfig
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
class TestCrawlE2E:
|
||||
"""End-to-end tests for crawl functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test client."""
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_start_crawl_minimal_request(self):
|
||||
"""Test starting a crawl with minimal parameters."""
|
||||
crawl_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
|
||||
assert crawl_job.id is not None
|
||||
assert crawl_job.url is not None
|
||||
|
||||
def test_start_crawl_with_options(self):
|
||||
"""Test starting a crawl with options."""
|
||||
crawl_job = self.client.start_crawl(
|
||||
"https://docs.firecrawl.dev",
|
||||
limit=5,
|
||||
max_discovery_depth=2
|
||||
)
|
||||
|
||||
assert crawl_job.id is not None
|
||||
assert crawl_job.url is not None
|
||||
|
||||
def test_start_crawl_with_prompt(self):
|
||||
"""Test starting a crawl with prompt."""
|
||||
crawl_job = self.client.start_crawl(
|
||||
"https://firecrawl.dev",
|
||||
prompt="Extract all blog posts",
|
||||
limit=3
|
||||
)
|
||||
|
||||
assert crawl_job.id is not None
|
||||
assert crawl_job.url is not None
|
||||
|
||||
def test_get_crawl_status(self):
|
||||
"""Test getting crawl status."""
|
||||
# First start a crawl
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
assert start_job.id is not None
|
||||
|
||||
job_id = start_job.id
|
||||
|
||||
# Get status
|
||||
status_job = self.client.get_crawl_status(job_id)
|
||||
|
||||
assert status_job.status in ["scraping", "completed", "failed"]
|
||||
assert status_job.completed >= 0
|
||||
assert status_job.expires_at is not None
|
||||
assert status_job.next is None
|
||||
assert isinstance(status_job.data, list)
|
||||
|
||||
def test_get_crawl_status_page(self):
|
||||
"""Fetch a single crawl page using the next URL."""
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=25)
|
||||
assert start_job.id is not None
|
||||
|
||||
pagination_config = PaginationConfig(auto_paginate=False)
|
||||
deadline = time.time() + 120
|
||||
status_job = None
|
||||
while time.time() < deadline:
|
||||
status_job = self.client.get_crawl_status(
|
||||
start_job.id,
|
||||
pagination_config=pagination_config,
|
||||
)
|
||||
if status_job.next:
|
||||
break
|
||||
if status_job.status in ["completed", "failed", "cancelled"]:
|
||||
break
|
||||
time.sleep(2)
|
||||
|
||||
assert status_job is not None
|
||||
if not status_job.next:
|
||||
pytest.skip("Crawl completed without pagination; skipping page fetch.")
|
||||
|
||||
next_page = self.client.get_crawl_status_page(status_job.next)
|
||||
assert isinstance(next_page.data, list)
|
||||
assert next_page.status in ["scraping", "completed", "failed", "cancelled"]
|
||||
|
||||
def test_cancel_crawl(self):
|
||||
"""Test canceling a crawl."""
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
assert start_job.id is not None
|
||||
|
||||
job_id = start_job.id
|
||||
cancel_job = self.client.cancel_crawl(job_id)
|
||||
|
||||
time.sleep(5)
|
||||
assert cancel_job == True
|
||||
|
||||
def test_get_crawl_errors(self):
|
||||
"""Test getting crawl errors."""
|
||||
# First start a crawl
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=3)
|
||||
assert start_job.id is not None
|
||||
|
||||
job_id = start_job.id
|
||||
|
||||
# Get errors (should work even if no errors exist)
|
||||
errors_response = self.client.get_crawl_errors(job_id)
|
||||
|
||||
# Verify the response structure
|
||||
assert hasattr(errors_response, 'errors')
|
||||
assert hasattr(errors_response, 'robots_blocked')
|
||||
assert isinstance(errors_response.errors, list)
|
||||
assert isinstance(errors_response.robots_blocked, list)
|
||||
|
||||
# Errors list should contain dictionaries with expected fields
|
||||
for error in errors_response.errors:
|
||||
assert isinstance(error, dict)
|
||||
assert 'id' in error
|
||||
assert 'timestamp' in error
|
||||
assert 'url' in error
|
||||
assert 'error' in error
|
||||
assert isinstance(error['id'], str)
|
||||
assert isinstance(error['timestamp'], str)
|
||||
assert isinstance(error['url'], str)
|
||||
assert isinstance(error['error'], str)
|
||||
|
||||
# Robots blocked should be a list of strings
|
||||
for blocked_url in errors_response.robots_blocked:
|
||||
assert isinstance(blocked_url, str)
|
||||
|
||||
def test_get_crawl_errors_with_invalid_job_id(self):
|
||||
"""Test getting crawl errors with an invalid job ID."""
|
||||
with pytest.raises(Exception):
|
||||
self.client.get_crawl_errors("invalid-job-id-12345")
|
||||
|
||||
def test_get_active_crawls(self):
|
||||
"""Test getting active crawls."""
|
||||
# Get active crawls
|
||||
active_crawls_response = self.client.active_crawls()
|
||||
|
||||
# Verify the response structure
|
||||
assert hasattr(active_crawls_response, 'success')
|
||||
assert hasattr(active_crawls_response, 'crawls')
|
||||
assert isinstance(active_crawls_response.success, bool)
|
||||
assert isinstance(active_crawls_response.crawls, list)
|
||||
|
||||
# Each crawl should have the required fields
|
||||
for crawl in active_crawls_response.crawls:
|
||||
assert hasattr(crawl, 'id')
|
||||
assert hasattr(crawl, 'team_id')
|
||||
assert hasattr(crawl, 'url')
|
||||
assert isinstance(crawl.id, str)
|
||||
assert isinstance(crawl.team_id, str)
|
||||
assert isinstance(crawl.url, str)
|
||||
|
||||
# Options field is optional but if present should be a dict
|
||||
if hasattr(crawl, 'options') and crawl.options is not None:
|
||||
assert isinstance(crawl.options, dict)
|
||||
|
||||
def test_get_active_crawls_with_running_crawl(self):
|
||||
"""Test getting active crawls when there's a running crawl."""
|
||||
# Start a crawl
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=5)
|
||||
assert start_job.id is not None
|
||||
|
||||
# Get active crawls
|
||||
active_crawls_response = self.client.active_crawls()
|
||||
|
||||
# Verify the response structure
|
||||
assert hasattr(active_crawls_response, 'success')
|
||||
assert hasattr(active_crawls_response, 'crawls')
|
||||
assert isinstance(active_crawls_response.success, bool)
|
||||
assert isinstance(active_crawls_response.crawls, list)
|
||||
|
||||
# The started crawl should be in the active crawls list
|
||||
active_crawl_ids = [crawl.id for crawl in active_crawls_response.crawls]
|
||||
assert start_job.id in active_crawl_ids
|
||||
|
||||
# Cancel the crawl to clean up
|
||||
self.client.cancel_crawl(start_job.id)
|
||||
|
||||
def test_crawl_with_wait(self):
|
||||
"""Test crawl with wait for completion."""
|
||||
crawl_job = self.client.crawl(
|
||||
"docs.firecrawl.dev",
|
||||
limit=3,
|
||||
max_discovery_depth=2,
|
||||
poll_interval=1,
|
||||
timeout=120,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
|
||||
assert crawl_job.status in ["completed", "failed"]
|
||||
assert crawl_job.completed >= 0
|
||||
assert crawl_job.total >= 0
|
||||
assert isinstance(crawl_job.data, list)
|
||||
|
||||
def test_crawl_with_prompt_and_wait(self):
|
||||
"""Test crawl with prompt and wait for completion."""
|
||||
crawl_job = self.client.crawl(
|
||||
"https://docs.firecrawl.dev",
|
||||
prompt="Extract all blog posts",
|
||||
limit=3,
|
||||
poll_interval=1,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
assert crawl_job.status in ["completed", "failed"]
|
||||
assert crawl_job.completed >= 0
|
||||
assert crawl_job.total >= 0
|
||||
assert isinstance(crawl_job.data, list)
|
||||
|
||||
def test_crawl_with_scrape_options(self):
|
||||
"""Test crawl with scrape options."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown", "links"],
|
||||
only_main_content=False,
|
||||
mobile=True,
|
||||
)
|
||||
|
||||
crawl_job = self.client.start_crawl(
|
||||
"https://docs.firecrawl.dev",
|
||||
limit=2,
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
assert crawl_job.id is not None
|
||||
|
||||
def test_crawl_with_json_format_object(self):
|
||||
"""Crawl with scrape_options including a JSON format object (prompt + schema)."""
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
|
||||
)
|
||||
crawl_job = self.client.start_crawl(
|
||||
"https://docs.firecrawl.dev",
|
||||
limit=2,
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
assert crawl_job.id is not None
|
||||
|
||||
def test_crawl_all_parameters(self):
|
||||
"""Test crawl with all possible parameters."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown", "html"],
|
||||
headers={"User-Agent": "Test Bot"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
|
||||
crawl_job = self.client.start_crawl(
|
||||
"https://docs.firecrawl.dev",
|
||||
prompt="Extract all blog posts and documentation",
|
||||
include_paths=["/blog/*", "/docs/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=3,
|
||||
ignore_sitemap=False,
|
||||
ignore_query_parameters=True,
|
||||
limit=5,
|
||||
crawl_entire_domain=True,
|
||||
allow_external_links=False,
|
||||
allow_subdomains=True,
|
||||
delay=1,
|
||||
max_concurrency=2,
|
||||
webhook="https://example.com/hook",
|
||||
scrape_options=scrape_opts,
|
||||
zero_data_retention=False,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
|
||||
assert crawl_job.id is not None
|
||||
|
||||
|
||||
def test_crawl_params_preview(self):
|
||||
"""Test crawl_params function."""
|
||||
params_data = self.client.crawl_params_preview(
|
||||
"https://docs.firecrawl.dev",
|
||||
"Extract all blog posts and documentation"
|
||||
)
|
||||
|
||||
assert params_data is not None
|
||||
assert params_data.limit is not None or params_data.include_paths is not None or params_data.max_discovery_depth is not None
|
||||
assert 'blog/.*' in params_data.include_paths
|
||||
assert 'docs/.*' in params_data.include_paths
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
class TestExtractE2E:
|
||||
"""E2E tests for v2 client extract (proxied to v1)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_extract_minimal_with_prompt(self):
|
||||
resp = self.client.extract(
|
||||
urls=["https://docs.firecrawl.dev"],
|
||||
prompt="Extract the main page title",
|
||||
)
|
||||
|
||||
assert hasattr(resp, "success")
|
||||
assert resp.success is True or resp.success is False
|
||||
# data may be None if backend omits; presence depends on implementation
|
||||
|
||||
def test_extract_with_schema(self):
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
|
||||
resp = self.client.extract(
|
||||
urls=["https://docs.firecrawl.dev"],
|
||||
schema=schema,
|
||||
prompt="Extract the main page title",
|
||||
show_sources=True,
|
||||
enable_web_search=False,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
|
||||
assert hasattr(resp, "success")
|
||||
# if backend includes sources, ensure structure is a dict (do not fail if omitted)
|
||||
if hasattr(resp, "sources") and resp.sources is not None:
|
||||
assert isinstance(resp.sources, dict)
|
||||
|
||||
# check if resp.data schema is equal to schema
|
||||
assert isinstance(resp.data, dict)
|
||||
assert resp.data["title"] is not None
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestMapE2E:
|
||||
"""End-to-end tests for map functionality (v2)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_map_minimal_request(self):
|
||||
resp = self.client.map("https://docs.firecrawl.dev")
|
||||
|
||||
assert hasattr(resp, "links") and resp.links is not None
|
||||
assert isinstance(resp.links, list)
|
||||
|
||||
# Basic sanity checks on at least one link
|
||||
if len(resp.links) > 0:
|
||||
first = resp.links[0]
|
||||
assert hasattr(first, "url")
|
||||
assert isinstance(first.url, str) and first.url.startswith("http")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sitemap",
|
||||
[
|
||||
"only",
|
||||
"skip",
|
||||
"include",
|
||||
],
|
||||
)
|
||||
def test_map_with_options(self, sitemap):
|
||||
resp = self.client.map(
|
||||
"https://docs.firecrawl.dev",
|
||||
search="docs",
|
||||
include_subdomains=True, limit=10,
|
||||
sitemap=sitemap,
|
||||
timeout=15000,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
|
||||
assert hasattr(resp, "links") and isinstance(resp.links, list)
|
||||
|
||||
# Limit should be respected (server-side)
|
||||
assert len(resp.links) <= 10
|
||||
|
||||
for link in resp.links:
|
||||
assert hasattr(link, "url")
|
||||
assert isinstance(link.url, str) and link.url.startswith("http")
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
from firecrawl.v2.types import Document, ScrapeOptions
|
||||
|
||||
load_dotenv()
|
||||
API_KEY = (os.getenv("API_KEY") or "").strip()
|
||||
API_URL = (os.getenv("API_URL") or "").strip()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not API_KEY or not API_URL,
|
||||
reason="API_KEY and API_URL are required for parse e2e tests",
|
||||
)
|
||||
class TestParseE2E:
|
||||
def setup_method(self):
|
||||
self.client = Firecrawl(api_key=API_KEY, api_url=API_URL)
|
||||
|
||||
def test_parse_uploaded_html(self):
|
||||
doc = self.client.parse(
|
||||
b"<!DOCTYPE html><html><body><h1>Python Parse E2E</h1></body></html>",
|
||||
filename="python-parse-e2e.html",
|
||||
content_type="text/html",
|
||||
options=ScrapeOptions(formats=["markdown"]),
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
assert doc.markdown is not None
|
||||
assert "Python Parse E2E" in doc.markdown
|
||||
@@ -0,0 +1,191 @@
|
||||
import os
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
import json as _json
|
||||
import pytest
|
||||
from firecrawl.v2.types import Viewport, ScreenshotAction, Document
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
class TestScrapeE2E:
|
||||
"""End-to-end tests for scrape functionality (v2)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def _assert_valid_document(self, doc: Document):
|
||||
assert isinstance(doc, Document)
|
||||
# At least one main content field should be present
|
||||
assert (
|
||||
(doc.markdown is not None and len(doc.markdown) > 0)
|
||||
or (doc.html is not None and len(doc.html) > 0)
|
||||
or (doc.raw_html is not None and len(doc.raw_html) > 0)
|
||||
or (doc.summary is not None and len(doc.summary) > 0)
|
||||
)
|
||||
# Metadata should exist with a source URL or title when available
|
||||
assert doc.metadata is not None
|
||||
|
||||
def test_scrape_minimal(self):
|
||||
"""Scrape a URL with minimal parameters and return a document."""
|
||||
doc = self.client.scrape("https://docs.firecrawl.dev")
|
||||
self._assert_valid_document(doc)
|
||||
|
||||
def test_scrape_with_options_markdown(self):
|
||||
"""Scrape with simple markdown format and options passed as kwargs."""
|
||||
doc = self.client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=["markdown"],
|
||||
only_main_content=False,
|
||||
mobile=False,
|
||||
)
|
||||
self._assert_valid_document(doc)
|
||||
|
||||
def test_scrape_with_screenshot_action_viewport(self):
|
||||
"""Scrape with a screenshot action including viewport passed as kwargs."""
|
||||
viewport = Viewport(width=800, height=600)
|
||||
action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)
|
||||
doc = self.client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=["markdown"],
|
||||
actions=[action],
|
||||
)
|
||||
self._assert_valid_document(doc)
|
||||
|
||||
@pytest.mark.parametrize("fmt,expect_field", [
|
||||
("markdown", "markdown"),
|
||||
("html", "html"),
|
||||
("raw_html", "raw_html"),
|
||||
("links", "links"),
|
||||
("screenshot", "screenshot"),
|
||||
("summary", "summary"),
|
||||
])
|
||||
def test_scrape_basic_formats(self, fmt, expect_field):
|
||||
"""Verify basic formats request succeeds and expected fields are present when applicable."""
|
||||
doc = self.client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[fmt],
|
||||
)
|
||||
# For formats that are not content (links/screenshot/json), skip main-content assertion
|
||||
if expect_field not in {"links", "screenshot"}:
|
||||
self._assert_valid_document(doc)
|
||||
if expect_field == "markdown":
|
||||
assert doc.markdown is not None
|
||||
elif expect_field == "html":
|
||||
assert doc.html is not None
|
||||
elif expect_field == "raw_html":
|
||||
assert doc.raw_html is not None
|
||||
elif expect_field == "screenshot":
|
||||
assert doc.screenshot is not None
|
||||
elif expect_field == "links":
|
||||
assert isinstance(doc.links, list)
|
||||
assert len(doc.links) > 0
|
||||
|
||||
def test_scrape_with_json_format_object(self):
|
||||
"""Scrape with JSON format object (requires prompt and schema)."""
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
doc = self.client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}],
|
||||
only_main_content=True,
|
||||
)
|
||||
# JSON format may not include main content fields; ensure request succeeded
|
||||
assert isinstance(doc, Document)
|
||||
# If backend returns extracted json content, it should be present under `json`
|
||||
# (Do not fail if backend omits it; existence depends on implementation)
|
||||
# if hasattr(doc, 'json'):
|
||||
# assert doc.json is not None
|
||||
|
||||
def test_scrape_invalid_url(self):
|
||||
"""Scrape should fail with empty or invalid URLs."""
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
self.client.scrape("")
|
||||
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
self.client.scrape(" ")
|
||||
|
||||
def test_scrape_with_all_params(self):
|
||||
"""Comprehensive scrape using multiple formats and options."""
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}},
|
||||
"required": ["title"],
|
||||
}
|
||||
doc = self.client.scrape(
|
||||
"https://docs.firecrawl.dev",
|
||||
formats=[
|
||||
"markdown",
|
||||
"raw_html",
|
||||
{"type": "screenshot", "full_page": False, "quality": 70},
|
||||
{"type": "json", "prompt": "Extract title", "schema": json_schema},
|
||||
{"type": "summary" },
|
||||
],
|
||||
headers={"User-Agent": "E2E"},
|
||||
include_tags=["main"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=True,
|
||||
timeout=20000,
|
||||
wait_for=500,
|
||||
mobile=False,
|
||||
parsers=["pdf"],
|
||||
actions=[],
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=False,
|
||||
fast_mode=False,
|
||||
use_mock=None,
|
||||
block_ads=False,
|
||||
proxy="basic",
|
||||
max_age=0,
|
||||
store_in_cache=False,
|
||||
integration="_e2e-test",
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
|
||||
def test_scrape_images_format(self):
|
||||
"""Test images format extraction."""
|
||||
doc = self.client.scrape(
|
||||
"https://firecrawl.dev",
|
||||
formats=["images"]
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
assert doc.images is not None
|
||||
assert isinstance(doc.images, list)
|
||||
assert len(doc.images) > 0
|
||||
# Should find firecrawl logo/branding images
|
||||
assert any("firecrawl" in img.lower() or "logo" in img.lower() for img in doc.images)
|
||||
|
||||
def test_scrape_images_with_multiple_formats(self):
|
||||
"""Test images format works with other formats."""
|
||||
doc = self.client.scrape(
|
||||
"https://github.com",
|
||||
formats=["markdown", "links", "images"]
|
||||
)
|
||||
assert isinstance(doc, Document)
|
||||
assert doc.markdown is not None
|
||||
assert doc.links is not None
|
||||
assert doc.images is not None
|
||||
assert isinstance(doc.images, list)
|
||||
assert len(doc.images) > 0
|
||||
|
||||
# Images should find content not available in links format
|
||||
image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico']
|
||||
link_images = [
|
||||
link for link in (doc.links or [])
|
||||
if any(ext in link.lower() for ext in image_extensions)
|
||||
]
|
||||
|
||||
# Should discover additional images beyond those with obvious extensions
|
||||
assert len(doc.images) >= len(link_images)
|
||||
@@ -0,0 +1,270 @@
|
||||
from firecrawl import Firecrawl
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl.types import SearchData, Document, ScrapeOptions, SearchResultWeb, SearchResultNews, SearchResultImages
|
||||
|
||||
load_dotenv()
|
||||
|
||||
firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def _collect_texts(entries):
|
||||
texts = []
|
||||
for r in entries or []:
|
||||
title = getattr(r, 'title', None) if hasattr(r, 'title') else None
|
||||
desc = getattr(r, 'description', None) if hasattr(r, 'description') else None
|
||||
if title:
|
||||
texts.append(str(title).lower())
|
||||
if desc:
|
||||
texts.append(str(desc).lower())
|
||||
return texts
|
||||
|
||||
def _is_document(entry) -> bool:
|
||||
try:
|
||||
from firecrawl.v2.types import Document
|
||||
return isinstance(entry, Document) or \
|
||||
hasattr(entry, 'markdown') or \
|
||||
hasattr(entry, 'html') or \
|
||||
hasattr(entry, 'raw_html') or \
|
||||
hasattr(entry, 'json') or \
|
||||
hasattr(entry, 'screenshot') or \
|
||||
hasattr(entry, 'change_tracking') or \
|
||||
hasattr(entry, 'summary')
|
||||
except Exception:
|
||||
return hasattr(entry, 'markdown') or \
|
||||
hasattr(entry, 'html') or \
|
||||
hasattr(entry, 'raw_html') or \
|
||||
hasattr(entry, 'json') or \
|
||||
hasattr(entry, 'screenshot') or \
|
||||
hasattr(entry, 'change_tracking') or \
|
||||
hasattr(entry, 'summary')
|
||||
|
||||
def test_search_minimal_request():
|
||||
results = firecrawl.search(
|
||||
query="What is the capital of France?"
|
||||
)
|
||||
|
||||
assert isinstance(results, SearchData)
|
||||
assert hasattr(results, 'web')
|
||||
assert results.web is not None
|
||||
assert len(results.web) > 0
|
||||
assert hasattr(results, 'news')
|
||||
assert results.news is None
|
||||
assert hasattr(results, 'images')
|
||||
assert results.images is None
|
||||
|
||||
for result in results.web:
|
||||
assert isinstance(result, SearchResultWeb)
|
||||
assert hasattr(result, 'url')
|
||||
assert hasattr(result, 'title')
|
||||
assert hasattr(result, 'description')
|
||||
assert result.url.startswith('http')
|
||||
assert result.title is not None
|
||||
assert result.description is not None
|
||||
|
||||
all_text = ' '.join(_collect_texts(results.web))
|
||||
|
||||
assert 'paris' in all_text
|
||||
|
||||
assert results.news is None
|
||||
assert results.images is None
|
||||
|
||||
|
||||
def test_search_with_sources():
|
||||
"""Test search with specific sources."""
|
||||
results = firecrawl.search(
|
||||
query="firecrawl",
|
||||
sources=["web", "news", "images"],
|
||||
limit=3
|
||||
)
|
||||
|
||||
assert isinstance(results, SearchData)
|
||||
|
||||
assert results.web is not None
|
||||
assert len(results.web) <= 3
|
||||
assert isinstance(results.web[0], SearchResultWeb)
|
||||
|
||||
if results.news is not None:
|
||||
assert len(results.news) <= 3
|
||||
assert isinstance(results.news[0], SearchResultNews)
|
||||
|
||||
if results.images is not None:
|
||||
assert len(results.images) <= 3
|
||||
assert isinstance(results.images[0], SearchResultImages)
|
||||
|
||||
web_titles = [result.title.lower() for result in results.web]
|
||||
web_descriptions = [result.description.lower() for result in results.web]
|
||||
all_web_text = ' '.join(web_titles + web_descriptions)
|
||||
|
||||
assert 'firecrawl' in all_web_text
|
||||
|
||||
def test_search_result_structure():
|
||||
"""Test that SearchResult objects have the correct structure."""
|
||||
results = firecrawl.search(
|
||||
query="test query",
|
||||
limit=1
|
||||
)
|
||||
|
||||
if results.web and len(results.web) > 0:
|
||||
result = results.web[0]
|
||||
|
||||
assert hasattr(result, 'url')
|
||||
assert hasattr(result, 'title')
|
||||
assert hasattr(result, 'description')
|
||||
|
||||
assert isinstance(result.url, str)
|
||||
assert isinstance(result.title, str) or result.title is None
|
||||
assert isinstance(result.description, str) or result.description is None
|
||||
|
||||
# Test URL format
|
||||
assert result.url.startswith('http')
|
||||
|
||||
def test_search_all_parameters():
|
||||
"""Test search with all available parameters (comprehensive e2e test)."""
|
||||
from firecrawl.types import ScrapeOptions, JsonFormat, Location, WaitAction
|
||||
|
||||
# Define a schema for JSON extraction
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"url": {"type": "string"}
|
||||
},
|
||||
"required": ["title", "description"]
|
||||
}
|
||||
|
||||
results = firecrawl.search(
|
||||
query="artificial intelligence",
|
||||
sources=[
|
||||
{"type": "web"},
|
||||
{"type": "news"}
|
||||
],
|
||||
limit=3,
|
||||
tbs="qdr:m", # Last month
|
||||
location="US",
|
||||
ignore_invalid_urls=True,
|
||||
timeout=60000,
|
||||
integration="_e2e-test",
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=[
|
||||
"markdown",
|
||||
"html",
|
||||
{
|
||||
"type": "json",
|
||||
"prompt": "Extract the title and description from the page",
|
||||
"schema": schema
|
||||
},
|
||||
{"type": "summary"}
|
||||
],
|
||||
headers={"User-Agent": "Firecrawl-Test/1.0"},
|
||||
include_tags=["h1", "h2", "p"],
|
||||
exclude_tags=["nav", "footer"],
|
||||
only_main_content=True,
|
||||
wait_for=2000,
|
||||
mobile=False,
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=True,
|
||||
block_ads=True,
|
||||
proxy="basic",
|
||||
max_age=3600000, # 1 hour cache
|
||||
store_in_cache=True,
|
||||
location=Location(
|
||||
country="US",
|
||||
languages=["en"]
|
||||
),
|
||||
actions=[
|
||||
WaitAction(milliseconds=1000)
|
||||
]
|
||||
# Note: raw_html and screenshot_full_page are not supported by v2 API yet
|
||||
)
|
||||
)
|
||||
|
||||
# Test structure
|
||||
assert isinstance(results, SearchData)
|
||||
assert hasattr(results, 'web')
|
||||
assert hasattr(results, 'news')
|
||||
assert hasattr(results, 'images')
|
||||
|
||||
# Test that web results exist
|
||||
assert results.web is not None
|
||||
assert len(results.web) <= 3 # Should respect limit
|
||||
|
||||
# Test that results contain expected content for non-document entries only
|
||||
non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]
|
||||
if non_doc_entries:
|
||||
all_web_text = ' '.join(_collect_texts(non_doc_entries))
|
||||
ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']
|
||||
assert any(term in all_web_text for term in ai_terms)
|
||||
|
||||
# Test that each result has proper structure
|
||||
for result in results.web:
|
||||
assert isinstance(result, (SearchResultWeb, Document))
|
||||
if isinstance(result, Document):
|
||||
# Document path: ensure content present
|
||||
assert (result.markdown is not None) or (result.html is not None)
|
||||
else:
|
||||
# LinkResult path
|
||||
assert hasattr(result, 'url')
|
||||
assert isinstance(result.url, str) and result.url.startswith('http')
|
||||
|
||||
# Test that news results exist (if API supports it)
|
||||
if results.news is not None:
|
||||
assert len(results.news) <= 3
|
||||
for result in results.news:
|
||||
assert isinstance(result, (SearchResultNews, Document))
|
||||
if isinstance(result, Document):
|
||||
assert (result.markdown is not None) or (result.html is not None)
|
||||
else:
|
||||
assert hasattr(result, 'url')
|
||||
assert isinstance(result.url, str) and result.url.startswith('http')
|
||||
|
||||
# Test that unspecified sources are None
|
||||
assert results.images is None
|
||||
|
||||
|
||||
def test_search_formats_flexibility():
|
||||
"""Test that both list and ScrapeFormats work for formats."""
|
||||
from firecrawl.types import ScrapeFormats
|
||||
|
||||
# Test with list format
|
||||
results1 = firecrawl.search(
|
||||
query="python programming",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=["markdown"]
|
||||
)
|
||||
)
|
||||
|
||||
# Test with ScrapeFormats object
|
||||
results2 = firecrawl.search(
|
||||
query="python programming",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=ScrapeFormats(markdown=True)
|
||||
)
|
||||
)
|
||||
|
||||
# Both should work without errors
|
||||
assert isinstance(results1, SearchData)
|
||||
assert isinstance(results2, SearchData)
|
||||
assert results1.web is not None
|
||||
assert results2.web is not None
|
||||
|
||||
def test_search_with_json_format_object():
|
||||
"""Search with scrape_options including a JSON format object (prompt + schema)."""
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"}
|
||||
},
|
||||
"required": ["title"],
|
||||
}
|
||||
results = firecrawl.search(
|
||||
query="site:docs.firecrawl.dev",
|
||||
limit=1,
|
||||
scrape_options=ScrapeOptions(
|
||||
formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
|
||||
),
|
||||
)
|
||||
assert isinstance(results, SearchData)
|
||||
assert results.web is not None and len(results.web) >= 0
|
||||
@@ -0,0 +1,26 @@
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
class TestUsageE2E:
|
||||
def setup_method(self):
|
||||
# Environment is exported by conftest at import time
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_get_concurrency(self):
|
||||
resp = self.client.get_concurrency()
|
||||
# Shape assertions (endpoint not live yet, but types are defined)
|
||||
assert hasattr(resp, "concurrency")
|
||||
assert hasattr(resp, "max_concurrency")
|
||||
|
||||
def test_get_credit_usage(self):
|
||||
resp = self.client.get_credit_usage()
|
||||
assert hasattr(resp, "remaining_credits")
|
||||
|
||||
def test_get_token_usage(self):
|
||||
resp = self.client.get_token_usage()
|
||||
assert hasattr(resp, "remaining_tokens")
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import os
|
||||
import time
|
||||
from dotenv import load_dotenv
|
||||
from firecrawl import Firecrawl
|
||||
|
||||
load_dotenv()
|
||||
|
||||
if not os.getenv("API_KEY"):
|
||||
raise ValueError("API_KEY is not set")
|
||||
|
||||
if not os.getenv("API_URL"):
|
||||
raise ValueError("API_URL is not set")
|
||||
|
||||
|
||||
class TestWatcherE2E:
|
||||
def setup_method(self):
|
||||
from firecrawl import Firecrawl
|
||||
self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
|
||||
|
||||
def test_crawl_watcher(self):
|
||||
# Start a small crawl job
|
||||
start_job = self.client.start_crawl("https://docs.firecrawl.dev", limit=2)
|
||||
job_id = start_job.id
|
||||
|
||||
statuses = []
|
||||
w = self.client.watcher(job_id, kind="crawl", poll_interval=1, timeout=120)
|
||||
w.add_listener(lambda s: statuses.append(s.status))
|
||||
w.start()
|
||||
|
||||
# Wait for terminal state up to 180 seconds
|
||||
deadline = time.time() + 180
|
||||
while time.time() < deadline:
|
||||
if statuses and statuses[-1] in ["completed", "failed"]:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
w.stop()
|
||||
|
||||
assert len(statuses) > 0
|
||||
assert statuses[-1] in ["completed", "failed"]
|
||||
|
||||
def test_batch_watcher(self):
|
||||
urls = [
|
||||
"https://docs.firecrawl.dev",
|
||||
"https://firecrawl.dev",
|
||||
]
|
||||
start_resp = self.client.start_batch_scrape(urls, formats=["markdown"], max_concurrency=1)
|
||||
job_id = start_resp.id
|
||||
|
||||
statuses = []
|
||||
w = self.client.watcher(job_id, kind="batch", poll_interval=1, timeout=180)
|
||||
w.add_listener(lambda s: statuses.append(s.status))
|
||||
w.start()
|
||||
|
||||
deadline = time.time() + 240
|
||||
while time.time() < deadline:
|
||||
if statuses and statuses[-1] in ["completed", "failed", "cancelled"]:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
w.stop()
|
||||
|
||||
assert len(statuses) > 0
|
||||
assert statuses[-1] in ["completed", "failed", "cancelled"]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import CrawlParamsRequest
|
||||
from firecrawl.v2.methods.aio import crawl as aio_crawl
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawl_params_request_validation():
|
||||
with pytest.raises(ValueError):
|
||||
await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="", prompt="x"))
|
||||
with pytest.raises(ValueError):
|
||||
await aio_crawl.crawl_params_preview(None, CrawlParamsRequest(url="https://x", prompt=""))
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
from firecrawl.v2.types import CrawlRequest, ScrapeOptions, WebhookConfig
|
||||
from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
|
||||
|
||||
|
||||
class TestAsyncCrawlRequestPreparation:
|
||||
def test_basic_request(self):
|
||||
req = CrawlRequest(url="https://example.com")
|
||||
payload = _prepare_crawl_request(req)
|
||||
assert payload["url"] == "https://example.com"
|
||||
|
||||
def test_field_mappings(self):
|
||||
req = CrawlRequest(
|
||||
url="https://example.com",
|
||||
include_paths=["/docs/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=2,
|
||||
sitemap="skip",
|
||||
ignore_query_parameters=True,
|
||||
crawl_entire_domain=True,
|
||||
allow_external_links=False,
|
||||
allow_subdomains=True,
|
||||
max_concurrency=5,
|
||||
zero_data_retention=True,
|
||||
)
|
||||
payload = _prepare_crawl_request(req)
|
||||
assert payload["includePaths"] == ["/docs/*"]
|
||||
assert payload["excludePaths"] == ["/admin/*"]
|
||||
assert payload["maxDiscoveryDepth"] == 2
|
||||
assert payload["sitemap"] == "skip"
|
||||
assert payload["ignoreQueryParameters"] is True
|
||||
assert payload["crawlEntireDomain"] is True
|
||||
assert payload["allowExternalLinks"] is False
|
||||
assert payload["allowSubdomains"] is True
|
||||
assert payload["maxConcurrency"] == 5
|
||||
assert payload["zeroDataRetention"] is True
|
||||
|
||||
def test_webhook_preparation(self):
|
||||
# string webhook
|
||||
req = CrawlRequest(url="https://example.com", webhook="https://example.com/hook")
|
||||
payload = _prepare_crawl_request(req)
|
||||
assert payload["webhook"] == "https://example.com/hook"
|
||||
|
||||
# object webhook
|
||||
req2 = CrawlRequest(url="https://example.com", webhook=WebhookConfig(url="https://x/h", headers={"X": "1"}, events=["completed"]))
|
||||
payload2 = _prepare_crawl_request(req2)
|
||||
assert isinstance(payload2["webhook"], dict)
|
||||
assert payload2["webhook"]["url"] == "https://x/h"
|
||||
assert payload2["webhook"]["headers"] == {"X": "1"}
|
||||
|
||||
def test_webhook_none_values_excluded(self):
|
||||
req = CrawlRequest(
|
||||
url="https://example.com",
|
||||
webhook=WebhookConfig(url="https://example.com/webhook", headers=None, metadata=None, events=None),
|
||||
)
|
||||
payload = _prepare_crawl_request(req)
|
||||
webhook = payload["webhook"]
|
||||
assert webhook["url"] == "https://example.com/webhook"
|
||||
assert "headers" not in webhook
|
||||
assert "metadata" not in webhook
|
||||
assert "events" not in webhook
|
||||
|
||||
def test_all_fields_including_integration(self):
|
||||
req = CrawlRequest(
|
||||
url="https://example.com",
|
||||
include_paths=["/docs/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=2,
|
||||
sitemap="include",
|
||||
ignore_query_parameters=True,
|
||||
crawl_entire_domain=False,
|
||||
allow_external_links=True,
|
||||
allow_subdomains=True,
|
||||
max_concurrency=3,
|
||||
zero_data_retention=False,
|
||||
integration=" _unit-test ",
|
||||
)
|
||||
payload = _prepare_crawl_request(req)
|
||||
assert payload["integration"] == "_unit-test"
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
from firecrawl.v2.types import CrawlRequest, ScrapeOptions
|
||||
from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
|
||||
import pytest
|
||||
|
||||
|
||||
class TestAsyncCrawlValidation:
|
||||
def test_invalid_url(self):
|
||||
with pytest.raises(ValueError):
|
||||
_prepare_crawl_request(CrawlRequest(url=""))
|
||||
with pytest.raises(ValueError):
|
||||
_prepare_crawl_request(CrawlRequest(url=" "))
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import MapOptions
|
||||
from firecrawl.v2.methods.aio.map import _prepare_map_request
|
||||
|
||||
|
||||
class TestAsyncMapRequestPreparation:
|
||||
def test_basic(self):
|
||||
payload = _prepare_map_request("https://example.com")
|
||||
assert payload["url"] == "https://example.com"
|
||||
|
||||
def test_fields(self):
|
||||
opts = MapOptions(search="docs", include_subdomains=True, limit=10, sitemap="only", timeout=15000, integration=" _unit-test ")
|
||||
payload = _prepare_map_request("https://example.com", opts)
|
||||
assert payload["search"] == "docs"
|
||||
assert payload["includeSubdomains"] is True
|
||||
assert payload["limit"] == 10
|
||||
assert payload["sitemap"] == "only"
|
||||
assert payload["timeout"] == 15000
|
||||
assert payload["integration"] == "_unit-test"
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
from firecrawl.v2.methods.aio.parse import _prepare_parse_request
|
||||
|
||||
|
||||
class TestAsyncParseRequestPreparation:
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_parse_request_from_bytes(self):
|
||||
options = ScrapeOptions(formats=["markdown"], only_main_content=True)
|
||||
fields, files = await _prepare_parse_request(
|
||||
b"<html><body><h1>Async Parse</h1></body></html>",
|
||||
options,
|
||||
filename="async-upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
payload = json.loads(fields["options"])
|
||||
assert payload["formats"] == ["markdown"]
|
||||
assert payload["onlyMainContent"] is True
|
||||
assert payload["origin"].startswith("python-sdk@")
|
||||
|
||||
filename, file_bytes, mime_type = files["file"]
|
||||
assert filename == "async-upload.html"
|
||||
assert b"Async Parse" in file_bytes
|
||||
assert mime_type == "text/html"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_parse_request_rejects_missing_path(self):
|
||||
with pytest.raises(ValueError, match="File path does not exist"):
|
||||
await _prepare_parse_request("/tmp/does-not-exist-async-upload.html")
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import ScrapeOptions, Location
|
||||
from firecrawl.v2.methods.aio.scrape import (
|
||||
_prepare_scrape_request,
|
||||
interact,
|
||||
stop_interaction,
|
||||
)
|
||||
|
||||
|
||||
class _FakeAsyncResponse:
|
||||
def __init__(self, status_code: int, payload):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return str(self._payload)
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
post_response: _FakeAsyncResponse,
|
||||
delete_response: _FakeAsyncResponse,
|
||||
):
|
||||
self.post_response = post_response
|
||||
self.delete_response = delete_response
|
||||
self.last_post = None
|
||||
self.last_delete = None
|
||||
|
||||
async def post(self, endpoint, payload):
|
||||
self.last_post = (endpoint, payload)
|
||||
return self.post_response
|
||||
|
||||
async def delete(self, endpoint):
|
||||
self.last_delete = endpoint
|
||||
return self.delete_response
|
||||
|
||||
|
||||
class TestAsyncScrapeRequestPreparation:
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_request_preparation(self):
|
||||
payload = await _prepare_scrape_request("https://example.com", None)
|
||||
assert payload["url"] == "https://example.com"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_options_conversion(self):
|
||||
opts = ScrapeOptions(
|
||||
formats=["markdown", {"type": "screenshot", "full_page": True, "quality": 80}],
|
||||
include_tags=["main"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=True,
|
||||
wait_for=500,
|
||||
timeout=30000,
|
||||
mobile=True,
|
||||
parsers=["pdf"],
|
||||
location=Location(country="us", languages=["en"]),
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=False,
|
||||
fast_mode=True,
|
||||
use_mock="test",
|
||||
block_ads=False,
|
||||
proxy="basic",
|
||||
max_age=1000,
|
||||
store_in_cache=False,
|
||||
lockdown=True,
|
||||
)
|
||||
payload = await _prepare_scrape_request("https://example.com", opts)
|
||||
assert payload["url"] == "https://example.com"
|
||||
assert isinstance(payload.get("formats"), list) and "markdown" in payload["formats"]
|
||||
assert payload["includeTags"] == ["main"]
|
||||
assert payload["excludeTags"] == ["nav"]
|
||||
assert payload["onlyMainContent"] is True
|
||||
assert payload["waitFor"] == 500
|
||||
assert payload["timeout"] == 30000
|
||||
assert payload["mobile"] is True
|
||||
assert payload["skipTlsVerification"] is False
|
||||
assert payload["removeBase64Images"] is False
|
||||
assert payload["fastMode"] is True
|
||||
assert payload["useMock"] == "test"
|
||||
assert payload["blockAds"] is False
|
||||
assert payload["proxy"] == "basic"
|
||||
assert payload["maxAge"] == 1000
|
||||
assert payload["storeInCache"] is False
|
||||
assert payload["lockdown"] is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interact_request_and_response_normalization(self):
|
||||
client = _FakeAsyncClient(
|
||||
post_response=_FakeAsyncResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"stdout": "ok",
|
||||
"exitCode": 0,
|
||||
},
|
||||
),
|
||||
delete_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
)
|
||||
response = await interact(
|
||||
client,
|
||||
"job-123",
|
||||
"console.log('ok')",
|
||||
timeout=30,
|
||||
origin="_unit-test",
|
||||
)
|
||||
|
||||
assert client.last_post[0] == "/v2/scrape/job-123/interact"
|
||||
assert client.last_post[1] == {
|
||||
"code": "console.log('ok')",
|
||||
"language": "node",
|
||||
"timeout": 30,
|
||||
"origin": "_unit-test",
|
||||
}
|
||||
assert response.success is True
|
||||
assert response.exit_code == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interact_with_prompt(self):
|
||||
client = _FakeAsyncClient(
|
||||
post_response=_FakeAsyncResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"output": "Clicked the button",
|
||||
"cdpUrl": "wss://browser.example.com/cdp",
|
||||
"liveViewUrl": "https://live.example.com/view",
|
||||
"interactiveLiveViewUrl": "https://live.example.com/interactive",
|
||||
"stdout": "",
|
||||
"exitCode": 0,
|
||||
},
|
||||
),
|
||||
delete_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
)
|
||||
response = await interact(
|
||||
client,
|
||||
"job-456",
|
||||
prompt="Click the login button",
|
||||
)
|
||||
|
||||
assert client.last_post[0] == "/v2/scrape/job-456/interact"
|
||||
assert client.last_post[1] == {
|
||||
"language": "node",
|
||||
"prompt": "Click the login button",
|
||||
}
|
||||
assert response.success is True
|
||||
assert response.output == "Clicked the button"
|
||||
assert response.cdp_url == "wss://browser.example.com/cdp"
|
||||
assert response.live_view_url == "https://live.example.com/view"
|
||||
assert response.interactive_live_view_url == "https://live.example.com/interactive"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interact_validates_required_inputs(self):
|
||||
client = _FakeAsyncClient(
|
||||
post_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
delete_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
)
|
||||
with pytest.raises(ValueError, match="Job ID cannot be empty"):
|
||||
await interact(client, "", "console.log('ok')")
|
||||
with pytest.raises(ValueError, match="Either 'code' or 'prompt' must be provided"):
|
||||
await interact(client, "job-123")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interact_raises_when_success_false(self):
|
||||
client = _FakeAsyncClient(
|
||||
post_response=_FakeAsyncResponse(
|
||||
200,
|
||||
{
|
||||
"success": False,
|
||||
"error": "Replay context is unavailable",
|
||||
},
|
||||
),
|
||||
delete_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
)
|
||||
with pytest.raises(Exception, match="Replay context is unavailable"):
|
||||
await interact(client, "job-123", "console.log('ok')")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_interaction_request_and_response_normalization(self):
|
||||
client = _FakeAsyncClient(
|
||||
post_response=_FakeAsyncResponse(200, {"success": True}),
|
||||
delete_response=_FakeAsyncResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"sessionDurationMs": 900,
|
||||
"creditsBilled": 2,
|
||||
},
|
||||
),
|
||||
)
|
||||
response = await stop_interaction(client, "job-123")
|
||||
|
||||
assert client.last_delete == "/v2/scrape/job-123/interact"
|
||||
assert response.success is True
|
||||
assert response.session_duration_ms == 900
|
||||
assert response.credits_billed == 2
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import SearchRequest, ScrapeOptions
|
||||
from firecrawl.v2.methods.aio.search import _prepare_search_request
|
||||
|
||||
|
||||
class TestAsyncSearchRequestPreparation:
|
||||
def test_basic_request_preparation(self):
|
||||
request = SearchRequest(query="test query")
|
||||
data = _prepare_search_request(request)
|
||||
assert data["query"] == "test query"
|
||||
assert "ignore_invalid_urls" not in data
|
||||
assert "scrape_options" not in data
|
||||
|
||||
def test_all_fields_conversion(self):
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False,
|
||||
)
|
||||
request = SearchRequest(
|
||||
query="test query",
|
||||
sources=["web", "news"],
|
||||
exclude_domains=["example.com"],
|
||||
limit=10,
|
||||
tbs="qdr:w",
|
||||
location="US",
|
||||
ignore_invalid_urls=False,
|
||||
timeout=30000,
|
||||
scrape_options=scrape_opts,
|
||||
integration=" _unit-test ",
|
||||
)
|
||||
data = _prepare_search_request(request)
|
||||
assert data["ignoreInvalidURLs"] is False
|
||||
assert data["excludeDomains"] == ["example.com"]
|
||||
assert "exclude_domains" not in data
|
||||
assert "scrapeOptions" in data
|
||||
assert data["integration"] == "_unit-test"
|
||||
|
||||
def test_exclude_none_behavior(self):
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
sources=None,
|
||||
limit=None,
|
||||
tbs=None,
|
||||
location=None,
|
||||
ignore_invalid_urls=None,
|
||||
timeout=None,
|
||||
scrape_options=None,
|
||||
)
|
||||
data = _prepare_search_request(request)
|
||||
assert "query" in data
|
||||
assert len(data) == 1
|
||||
|
||||
def test_domain_filters_are_mutually_exclusive(self):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="include_domains and exclude_domains cannot both be specified",
|
||||
):
|
||||
SearchRequest(
|
||||
query="test",
|
||||
include_domains=["firecrawl.dev"],
|
||||
exclude_domains=["example.com"],
|
||||
)
|
||||
|
||||
def test_empty_scrape_options(self):
|
||||
request = SearchRequest(query="test", scrape_options=ScrapeOptions())
|
||||
data = _prepare_search_request(request)
|
||||
assert "scrapeOptions" in data
|
||||
scrape_data = data["scrapeOptions"]
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert "mobile" in scrape_data
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
from firecrawl.v2.types import (
|
||||
CrawlRequest,
|
||||
MapOptions,
|
||||
ScrapeOptions,
|
||||
SearchRequest,
|
||||
ThreatProtectionOptions,
|
||||
)
|
||||
from firecrawl.v2.methods.aio.agent import _prepare_agent_request
|
||||
from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch_request
|
||||
from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
|
||||
from firecrawl.v2.methods.aio.extract import _prepare_extract_request
|
||||
from firecrawl.v2.methods.aio.map import _prepare_map_request
|
||||
from firecrawl.v2.methods.aio.search import _prepare_search_request
|
||||
|
||||
|
||||
EXPECTED_WIRE_PAYLOAD = {
|
||||
"mode": "normal",
|
||||
"riskScoreThreshold": 80,
|
||||
"blacklist": ["*.blocked.example.com"],
|
||||
"whitelist": ["allowed.example.com"],
|
||||
"blockedTlds": ["zip"],
|
||||
"failurePolicy": "open",
|
||||
}
|
||||
|
||||
|
||||
def _threat_protection() -> ThreatProtectionOptions:
|
||||
return ThreatProtectionOptions(
|
||||
mode="normal",
|
||||
risk_score_threshold=80,
|
||||
blacklist=["*.blocked.example.com"],
|
||||
whitelist=["allowed.example.com"],
|
||||
blocked_tlds=["zip"],
|
||||
failure_policy="open",
|
||||
)
|
||||
|
||||
|
||||
class TestAioThreatProtectionRequestPreparation:
|
||||
"""The aio request preparers must serialize threat_protection too."""
|
||||
|
||||
def test_batch_scrape_top_level(self):
|
||||
data = _prepare_batch_request(
|
||||
["https://example.com"],
|
||||
options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_crawl_scrape_options(self):
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
data = _prepare_crawl_request(request)
|
||||
assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
|
||||
def test_search_top_level_and_scrape_options(self):
|
||||
request = SearchRequest(
|
||||
query="firecrawl",
|
||||
threat_protection=_threat_protection(),
|
||||
scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
data = _prepare_search_request(request)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_map_top_level(self):
|
||||
options = MapOptions(threat_protection=_threat_protection())
|
||||
data = _prepare_map_request("https://example.com", options)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_extract_top_level(self):
|
||||
data = _prepare_extract_request(
|
||||
["https://example.com"],
|
||||
prompt="extract",
|
||||
threat_protection=_threat_protection(),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
|
||||
def test_agent_top_level(self):
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="find pricing",
|
||||
threat_protection=_threat_protection(),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
from firecrawl.v2.types import ScrapeOptions, Location
|
||||
from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch
|
||||
|
||||
|
||||
class TestAsyncBatchRequestPreparation:
|
||||
def test_urls_validation_and_conversion(self):
|
||||
payload = _prepare_batch(["https://example.com", "http://foo.bar"], options=None)
|
||||
assert payload["urls"] == ["https://example.com", "http://foo.bar"]
|
||||
|
||||
def test_options_and_batch_fields(self):
|
||||
opts = ScrapeOptions(formats=["markdown"], only_main_content=True)
|
||||
payload = _prepare_batch(
|
||||
["https://example.com"],
|
||||
options=opts,
|
||||
webhook="https://hook.example",
|
||||
append_to_id="00000000-0000-0000-0000-000000000000",
|
||||
ignore_invalid_urls=True,
|
||||
max_concurrency=3,
|
||||
zero_data_retention=True,
|
||||
integration="zapier",
|
||||
)
|
||||
assert payload["webhook"] == "https://hook.example"
|
||||
assert payload["appendToId"] == "00000000-0000-0000-0000-000000000000"
|
||||
assert payload["ignoreInvalidURLs"] is True
|
||||
assert payload["maxConcurrency"] == 3
|
||||
assert payload["zeroDataRetention"] is True
|
||||
assert payload["integration"] == "zapier"
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import asyncio
|
||||
import time
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.client_async import AsyncFirecrawlClient
|
||||
from firecrawl.v2.utils.http_client_async import AsyncHttpClient
|
||||
from firecrawl.v2.utils.http_client import HttpClient
|
||||
from firecrawl.v2.methods.aio import batch as aio_batch
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scrape_concurrency(monkeypatch):
|
||||
async def fake_post(self, endpoint, data, headers=None, timeout=None):
|
||||
await asyncio.sleep(0.1)
|
||||
return httpx.Response(200, json={"success": True, "data": {}})
|
||||
|
||||
monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
|
||||
|
||||
client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
|
||||
|
||||
start = time.perf_counter()
|
||||
await asyncio.gather(
|
||||
client.scrape("https://firecrawl.dev"),
|
||||
client.scrape("https://firecrawl.dev"),
|
||||
client.scrape("https://firecrawl.dev")
|
||||
)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
# If calls run concurrently, total should be close to single 0.1s delay, not 0.3s
|
||||
assert elapsed < 0.25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_event_loop_not_blocked(monkeypatch):
|
||||
ticks = 0
|
||||
|
||||
async def ticker():
|
||||
nonlocal ticks
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0.05)
|
||||
ticks += 1
|
||||
|
||||
async def fake_post(self, endpoint, data, headers=None, timeout=None):
|
||||
await asyncio.sleep(0.2)
|
||||
return httpx.Response(200, json={"success": True, "data": {}})
|
||||
|
||||
monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
|
||||
|
||||
client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
|
||||
|
||||
await asyncio.gather(ticker(), client.scrape("https://a"))
|
||||
# If scrape awaited properly, ticker should have progressed several steps
|
||||
assert ticks >= 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_batch_scrape_polling_interval(monkeypatch):
|
||||
# Simulate one scraping status then completed
|
||||
class S: # simple status holder
|
||||
def __init__(self, status):
|
||||
self.status = status
|
||||
|
||||
states = ["scraping", "completed"]
|
||||
|
||||
async def fake_status(client, job_id):
|
||||
state = states.pop(0)
|
||||
return S(state)
|
||||
|
||||
monkeypatch.setattr(aio_batch, "get_batch_scrape_status", fake_status)
|
||||
|
||||
client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
|
||||
|
||||
start = time.perf_counter()
|
||||
await client.wait_batch_scrape("job-1", poll_interval=0.1, timeout=2)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
# Should take roughly one poll interval to reach completed
|
||||
assert 0.09 <= elapsed <= 0.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_transport_used_no_threads(monkeypatch):
|
||||
# Make any to_thread usage blow up
|
||||
monkeypatch.setattr(asyncio, "to_thread", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("to_thread not allowed")))
|
||||
# Make sync HttpClient unusable
|
||||
def _boom(*a, **k):
|
||||
raise RuntimeError("sync client should not be used")
|
||||
monkeypatch.setattr(HttpClient, "post", _boom)
|
||||
monkeypatch.setattr(HttpClient, "get", _boom)
|
||||
monkeypatch.setattr(HttpClient, "delete", _boom)
|
||||
|
||||
# Track true async concurrency
|
||||
active = 0
|
||||
max_active = 0
|
||||
async def fake_post(self, endpoint, data, headers=None, timeout=None):
|
||||
nonlocal active, max_active
|
||||
active += 1
|
||||
max_active = max(max_active, active)
|
||||
try:
|
||||
await asyncio.sleep(0.1)
|
||||
return httpx.Response(200, json={"success": True, "data": {}})
|
||||
finally:
|
||||
active -= 1
|
||||
|
||||
monkeypatch.setattr(AsyncHttpClient, "post", fake_post)
|
||||
|
||||
client = AsyncFirecrawlClient(api_key="test", api_url="http://localhost")
|
||||
|
||||
await asyncio.gather(
|
||||
client.scrape("https://firecrawl.dev"),
|
||||
client.scrape("https://firecrawl.dev"),
|
||||
client.search("q"), # uses async search
|
||||
)
|
||||
|
||||
assert max_active >= 2
|
||||
|
||||
@@ -0,0 +1,367 @@
|
||||
"""
|
||||
Unit tests for agent methods with mocked HTTP client.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import Mock, patch
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
from firecrawl.v2.methods.agent import (
|
||||
start_agent,
|
||||
agent,
|
||||
get_agent_status,
|
||||
cancel_agent,
|
||||
wait_agent
|
||||
)
|
||||
from firecrawl.v2.types import AgentResponse
|
||||
from firecrawl.v2.utils.error_handler import BadRequestError
|
||||
|
||||
|
||||
class TestAgentMethods:
|
||||
"""Unit tests for agent methods with mocked HTTP client."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.mock_client = Mock()
|
||||
self.job_id = "test-agent-123"
|
||||
|
||||
# Sample agent response
|
||||
self.sample_response = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "completed",
|
||||
"data": {
|
||||
"founders": [
|
||||
{"name": "John Doe", "role": "CEO"},
|
||||
{"name": "Jane Smith", "role": "CTO"}
|
||||
]
|
||||
},
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z"
|
||||
}
|
||||
|
||||
def test_start_agent_basic(self):
|
||||
"""Test starting an agent job with basic parameters."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
result = start_agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Find information about Firecrawl"
|
||||
)
|
||||
|
||||
# Check that post was called with correct endpoint
|
||||
self.mock_client.post.assert_called_once()
|
||||
call_args = self.mock_client.post.call_args
|
||||
assert call_args[0][0] == "/v2/agent"
|
||||
|
||||
# Check request body (second positional argument)
|
||||
body = call_args[0][1]
|
||||
assert body["prompt"] == "Find information about Firecrawl"
|
||||
assert "urls" not in body
|
||||
|
||||
# Check result
|
||||
assert isinstance(result, AgentResponse)
|
||||
assert result.id == self.job_id
|
||||
assert result.status == "processing"
|
||||
|
||||
def test_start_agent_with_urls(self):
|
||||
"""Test starting an agent job with URLs."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
urls = ["https://example.com", "https://test.com"]
|
||||
result = start_agent(
|
||||
self.mock_client,
|
||||
urls,
|
||||
prompt="Extract data"
|
||||
)
|
||||
|
||||
call_args = self.mock_client.post.call_args
|
||||
body = call_args[0][1]
|
||||
assert body["urls"] == urls
|
||||
|
||||
def test_start_agent_with_dict_schema(self):
|
||||
"""Test starting an agent job with dict schema."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"}
|
||||
}
|
||||
}
|
||||
|
||||
result = start_agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Extract data",
|
||||
schema=schema
|
||||
)
|
||||
|
||||
call_args = self.mock_client.post.call_args
|
||||
body = call_args[0][1]
|
||||
assert body["schema"] == schema
|
||||
|
||||
def test_start_agent_with_pydantic_schema(self):
|
||||
"""Test starting an agent job with Pydantic schema."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
class Founder(BaseModel):
|
||||
name: str = Field(description="Full name")
|
||||
role: Optional[str] = Field(None, description="Role")
|
||||
|
||||
class FoundersSchema(BaseModel):
|
||||
founders: List[Founder] = Field(description="List of founders")
|
||||
|
||||
result = start_agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Find founders",
|
||||
schema=FoundersSchema
|
||||
)
|
||||
|
||||
call_args = self.mock_client.post.call_args
|
||||
body = call_args[0][1]
|
||||
assert "schema" in body
|
||||
assert body["schema"]["type"] == "object"
|
||||
assert "founders" in body["schema"]["properties"]
|
||||
|
||||
def test_start_agent_with_all_params(self):
|
||||
"""Test starting an agent job with all parameters."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
schema = {"type": "object"}
|
||||
urls = ["https://example.com"]
|
||||
|
||||
result = start_agent(
|
||||
self.mock_client,
|
||||
urls,
|
||||
prompt="Complete test",
|
||||
schema=schema,
|
||||
integration="test-integration",
|
||||
max_credits=50,
|
||||
strict_constrain_to_urls=True
|
||||
)
|
||||
|
||||
call_args = self.mock_client.post.call_args
|
||||
body = call_args[0][1]
|
||||
assert body["prompt"] == "Complete test"
|
||||
assert body["urls"] == urls
|
||||
assert body["schema"] == schema
|
||||
assert body["integration"] == "test-integration"
|
||||
assert body["maxCredits"] == 50
|
||||
assert body["strictConstrainToURLs"] is True
|
||||
|
||||
def test_get_agent_status(self):
|
||||
"""Test getting agent status."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = self.sample_response
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
result = get_agent_status(self.mock_client, self.job_id)
|
||||
|
||||
# Check that get was called with correct endpoint
|
||||
self.mock_client.get.assert_called_once_with(f"/v2/agent/{self.job_id}")
|
||||
|
||||
# Check result
|
||||
assert isinstance(result, AgentResponse)
|
||||
assert result.id == self.job_id
|
||||
assert result.status == "completed"
|
||||
|
||||
def test_cancel_agent(self):
|
||||
"""Test canceling an agent job."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {"success": True}
|
||||
|
||||
self.mock_client.delete.return_value = mock_response
|
||||
|
||||
result = cancel_agent(self.mock_client, self.job_id)
|
||||
|
||||
# Check that delete was called with correct endpoint
|
||||
self.mock_client.delete.assert_called_once_with(f"/v2/agent/{self.job_id}")
|
||||
|
||||
# Check result
|
||||
assert result is True
|
||||
|
||||
@patch('time.sleep')
|
||||
def test_wait_agent_completed(self, mock_sleep):
|
||||
"""Test waiting for agent to complete."""
|
||||
# First call returns processing, second returns completed
|
||||
mock_response_processing = Mock()
|
||||
mock_response_processing.ok = True
|
||||
mock_response_processing.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
mock_response_completed = Mock()
|
||||
mock_response_completed.ok = True
|
||||
mock_response_completed.json.return_value = self.sample_response
|
||||
|
||||
self.mock_client.get.side_effect = [
|
||||
mock_response_processing,
|
||||
mock_response_completed
|
||||
]
|
||||
|
||||
result = wait_agent(self.mock_client, self.job_id, poll_interval=1)
|
||||
|
||||
# Should have called get twice
|
||||
assert self.mock_client.get.call_count == 2
|
||||
assert result.status == "completed"
|
||||
assert mock_sleep.call_count == 1
|
||||
|
||||
@patch('time.sleep')
|
||||
def test_wait_agent_timeout(self, mock_sleep):
|
||||
"""Test waiting for agent with timeout."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
# Mock time.time to simulate timeout
|
||||
with patch('time.time', side_effect=[0, 0, 5]): # Start at 0, timeout at 5
|
||||
result = wait_agent(
|
||||
self.mock_client,
|
||||
self.job_id,
|
||||
poll_interval=1,
|
||||
timeout=3 # Timeout after 3 seconds
|
||||
)
|
||||
|
||||
# Should return processing status due to timeout
|
||||
assert result.status == "processing"
|
||||
|
||||
@patch('time.sleep')
|
||||
def test_agent_complete_flow(self, mock_sleep):
|
||||
"""Test the complete agent flow (start + wait)."""
|
||||
# Mock start_agent response
|
||||
mock_start_response = Mock()
|
||||
mock_start_response.ok = True
|
||||
mock_start_response.json.return_value = {
|
||||
"success": True,
|
||||
"id": self.job_id,
|
||||
"status": "processing"
|
||||
}
|
||||
|
||||
# Mock get_agent_status responses
|
||||
mock_status_response = Mock()
|
||||
mock_status_response.ok = True
|
||||
mock_status_response.json.return_value = self.sample_response
|
||||
|
||||
self.mock_client.post.return_value = mock_start_response
|
||||
self.mock_client.get.return_value = mock_status_response
|
||||
|
||||
result = agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Find information",
|
||||
poll_interval=1
|
||||
)
|
||||
|
||||
# Should have called post once and get once
|
||||
assert self.mock_client.post.call_count == 1
|
||||
assert self.mock_client.get.call_count == 1
|
||||
|
||||
# Check result
|
||||
assert isinstance(result, AgentResponse)
|
||||
assert result.status == "completed"
|
||||
assert result.data is not None
|
||||
|
||||
def test_agent_immediate_completion(self):
|
||||
"""Test agent that completes immediately (no job ID)."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"data": {"result": "done"}
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
result = agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Quick task"
|
||||
)
|
||||
|
||||
# Should only call post, not get
|
||||
assert self.mock_client.post.call_count == 1
|
||||
assert self.mock_client.get.call_count == 0
|
||||
assert result.status == "completed"
|
||||
|
||||
def test_start_agent_error_handling(self):
|
||||
"""Test error handling in start_agent."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = False
|
||||
mock_response.status_code = 400
|
||||
mock_response.text = "Bad Request"
|
||||
# Mock response.json() to return error details
|
||||
mock_response.json.return_value = {
|
||||
"error": "Invalid request",
|
||||
"details": "Bad Request"
|
||||
}
|
||||
|
||||
self.mock_client.post.return_value = mock_response
|
||||
|
||||
with pytest.raises(BadRequestError) as exc_info:
|
||||
start_agent(
|
||||
self.mock_client,
|
||||
None,
|
||||
prompt="Test prompt"
|
||||
)
|
||||
|
||||
# Verify the exception has the correct status code
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "agent" in str(exc_info.value).lower()
|
||||
|
||||
+339
@@ -0,0 +1,339 @@
|
||||
"""
|
||||
Unit tests for agent request preparation.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
from firecrawl.v2.methods.agent import _prepare_agent_request
|
||||
from firecrawl.v2.types import AgentWebhookConfig
|
||||
|
||||
|
||||
class TestAgentRequestPreparation:
|
||||
"""Unit tests for agent request preparation."""
|
||||
|
||||
def test_basic_request_preparation(self):
|
||||
"""Test basic request preparation with minimal fields."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Find information about Firecrawl"
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Find information about Firecrawl"
|
||||
assert "urls" not in data
|
||||
assert "schema" not in data
|
||||
|
||||
def test_request_with_urls(self):
|
||||
"""Test request preparation with URLs."""
|
||||
urls = ["https://example.com", "https://test.com"]
|
||||
data = _prepare_agent_request(
|
||||
urls,
|
||||
prompt="Extract data from these pages"
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Extract data from these pages"
|
||||
assert data["urls"] == urls
|
||||
|
||||
def test_request_with_dict_schema(self):
|
||||
"""Test request preparation with dict schema."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"}
|
||||
}
|
||||
}
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Extract person data",
|
||||
schema=schema
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Extract person data"
|
||||
assert data["schema"] == schema
|
||||
|
||||
def test_request_with_pydantic_schema(self):
|
||||
"""Test request preparation with Pydantic BaseModel schema."""
|
||||
class Person(BaseModel):
|
||||
name: str = Field(description="Person's name")
|
||||
age: Optional[int] = Field(None, description="Person's age")
|
||||
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Extract person data",
|
||||
schema=Person
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Extract person data"
|
||||
assert "schema" in data
|
||||
assert data["schema"]["type"] == "object"
|
||||
assert "properties" in data["schema"]
|
||||
assert "name" in data["schema"]["properties"]
|
||||
assert "age" in data["schema"]["properties"]
|
||||
|
||||
def test_request_with_pydantic_schema_instance(self):
|
||||
"""Test request preparation with Pydantic model instance."""
|
||||
class Person(BaseModel):
|
||||
name: str = Field(description="Person's name")
|
||||
age: Optional[int] = Field(None, description="Person's age")
|
||||
|
||||
person_instance = Person(name="John", age=30)
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Extract person data",
|
||||
schema=person_instance
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Extract person data"
|
||||
assert "schema" in data
|
||||
# Should use the class schema, not the instance data
|
||||
assert data["schema"]["type"] == "object"
|
||||
|
||||
def test_request_with_nested_pydantic_schema(self):
|
||||
"""Test request preparation with nested Pydantic schema."""
|
||||
class Founder(BaseModel):
|
||||
name: str = Field(description="Full name of the founder")
|
||||
role: Optional[str] = Field(None, description="Role or position")
|
||||
|
||||
class FoundersSchema(BaseModel):
|
||||
founders: List[Founder] = Field(description="List of founders")
|
||||
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Find the founders",
|
||||
schema=FoundersSchema
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Find the founders"
|
||||
assert "schema" in data
|
||||
assert data["schema"]["type"] == "object"
|
||||
assert "founders" in data["schema"]["properties"]
|
||||
assert data["schema"]["properties"]["founders"]["type"] == "array"
|
||||
|
||||
def test_request_with_integration(self):
|
||||
"""Test request preparation with integration tag."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
integration=" test-integration "
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Test prompt"
|
||||
assert data["integration"] == "test-integration"
|
||||
|
||||
def test_request_with_max_credits(self):
|
||||
"""Test request preparation with max credits."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
max_credits=100
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Test prompt"
|
||||
assert data["maxCredits"] == 100
|
||||
|
||||
def test_request_with_strict_constrain_to_urls(self):
|
||||
"""Test request preparation with strict_constrain_to_urls."""
|
||||
data = _prepare_agent_request(
|
||||
["https://example.com"],
|
||||
prompt="Test prompt",
|
||||
strict_constrain_to_urls=True
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Test prompt"
|
||||
assert data["strictConstrainToURLs"] is True
|
||||
|
||||
def test_request_all_fields(self):
|
||||
"""Test request preparation with all fields."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"test": {"type": "string"}}
|
||||
}
|
||||
urls = ["https://example.com"]
|
||||
|
||||
data = _prepare_agent_request(
|
||||
urls,
|
||||
prompt="Complete test",
|
||||
schema=schema,
|
||||
integration="test-integration",
|
||||
max_credits=50,
|
||||
strict_constrain_to_urls=True
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Complete test"
|
||||
assert data["urls"] == urls
|
||||
assert data["schema"] == schema
|
||||
assert data["integration"] == "test-integration"
|
||||
assert data["maxCredits"] == 50
|
||||
assert data["strictConstrainToURLs"] is True
|
||||
|
||||
def test_request_with_empty_integration(self):
|
||||
"""Test that empty integration is not included."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
integration=" "
|
||||
)
|
||||
|
||||
assert "integration" not in data
|
||||
|
||||
def test_request_with_zero_max_credits(self):
|
||||
"""Test that zero max_credits is not included."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
max_credits=0
|
||||
)
|
||||
|
||||
assert "maxCredits" not in data
|
||||
|
||||
def test_request_with_false_strict_constrain(self):
|
||||
"""Test that False strict_constrain_to_urls is not included."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
strict_constrain_to_urls=False
|
||||
)
|
||||
|
||||
assert "strictConstrainToURLs" not in data
|
||||
|
||||
def test_request_with_invalid_schema_type_string(self):
|
||||
"""Test that invalid schema types raise ValueError."""
|
||||
with pytest.raises(ValueError, match="Invalid schema type"):
|
||||
_prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
schema="invalid_string_schema"
|
||||
)
|
||||
|
||||
def test_request_with_invalid_schema_type_int(self):
|
||||
"""Test that invalid schema types raise ValueError."""
|
||||
with pytest.raises(ValueError, match="Invalid schema type"):
|
||||
_prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
schema=123
|
||||
)
|
||||
|
||||
def test_request_with_invalid_schema_type_list(self):
|
||||
"""Test that invalid schema types raise ValueError."""
|
||||
with pytest.raises(ValueError, match="Invalid schema type"):
|
||||
_prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
schema=["not", "a", "valid", "schema"]
|
||||
)
|
||||
|
||||
def test_request_with_string_webhook(self):
|
||||
"""Test request preparation with string webhook URL."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
webhook="https://example.com/webhook"
|
||||
)
|
||||
|
||||
assert data["webhook"] == "https://example.com/webhook"
|
||||
|
||||
def test_request_with_webhook_config(self):
|
||||
"""Test request preparation with AgentWebhookConfig object."""
|
||||
webhook_config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
events=["completed", "failed"]
|
||||
)
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
assert data["webhook"]["url"] == "https://example.com/webhook"
|
||||
assert data["webhook"]["headers"] == {"Authorization": "Bearer token"}
|
||||
assert data["webhook"]["events"] == ["completed", "failed"]
|
||||
|
||||
def test_request_without_webhook(self):
|
||||
"""Test that webhook is not included when None."""
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt"
|
||||
)
|
||||
|
||||
assert "webhook" not in data
|
||||
|
||||
def test_webhook_config_excludes_none_values(self):
|
||||
"""Test that None values are excluded from webhook config."""
|
||||
webhook_config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook"
|
||||
)
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
assert "headers" not in data["webhook"]
|
||||
assert "metadata" not in data["webhook"]
|
||||
assert "events" not in data["webhook"]
|
||||
|
||||
def test_agent_specific_webhook_events(self):
|
||||
"""Test that agent-specific events (action, cancelled) are accepted."""
|
||||
webhook_config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
events=["started", "action", "completed", "failed", "cancelled"]
|
||||
)
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
assert "action" in data["webhook"]["events"]
|
||||
assert "cancelled" in data["webhook"]["events"]
|
||||
|
||||
def test_webhook_with_metadata(self):
|
||||
"""Test webhook config with metadata."""
|
||||
webhook_config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
metadata={"project": "test", "env": "staging"}
|
||||
)
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="Test prompt",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
assert data["webhook"]["metadata"] == {"project": "test", "env": "staging"}
|
||||
|
||||
def test_request_all_fields_with_webhook(self):
|
||||
"""Test request preparation with all fields including webhook."""
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {"test": {"type": "string"}}
|
||||
}
|
||||
urls = ["https://example.com"]
|
||||
webhook_config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
events=["completed"]
|
||||
)
|
||||
|
||||
data = _prepare_agent_request(
|
||||
urls,
|
||||
prompt="Complete test",
|
||||
schema=schema,
|
||||
integration="test-integration",
|
||||
max_credits=50,
|
||||
strict_constrain_to_urls=True,
|
||||
model="spark-1-pro",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
assert data["prompt"] == "Complete test"
|
||||
assert data["urls"] == urls
|
||||
assert data["schema"] == schema
|
||||
assert data["integration"] == "test-integration"
|
||||
assert data["maxCredits"] == 50
|
||||
assert data["strictConstrainToURLs"] is True
|
||||
assert data["model"] == "spark-1-pro"
|
||||
assert data["webhook"]["url"] == "https://example.com/webhook"
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Unit tests for AgentWebhookConfig."""
|
||||
|
||||
import pytest
|
||||
from firecrawl.v2.types import AgentWebhookConfig
|
||||
|
||||
|
||||
class TestAgentWebhookConfig:
|
||||
"""Test AgentWebhookConfig class functionality."""
|
||||
|
||||
def test_minimal_config(self):
|
||||
"""Test creating config with only URL."""
|
||||
config = AgentWebhookConfig(url="https://example.com/webhook")
|
||||
assert config.url == "https://example.com/webhook"
|
||||
assert config.headers is None
|
||||
assert config.metadata is None
|
||||
assert config.events is None
|
||||
|
||||
def test_full_config(self):
|
||||
"""Test creating config with all fields."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
metadata={"project": "test"},
|
||||
events=["started", "action", "completed"]
|
||||
)
|
||||
assert config.url == "https://example.com/webhook"
|
||||
assert config.headers == {"Authorization": "Bearer token"}
|
||||
assert config.metadata == {"project": "test"}
|
||||
assert config.events == ["started", "action", "completed"]
|
||||
|
||||
def test_url_required(self):
|
||||
"""Test that URL is required."""
|
||||
with pytest.raises(Exception):
|
||||
AgentWebhookConfig()
|
||||
|
||||
def test_serialization_excludes_none(self):
|
||||
"""Test model_dump excludes None values."""
|
||||
config = AgentWebhookConfig(url="https://example.com/webhook")
|
||||
data = config.model_dump(exclude_none=True)
|
||||
assert data == {"url": "https://example.com/webhook"}
|
||||
|
||||
def test_serialization_includes_all_fields(self):
|
||||
"""Test model_dump includes all non-None fields."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"X-Custom": "value"},
|
||||
events=["completed", "failed"]
|
||||
)
|
||||
data = config.model_dump(exclude_none=True)
|
||||
assert data == {
|
||||
"url": "https://example.com/webhook",
|
||||
"headers": {"X-Custom": "value"},
|
||||
"events": ["completed", "failed"]
|
||||
}
|
||||
|
||||
def test_agent_specific_events(self):
|
||||
"""Test that agent-specific events are valid."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
events=["started", "action", "completed", "failed", "cancelled"]
|
||||
)
|
||||
assert "action" in config.events
|
||||
assert "cancelled" in config.events
|
||||
|
||||
def test_single_event(self):
|
||||
"""Test config with single event."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
events=["completed"]
|
||||
)
|
||||
assert config.events == ["completed"]
|
||||
|
||||
def test_empty_headers(self):
|
||||
"""Test config with empty headers dict."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={}
|
||||
)
|
||||
assert config.headers == {}
|
||||
|
||||
def test_multiple_headers(self):
|
||||
"""Test config with multiple headers."""
|
||||
config = AgentWebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={
|
||||
"Authorization": "Bearer token",
|
||||
"X-Custom-Header": "value",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
)
|
||||
assert len(config.headers) == 3
|
||||
assert config.headers["Authorization"] == "Bearer token"
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from firecrawl.v2.methods.agent import _prepare_agent_request
|
||||
from firecrawl.v2.methods.aio.batch import _prepare as prepare_async_batch_request
|
||||
from firecrawl.v2.methods.batch import prepare_batch_scrape_request
|
||||
from firecrawl.v2.methods.map import _prepare_map_request
|
||||
from firecrawl.v2.methods.parse import _prepare_parse_request
|
||||
from firecrawl.v2.types import AuditMetadata, MapOptions, ParseOptions, ScrapeOptions
|
||||
from firecrawl.v2.utils.validation import prepare_scrape_options
|
||||
|
||||
|
||||
AUDIT_METADATA = AuditMetadata(username="alice@example.com")
|
||||
SERIALIZED_AUDIT_METADATA = {"username": "alice@example.com"}
|
||||
|
||||
|
||||
def test_scrape_and_batch_audit_metadata_request_preparation():
|
||||
options = ScrapeOptions(audit_metadata=AUDIT_METADATA)
|
||||
|
||||
assert prepare_scrape_options(options)["auditMetadata"] == SERIALIZED_AUDIT_METADATA
|
||||
assert (
|
||||
prepare_batch_scrape_request(
|
||||
["https://example.com"],
|
||||
audit_metadata=AUDIT_METADATA,
|
||||
)["auditMetadata"]
|
||||
== SERIALIZED_AUDIT_METADATA
|
||||
)
|
||||
assert (
|
||||
prepare_async_batch_request(
|
||||
["https://example.com"],
|
||||
audit_metadata=AUDIT_METADATA,
|
||||
)["auditMetadata"]
|
||||
== SERIALIZED_AUDIT_METADATA
|
||||
)
|
||||
|
||||
|
||||
def test_map_and_agent_audit_metadata_request_preparation():
|
||||
assert (
|
||||
_prepare_map_request(
|
||||
"https://example.com",
|
||||
MapOptions(audit_metadata=AUDIT_METADATA),
|
||||
)["auditMetadata"]
|
||||
== SERIALIZED_AUDIT_METADATA
|
||||
)
|
||||
assert (
|
||||
_prepare_agent_request(
|
||||
None,
|
||||
prompt="Extract the page",
|
||||
audit_metadata=AUDIT_METADATA,
|
||||
)["auditMetadata"]
|
||||
== SERIALIZED_AUDIT_METADATA
|
||||
)
|
||||
|
||||
|
||||
def test_parse_audit_metadata_request_preparation():
|
||||
fields, _ = _prepare_parse_request(
|
||||
b"<html></html>",
|
||||
ParseOptions(audit_metadata=AUDIT_METADATA),
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
assert json.loads(fields["options"])["auditMetadata"] == SERIALIZED_AUDIT_METADATA
|
||||
|
||||
|
||||
def test_audit_metadata_rejects_unsupported_fields():
|
||||
with pytest.raises(ValidationError):
|
||||
AuditMetadata(username="alice@example.com", session="session-123")
|
||||
@@ -0,0 +1,90 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import ScrapeOptions, Location, WebhookConfig
|
||||
from firecrawl.v2.methods.batch import prepare_batch_scrape_request
|
||||
|
||||
|
||||
class TestBatchScrapeRequestPreparation:
|
||||
"""Unit tests for batch scrape request preparation."""
|
||||
|
||||
def test_urls_validation(self):
|
||||
# empty list
|
||||
with pytest.raises(ValueError):
|
||||
prepare_batch_scrape_request([])
|
||||
# invalid protocol
|
||||
with pytest.raises(ValueError):
|
||||
prepare_batch_scrape_request(["example.com"]) # missing http(s)
|
||||
# valid
|
||||
data = prepare_batch_scrape_request(["https://example.com", "http://foo.bar"])
|
||||
assert data["urls"] == ["https://example.com", "http://foo.bar"]
|
||||
|
||||
def test_flatten_scrape_options(self):
|
||||
opts = ScrapeOptions(
|
||||
formats=["markdown", "change_tracking", {"type": "screenshot", "full_page": True, "quality": 80}],
|
||||
include_tags=["main"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=True,
|
||||
wait_for=500,
|
||||
timeout=30000,
|
||||
mobile=True,
|
||||
parsers=["pdf"],
|
||||
actions=[{"type": "screenshot", "full_page": True}],
|
||||
location=Location(country="us", languages=["en"]),
|
||||
skip_tls_verification=False,
|
||||
remove_base64_images=False,
|
||||
fast_mode=True,
|
||||
use_mock="test",
|
||||
block_ads=False,
|
||||
proxy="basic",
|
||||
max_age=1000,
|
||||
store_in_cache=False,
|
||||
)
|
||||
data = prepare_batch_scrape_request(["https://example.com"], options=opts)
|
||||
|
||||
# Formats should be at top-level as list, with screenshot normalized to object w/ fullPage
|
||||
assert isinstance(data.get("formats"), list)
|
||||
assert "markdown" in data["formats"]
|
||||
# snake_case format should be converted to camelCase
|
||||
assert "changeTracking" in data["formats"]
|
||||
found_obj = next((f for f in data["formats"] if isinstance(f, dict) and f.get("type") == "screenshot"), None)
|
||||
assert found_obj is not None and found_obj.get("fullPage") is True and found_obj.get("quality") == 80
|
||||
|
||||
# Field conversions to camelCase
|
||||
assert data["includeTags"] == ["main"]
|
||||
assert data["excludeTags"] == ["nav"]
|
||||
assert data["onlyMainContent"] is True
|
||||
assert data["waitFor"] == 500
|
||||
assert data["timeout"] == 30000
|
||||
assert data["mobile"] is True
|
||||
assert data["parsers"] == ["pdf"]
|
||||
assert isinstance(data["actions"], list) and data["actions"][0]["type"] == "screenshot"
|
||||
assert isinstance(data["location"], dict) and data["location"]["country"] == "us"
|
||||
assert data["skipTlsVerification"] is False
|
||||
assert data["removeBase64Images"] is False
|
||||
assert data["fastMode"] is True
|
||||
assert data["useMock"] == "test"
|
||||
assert data["blockAds"] is False
|
||||
assert data["proxy"] == "basic"
|
||||
assert data["maxAge"] == 1000
|
||||
assert data["storeInCache"] is False
|
||||
|
||||
def test_batch_specific_fields(self):
|
||||
webhook = WebhookConfig(url="https://hook.test", headers={"X": "Y"}, events=["completed"])
|
||||
data = prepare_batch_scrape_request(
|
||||
["https://example.com"],
|
||||
webhook=webhook,
|
||||
append_to_id="00000000-0000-0000-0000-000000000000",
|
||||
ignore_invalid_urls=True,
|
||||
max_concurrency=5,
|
||||
zero_data_retention=True,
|
||||
integration="_unit-test",
|
||||
)
|
||||
assert isinstance(data["webhook"], dict) and data["webhook"]["url"] == "https://hook.test"
|
||||
assert data["appendToId"] == "00000000-0000-0000-0000-000000000000"
|
||||
assert data["ignoreInvalidURLs"] is True
|
||||
assert data["maxConcurrency"] == 5
|
||||
assert data["zeroDataRetention"] is True
|
||||
assert data["integration"] == "_unit-test"
|
||||
|
||||
def test_string_webhook_is_passed_verbatim(self):
|
||||
data = prepare_batch_scrape_request(["https://example.com"], webhook="https://hook.simple")
|
||||
assert data["webhook"] == "https://hook.simple"
|
||||
@@ -0,0 +1,214 @@
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock
|
||||
from firecrawl.v2.methods.scrape import scrape
|
||||
from firecrawl.v2.types import ScrapeOptions, Document
|
||||
|
||||
|
||||
class TestBrandingFormat:
|
||||
"""Unit tests for branding format support."""
|
||||
|
||||
def test_scrape_with_branding_format_returns_branding_data(self):
|
||||
"""Test that scraping with branding format returns branding data."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example",
|
||||
"branding": {
|
||||
"colorScheme": "light",
|
||||
"colors": {
|
||||
"primary": "#E11D48",
|
||||
"secondary": "#3B82F6",
|
||||
"accent": "#F59E0B"
|
||||
},
|
||||
"typography": {
|
||||
"fontFamilies": {
|
||||
"primary": "Inter",
|
||||
"heading": "Poppins"
|
||||
},
|
||||
"fontSizes": {
|
||||
"h1": "2.5rem",
|
||||
"body": "1rem"
|
||||
}
|
||||
},
|
||||
"spacing": {
|
||||
"baseUnit": 8
|
||||
},
|
||||
"components": {
|
||||
"buttonPrimary": {
|
||||
"background": "#E11D48",
|
||||
"textColor": "#FFFFFF",
|
||||
"borderRadius": "0.5rem"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
|
||||
|
||||
assert result.branding is not None
|
||||
assert result.branding.color_scheme == "light"
|
||||
assert result.branding.colors["primary"] == "#E11D48"
|
||||
assert result.branding.typography["fontFamilies"]["primary"] == "Inter"
|
||||
assert result.branding.spacing["baseUnit"] == 8
|
||||
assert result.branding.components["buttonPrimary"]["background"] == "#E11D48"
|
||||
|
||||
def test_scrape_with_branding_and_markdown_formats_returns_both(self):
|
||||
"""Test that scraping with both branding and markdown formats returns both."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example Content",
|
||||
"branding": {
|
||||
"colorScheme": "dark",
|
||||
"colors": {
|
||||
"primary": "#10B981"
|
||||
},
|
||||
"typography": {
|
||||
"fontFamilies": {
|
||||
"primary": "Roboto"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "branding"]))
|
||||
|
||||
assert result.markdown == "# Example Content"
|
||||
assert result.branding is not None
|
||||
assert result.branding.color_scheme == "dark"
|
||||
assert result.branding.colors["primary"] == "#10B981"
|
||||
|
||||
def test_scrape_without_branding_format_does_not_return_branding(self):
|
||||
"""Test that scraping without branding format does not return branding."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example"
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))
|
||||
|
||||
assert result.markdown == "# Example"
|
||||
assert result.branding is None
|
||||
|
||||
def test_branding_format_with_all_nested_fields(self):
|
||||
"""Test branding format with all nested fields populated."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"branding": {
|
||||
"colorScheme": "light",
|
||||
"logo": "https://example.com/logo.png",
|
||||
"fonts": [
|
||||
{"family": "Inter", "weight": 400},
|
||||
{"family": "Poppins", "weight": 700}
|
||||
],
|
||||
"colors": {
|
||||
"primary": "#E11D48",
|
||||
"background": "#FFFFFF"
|
||||
},
|
||||
"typography": {
|
||||
"fontFamilies": {"primary": "Inter"},
|
||||
"fontStacks": {"body": ["Inter", "sans-serif"]},
|
||||
"fontSizes": {"h1": "2.5rem"},
|
||||
"lineHeights": {"body": 1.5},
|
||||
"fontWeights": {"regular": 400}
|
||||
},
|
||||
"spacing": {
|
||||
"baseUnit": 8,
|
||||
"padding": {"sm": 8, "md": 16}
|
||||
},
|
||||
"components": {
|
||||
"buttonPrimary": {
|
||||
"background": "#E11D48",
|
||||
"textColor": "#FFFFFF"
|
||||
}
|
||||
},
|
||||
"icons": {
|
||||
"style": "outline",
|
||||
"primaryColor": "#E11D48"
|
||||
},
|
||||
"images": {
|
||||
"logo": "https://example.com/logo.png",
|
||||
"favicon": "https://example.com/favicon.ico"
|
||||
},
|
||||
"animations": {
|
||||
"transitionDuration": "200ms",
|
||||
"easing": "ease-in-out"
|
||||
},
|
||||
"layout": {
|
||||
"grid": {"columns": 12, "maxWidth": "1200px"},
|
||||
"headerHeight": "64px"
|
||||
},
|
||||
"tone": {
|
||||
"voice": "professional",
|
||||
"emojiUsage": "minimal"
|
||||
},
|
||||
"personality": {
|
||||
"tone": "professional",
|
||||
"energy": "medium",
|
||||
"targetAudience": "developers"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
|
||||
|
||||
assert result.branding is not None
|
||||
assert result.branding.color_scheme == "light"
|
||||
assert result.branding.logo == "https://example.com/logo.png"
|
||||
assert len(result.branding.fonts) == 2
|
||||
assert result.branding.typography["fontStacks"]["body"] == ["Inter", "sans-serif"]
|
||||
assert result.branding.spacing["padding"] == {"sm": 8, "md": 16}
|
||||
assert result.branding.icons["style"] == "outline"
|
||||
assert result.branding.images["favicon"] == "https://example.com/favicon.ico"
|
||||
assert result.branding.animations["easing"] == "ease-in-out"
|
||||
assert result.branding.layout["grid"]["columns"] == 12
|
||||
assert result.branding.personality["tone"] == "professional"
|
||||
|
||||
def test_branding_colorscheme_normalization(self):
|
||||
"""Test that colorScheme is normalized to color_scheme."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"branding": {
|
||||
"colorScheme": "dark",
|
||||
"colors": {"primary": "#000000"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["branding"]))
|
||||
|
||||
assert result.branding is not None
|
||||
assert result.branding.color_scheme == "dark"
|
||||
assert not hasattr(result.branding, "colorScheme")
|
||||
@@ -0,0 +1,70 @@
|
||||
"""
|
||||
Unit tests for crawl params functionality in Firecrawl v2 SDK.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from firecrawl.v2.types import CrawlParamsRequest, CrawlParamsData
|
||||
|
||||
|
||||
class TestCrawlParamsRequest:
|
||||
"""Unit tests for CrawlParamsRequest."""
|
||||
|
||||
def test_crawl_params_request_creation(self):
|
||||
"""Test creating CrawlParamsRequest with valid data."""
|
||||
request = CrawlParamsRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts"
|
||||
)
|
||||
|
||||
assert request.url == "https://example.com"
|
||||
assert request.prompt == "Extract all blog posts"
|
||||
|
||||
def test_crawl_params_request_serialization(self):
|
||||
"""Test that CrawlParamsRequest serializes correctly."""
|
||||
request = CrawlParamsRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts and documentation"
|
||||
)
|
||||
|
||||
data = request.model_dump()
|
||||
|
||||
assert data["url"] == "https://example.com"
|
||||
assert data["prompt"] == "Extract all blog posts and documentation"
|
||||
|
||||
|
||||
class TestCrawlParamsData:
|
||||
"""Unit tests for CrawlParamsData."""
|
||||
|
||||
def test_crawl_params_data_creation(self):
|
||||
"""Test creating CrawlParamsData with minimal data."""
|
||||
data = CrawlParamsData()
|
||||
|
||||
assert data.include_paths is None
|
||||
assert data.exclude_paths is None
|
||||
assert data.max_discovery_depth is None
|
||||
assert data.sitemap is None
|
||||
assert data.limit is None
|
||||
assert data.crawl_entire_domain is False
|
||||
assert data.allow_external_links is False
|
||||
assert data.scrape_options is None
|
||||
assert data.warning is None
|
||||
|
||||
def test_crawl_params_data_with_values(self):
|
||||
"""Test creating CrawlParamsData with values."""
|
||||
data = CrawlParamsData(
|
||||
include_paths=["/blog/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=3,
|
||||
limit=50,
|
||||
crawl_entire_domain=True,
|
||||
allow_external_links=False,
|
||||
warning="Test warning"
|
||||
)
|
||||
|
||||
assert data.include_paths == ["/blog/*"]
|
||||
assert data.exclude_paths == ["/admin/*"]
|
||||
assert data.max_discovery_depth == 3
|
||||
assert data.limit == 50
|
||||
assert data.crawl_entire_domain is True
|
||||
assert data.allow_external_links is False
|
||||
assert data.warning == "Test warning"
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import CrawlRequest, ScrapeOptions
|
||||
from firecrawl.v2.methods.crawl import _prepare_crawl_request
|
||||
|
||||
|
||||
class TestCrawlRequestPreparation:
|
||||
"""Unit tests for crawl request preparation."""
|
||||
|
||||
def test_basic_request_preparation(self):
|
||||
"""Test basic request preparation with minimal fields."""
|
||||
request = CrawlRequest(url="https://example.com")
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Check basic fields
|
||||
assert data["url"] == "https://example.com"
|
||||
|
||||
# Check that no options are present
|
||||
assert "limit" not in data
|
||||
assert "prompt" not in data
|
||||
|
||||
def test_crawl_options_conversion(self):
|
||||
"""Test that CrawlOptions fields are converted to camelCase."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=10,
|
||||
max_discovery_depth=3,
|
||||
sitemap="skip",
|
||||
crawl_entire_domain=False,
|
||||
allow_external_links=True
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Check basic field
|
||||
assert data["url"] == "https://example.com"
|
||||
|
||||
# Check snake_case to camelCase conversions
|
||||
assert "limit" in data
|
||||
assert data["limit"] == 10
|
||||
assert "maxDiscoveryDepth" in data
|
||||
assert data["maxDiscoveryDepth"] == 3
|
||||
assert "sitemap" in data
|
||||
assert data["sitemap"] == "skip"
|
||||
assert "crawlEntireDomain" in data
|
||||
assert data["crawlEntireDomain"] is False
|
||||
assert "allowExternalLinks" in data
|
||||
assert data["allowExternalLinks"] is True
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "ignore_sitemap" not in data
|
||||
assert "crawl_entire_domain" not in data
|
||||
assert "allow_external_links" not in data
|
||||
|
||||
def test_scrape_options_conversion(self):
|
||||
"""Test that nested ScrapeOptions are converted to camelCase."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown", "html"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
assert "scrapeOptions" in data
|
||||
assert "scrape_options" not in data
|
||||
|
||||
# Check nested conversions
|
||||
scrape_data = data["scrapeOptions"]
|
||||
assert "includeTags" in scrape_data
|
||||
assert scrape_data["includeTags"] == ["h1", "h2"]
|
||||
assert "excludeTags" in scrape_data
|
||||
assert scrape_data["excludeTags"] == ["nav"]
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert scrape_data["onlyMainContent"] is False
|
||||
assert "waitFor" in scrape_data
|
||||
assert scrape_data["waitFor"] == 2000
|
||||
assert "skipTlsVerification" in scrape_data
|
||||
assert scrape_data["skipTlsVerification"] is True
|
||||
assert "removeBase64Images" in scrape_data
|
||||
assert scrape_data["removeBase64Images"] is False
|
||||
|
||||
def test_all_fields_conversion(self):
|
||||
"""Test request preparation with all possible fields."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
headers={"User-Agent": "Test"},
|
||||
only_main_content=False,
|
||||
mobile=True
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts and documentation",
|
||||
include_paths=["/blog/*", "/docs/*"],
|
||||
exclude_paths=["/admin/*"],
|
||||
max_discovery_depth=3,
|
||||
sitemap="include",
|
||||
limit=100,
|
||||
crawl_entire_domain=True,
|
||||
allow_external_links=False,
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Check basic fields
|
||||
assert data["url"] == "https://example.com"
|
||||
assert data["prompt"] == "Extract all blog posts and documentation"
|
||||
|
||||
# Check all CrawlOptions fields
|
||||
assert "includePaths" in data
|
||||
assert data["includePaths"] == ["/blog/*", "/docs/*"]
|
||||
assert "excludePaths" in data
|
||||
assert data["excludePaths"] == ["/admin/*"]
|
||||
assert "maxDiscoveryDepth" in data
|
||||
assert data["maxDiscoveryDepth"] == 3
|
||||
assert "sitemap" in data
|
||||
assert data["sitemap"] == "include"
|
||||
assert "limit" in data
|
||||
assert data["limit"] == 100
|
||||
assert "crawlEntireDomain" in data
|
||||
assert data["crawlEntireDomain"] is True
|
||||
assert "allowExternalLinks" in data
|
||||
assert data["allowExternalLinks"] is False
|
||||
|
||||
# Check nested scrape options
|
||||
assert "scrapeOptions" in data
|
||||
scrape_data = data["scrapeOptions"]
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert scrape_data["onlyMainContent"] is False
|
||||
assert "mobile" in scrape_data
|
||||
assert scrape_data["mobile"] is True
|
||||
|
||||
def test_none_values_handling(self):
|
||||
"""Test that None values are handled correctly."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
prompt=None,
|
||||
limit=None,
|
||||
scrape_options=None
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Only the required field should be present
|
||||
assert "url" in data
|
||||
assert len(data) == 1 # Only url should be present
|
||||
|
||||
def test_prompt_parameter(self):
|
||||
"""Test that prompt parameter is included when provided."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts"
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
assert "url" in data
|
||||
assert "prompt" in data
|
||||
assert data["prompt"] == "Extract all blog posts"
|
||||
|
||||
def test_empty_options(self):
|
||||
"""Test that empty options are handled correctly."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com"
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Should only have the required url field
|
||||
assert "url" in data
|
||||
assert len(data) == 1 # Only url should be present
|
||||
|
||||
def test_validation_integration(self):
|
||||
"""Test that validation is called during preparation."""
|
||||
# This should raise an error due to validation
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
request = CrawlRequest(url="")
|
||||
_prepare_crawl_request(request)
|
||||
|
||||
# This should raise an error due to validation
|
||||
with pytest.raises(ValueError, match="Limit must be positive"):
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=0
|
||||
)
|
||||
_prepare_crawl_request(request)
|
||||
|
||||
def test_scrape_options_shared_function_integration(self):
|
||||
"""Test that the shared prepare_scrape_options function is being used."""
|
||||
# Test with all snake_case fields to ensure conversion
|
||||
scrape_opts = ScrapeOptions(
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
wait_for=2000,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
|
||||
# Check that scrapeOptions is present and converted
|
||||
assert "scrapeOptions" in data
|
||||
scrape_data = data["scrapeOptions"]
|
||||
|
||||
# Check all conversions are working
|
||||
assert "includeTags" in scrape_data
|
||||
assert "excludeTags" in scrape_data
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert "waitFor" in scrape_data
|
||||
assert "skipTlsVerification" in scrape_data
|
||||
assert "removeBase64Images" in scrape_data
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "include_tags" not in scrape_data
|
||||
assert "exclude_tags" not in scrape_data
|
||||
assert "only_main_content" not in scrape_data
|
||||
assert "wait_for" not in scrape_data
|
||||
assert "skip_tls_verification" not in scrape_data
|
||||
assert "remove_base64_images" not in scrape_data
|
||||
assert "raw_html" not in scrape_data
|
||||
assert "screenshot_full_page" not in scrape_data
|
||||
@@ -0,0 +1,107 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import CrawlRequest, ScrapeOptions
|
||||
from firecrawl.v2.methods.crawl import _validate_crawl_request
|
||||
|
||||
|
||||
class TestCrawlRequestValidation:
|
||||
"""Unit tests for crawl request validation."""
|
||||
|
||||
def test_validate_empty_url(self):
|
||||
"""Test validation with empty URL."""
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
request = CrawlRequest(url="")
|
||||
_validate_crawl_request(request)
|
||||
|
||||
def test_validate_whitespace_url(self):
|
||||
"""Test validation with whitespace-only URL."""
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
request = CrawlRequest(url=" ")
|
||||
_validate_crawl_request(request)
|
||||
|
||||
def test_validate_valid_url(self):
|
||||
"""Test validation with valid URL."""
|
||||
request = CrawlRequest(url="https://example.com")
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_invalid_limit(self):
|
||||
"""Test validation with invalid limit."""
|
||||
with pytest.raises(ValueError, match="Limit must be positive"):
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=0
|
||||
)
|
||||
_validate_crawl_request(request)
|
||||
|
||||
def test_validate_negative_limit(self):
|
||||
"""Test validation with negative limit."""
|
||||
with pytest.raises(ValueError, match="Limit must be positive"):
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=-5
|
||||
)
|
||||
_validate_crawl_request(request)
|
||||
|
||||
def test_validate_valid_limit(self):
|
||||
"""Test validation with valid limit."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=10
|
||||
)
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_with_prompt(self):
|
||||
"""Test validation with prompt."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts"
|
||||
)
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_with_prompt_and_options(self):
|
||||
"""Test validation with prompt and options."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
prompt="Extract all blog posts",
|
||||
limit=10
|
||||
)
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_none_options(self):
|
||||
"""Test validation with None options."""
|
||||
request = CrawlRequest(url="https://example.com")
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_complex_options(self):
|
||||
"""Test validation with complex options."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
only_main_content=False,
|
||||
mobile=True
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
limit=50,
|
||||
max_discovery_depth=3,
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
def test_validate_scrape_options_integration(self):
|
||||
"""Test that scrape_options validation is integrated."""
|
||||
# Test with valid scrape options
|
||||
scrape_opts = ScrapeOptions(formats=["markdown"], timeout=30000)
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
_validate_crawl_request(request) # Should not raise
|
||||
|
||||
# Test with invalid scrape options (should raise error)
|
||||
invalid_scrape_opts = ScrapeOptions(timeout=-1000)
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=invalid_scrape_opts
|
||||
)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
_validate_crawl_request(request)
|
||||
@@ -0,0 +1,54 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import MapOptions
|
||||
from firecrawl.v2.methods.map import _prepare_map_request
|
||||
|
||||
|
||||
class TestMapRequestPreparation:
|
||||
"""Unit tests for map request preparation."""
|
||||
|
||||
def test_basic_request_preparation(self):
|
||||
data = _prepare_map_request("https://example.com")
|
||||
assert data["url"] == "https://example.com"
|
||||
# Default sitemap handling should be "include" when no flags provided
|
||||
assert "sitemap" not in data # we only send when options provided
|
||||
|
||||
def test_sitemap_transformations(self):
|
||||
# sitemap -> "only"
|
||||
opts = MapOptions(sitemap="only")
|
||||
data = _prepare_map_request("https://example.com", opts)
|
||||
assert data["sitemap"] == "only"
|
||||
|
||||
# sitemap -> "skip"
|
||||
opts = MapOptions(sitemap="skip")
|
||||
data = _prepare_map_request("https://example.com", opts)
|
||||
assert data["sitemap"] == "skip"
|
||||
|
||||
# default when options present but sitemap left as default -> include
|
||||
opts = MapOptions(search="docs")
|
||||
data = _prepare_map_request("https://example.com", opts)
|
||||
assert data["sitemap"] == "include"
|
||||
|
||||
def test_field_conversions(self):
|
||||
opts = MapOptions(
|
||||
search="docs",
|
||||
include_subdomains=True,
|
||||
limit=25,
|
||||
sitemap="only",
|
||||
timeout=15000,
|
||||
integration=" _unit-test ",
|
||||
)
|
||||
data = _prepare_map_request("https://example.com", opts)
|
||||
|
||||
assert data["url"] == "https://example.com"
|
||||
assert data["search"] == "docs"
|
||||
assert data["includeSubdomains"] is True
|
||||
assert data["limit"] == 25
|
||||
assert data["sitemap"] == "only"
|
||||
assert data["timeout"] == 15000
|
||||
assert data["integration"] == "_unit-test"
|
||||
|
||||
def test_invalid_url(self):
|
||||
with pytest.raises(ValueError):
|
||||
_prepare_map_request("")
|
||||
with pytest.raises(ValueError):
|
||||
_prepare_map_request(" ")
|
||||
@@ -0,0 +1,225 @@
|
||||
import pytest
|
||||
from unittest.mock import Mock
|
||||
from firecrawl.v2.methods.scrape import scrape
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
|
||||
|
||||
class TestMenuFormat:
|
||||
"""Unit tests for menu format support."""
|
||||
|
||||
def test_scrape_with_menu_format_returns_menu_data(self):
|
||||
"""Test that scraping with menu format returns menu data."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example Menu",
|
||||
"menu": {
|
||||
"isMenu": True,
|
||||
"confidence": 0.95,
|
||||
"currency": "USD",
|
||||
"sourceUrl": "https://example.com/menu",
|
||||
"merchant": {"name": "Acme Diner", "type": "restaurant"},
|
||||
"sections": [
|
||||
{
|
||||
"id": "mains",
|
||||
"name": "Mains",
|
||||
"description": "Hearty plates",
|
||||
"items": [
|
||||
{
|
||||
"id": "burger",
|
||||
"name": "Classic Burger",
|
||||
"description": "Beef patty with cheese",
|
||||
"images": [{"url": "https://example.com/burger.jpg", "alt": "Burger"}],
|
||||
"price": {"amount": 12.5, "currency": "USD", "formatted": "$12.50"},
|
||||
"availability": {"inStock": True, "text": "Available"},
|
||||
"dietary": ["contains-gluten"],
|
||||
"calories": 800,
|
||||
"optionGroups": [],
|
||||
"identifiers": {"merchantItemId": "ITEM-1"},
|
||||
"url": "https://example.com/menu#burger",
|
||||
"sourceUrl": "https://example.com/menu",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["menu"]))
|
||||
|
||||
assert result.menu is not None
|
||||
assert result.menu.is_menu is True
|
||||
assert result.menu.confidence == 0.95
|
||||
assert result.menu.currency == "USD"
|
||||
assert result.menu.source_url == "https://example.com/menu"
|
||||
assert result.menu.merchant.name == "Acme Diner"
|
||||
assert result.menu.merchant.type == "restaurant"
|
||||
section = result.menu.sections[0]
|
||||
assert section.name == "Mains"
|
||||
item = section.items[0]
|
||||
assert item.name == "Classic Burger"
|
||||
assert item.price.amount == 12.5
|
||||
assert item.price.currency == "USD"
|
||||
assert item.availability.in_stock is True
|
||||
assert item.availability.text == "Available"
|
||||
assert item.images[0].url == "https://example.com/burger.jpg"
|
||||
assert item.dietary[0] == "contains-gluten"
|
||||
assert item.calories == 800
|
||||
assert item.identifiers.merchant_item_id == "ITEM-1"
|
||||
assert item.source_url == "https://example.com/menu"
|
||||
|
||||
def test_scrape_with_menu_and_markdown_formats_returns_both(self):
|
||||
"""Test that scraping with both menu and markdown formats returns both."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example Content",
|
||||
"menu": {
|
||||
"isMenu": True,
|
||||
"confidence": 0.8,
|
||||
"sourceUrl": "https://example.com/cafe",
|
||||
"merchant": {"name": "Cafe Acme"},
|
||||
"sections": [
|
||||
{
|
||||
"id": "drinks",
|
||||
"name": "Drinks",
|
||||
"items": [
|
||||
{
|
||||
"id": "coffee",
|
||||
"name": "Coffee",
|
||||
"images": [],
|
||||
"price": {"amount": 3.5, "currency": "USD"},
|
||||
"availability": {"inStock": True},
|
||||
"dietary": [],
|
||||
"optionGroups": [],
|
||||
"identifiers": {},
|
||||
"sourceUrl": "https://example.com/cafe",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "menu"]))
|
||||
|
||||
assert result.markdown == "# Example Content"
|
||||
assert result.menu is not None
|
||||
assert result.menu.merchant.name == "Cafe Acme"
|
||||
assert result.menu.sections[0].items[0].price.amount == 3.5
|
||||
|
||||
def test_scrape_without_menu_format_does_not_return_menu(self):
|
||||
"""Test that scraping without menu format does not return menu."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example"
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))
|
||||
|
||||
assert result.markdown == "# Example"
|
||||
assert result.menu is None
|
||||
|
||||
def test_non_menu_page_yields_warning_and_no_menu(self):
|
||||
"""Test that a non-menu page scraped with menu format yields a warning and no menu."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Blog Post",
|
||||
"warning": "No menu found on this page."
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["menu"]))
|
||||
|
||||
assert result.menu is None
|
||||
assert "No menu found" in result.warning
|
||||
|
||||
def test_menu_format_with_multiple_sections_and_items(self):
|
||||
"""Test menu format with multiple sections and items, including camelCase aliasing."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"menu": {
|
||||
"isMenu": True,
|
||||
"confidence": 0.9,
|
||||
"sourceUrl": "https://example.com/menu",
|
||||
"merchant": {"name": "Acme Bistro", "type": "restaurant", "location": {"city": "Springfield"}},
|
||||
"sections": [
|
||||
{
|
||||
"id": "starters",
|
||||
"name": "Starters",
|
||||
"items": [
|
||||
{
|
||||
"id": "soup",
|
||||
"name": "Tomato Soup",
|
||||
"images": [],
|
||||
"price": {"amount": 6.0, "currency": "USD"},
|
||||
"availability": {"inStock": True},
|
||||
"dietary": ["vegetarian"],
|
||||
"optionGroups": [],
|
||||
"identifiers": {},
|
||||
"sourceUrl": "https://example.com/menu",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "desserts",
|
||||
"name": "Desserts",
|
||||
"items": [
|
||||
{
|
||||
"id": "cake",
|
||||
"name": "Chocolate Cake",
|
||||
"images": [{"url": "https://example.com/cake.jpg"}],
|
||||
"availability": {"inStock": False, "text": "Sold out"},
|
||||
"dietary": [],
|
||||
"optionGroups": [],
|
||||
"identifiers": {},
|
||||
"sourceUrl": "https://example.com/menu",
|
||||
}
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["menu"]))
|
||||
|
||||
assert result.menu is not None
|
||||
assert len(result.menu.sections) == 2
|
||||
assert result.menu.merchant.location == {"city": "Springfield"}
|
||||
assert result.menu.sections[0].items[0].dietary[0] == "vegetarian"
|
||||
assert result.menu.sections[1].name == "Desserts"
|
||||
assert result.menu.sections[1].items[0].images[0].url == "https://example.com/cake.jpg"
|
||||
assert result.menu.sections[1].items[0].availability.in_stock is False
|
||||
assert result.menu.sections[1].items[0].availability.text == "Sold out"
|
||||
@@ -0,0 +1,895 @@
|
||||
"""
|
||||
Unit tests for Firecrawl v2 pagination functionality.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from unittest.mock import Mock, patch, AsyncMock
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from firecrawl.v2.types import (
|
||||
PaginationConfig,
|
||||
CrawlJob,
|
||||
BatchScrapeJob,
|
||||
Document,
|
||||
DocumentMetadata
|
||||
)
|
||||
from firecrawl.v2.methods.crawl import get_crawl_status, get_crawl_status_page, _fetch_all_pages
|
||||
from firecrawl.v2.methods.batch import get_batch_scrape_status, get_batch_scrape_status_page, _fetch_all_batch_pages
|
||||
from firecrawl.v2.methods.aio.crawl import (
|
||||
get_crawl_status as get_crawl_status_async,
|
||||
get_crawl_status_page as get_crawl_status_page_async,
|
||||
_fetch_all_pages_async,
|
||||
)
|
||||
from firecrawl.v2.methods.aio.batch import (
|
||||
get_batch_scrape_status as get_batch_scrape_status_async,
|
||||
get_batch_scrape_status_page as get_batch_scrape_status_page_async,
|
||||
_fetch_all_batch_pages_async,
|
||||
)
|
||||
|
||||
|
||||
class TestPaginationConfig:
|
||||
"""Test PaginationConfig model."""
|
||||
|
||||
def test_default_values(self):
|
||||
"""Test default values for PaginationConfig."""
|
||||
config = PaginationConfig()
|
||||
assert config.auto_paginate is True
|
||||
assert config.max_pages is None
|
||||
assert config.max_results is None
|
||||
assert config.max_wait_time is None
|
||||
|
||||
def test_custom_values(self):
|
||||
"""Test custom values for PaginationConfig."""
|
||||
config = PaginationConfig(
|
||||
auto_paginate=False,
|
||||
max_pages=5,
|
||||
max_results=100,
|
||||
max_wait_time=30
|
||||
)
|
||||
assert config.auto_paginate is False
|
||||
assert config.max_pages == 5
|
||||
assert config.max_results == 100
|
||||
assert config.max_wait_time == 30
|
||||
|
||||
|
||||
class TestCrawlPagination:
|
||||
"""Test crawl pagination functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.mock_client = Mock()
|
||||
self.job_id = "test-crawl-123"
|
||||
|
||||
# Sample document data
|
||||
self.sample_doc = {
|
||||
"url": "https://example.com",
|
||||
"markdown": "# Test Content",
|
||||
"metadata": {
|
||||
"title": "Test Page",
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
|
||||
def test_get_crawl_status_no_pagination(self):
|
||||
"""Test get_crawl_status with auto_paginate=False."""
|
||||
# Mock response with next URL
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 10,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
# Test with auto_paginate=False
|
||||
pagination_config = PaginationConfig(auto_paginate=False)
|
||||
result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
|
||||
assert len(result.data) == 1
|
||||
assert isinstance(result.data[0], Document)
|
||||
|
||||
def test_get_crawl_status_propagates_request_timeout(self):
|
||||
"""Ensure request_timeout is forwarded to the HTTP client."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
timeout_seconds = 5.5
|
||||
import firecrawl.v2.methods.crawl as crawl_module
|
||||
|
||||
assert crawl_module.__file__.endswith("firecrawl/v2/methods/crawl.py")
|
||||
assert crawl_module.get_crawl_status.__kwdefaults__ is not None
|
||||
assert "request_timeout" in crawl_module.get_crawl_status.__kwdefaults__
|
||||
result = get_crawl_status(
|
||||
self.mock_client,
|
||||
self.job_id,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_called_with(
|
||||
f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
|
||||
)
|
||||
|
||||
def test_get_crawl_status_page(self):
|
||||
"""Test get_crawl_status_page returns a single page."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3",
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
|
||||
|
||||
result = get_crawl_status_page(self.mock_client, next_url)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3"
|
||||
assert len(result.data) == 1
|
||||
self.mock_client.get.assert_called_with(next_url, timeout=None)
|
||||
|
||||
def test_get_crawl_status_page_propagates_request_timeout(self):
|
||||
"""Ensure request_timeout is forwarded to crawl status page requests."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
next_url = "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
|
||||
timeout_seconds = 4.2
|
||||
result = get_crawl_status_page(
|
||||
self.mock_client,
|
||||
next_url,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)
|
||||
|
||||
def test_get_crawl_status_with_pagination(self):
|
||||
"""Test get_crawl_status with auto_paginate=True."""
|
||||
# Mock first page response
|
||||
mock_response1 = Mock()
|
||||
mock_response1.ok = True
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
# Mock second page response
|
||||
mock_response2 = Mock()
|
||||
mock_response2.ok = True
|
||||
mock_response2.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 20,
|
||||
"total": 20,
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Test with auto_paginate=True
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next is None # Should be None when auto_paginate=True
|
||||
assert len(result.data) == 2
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
def test_get_crawl_status_max_pages_limit(self):
|
||||
"""Test get_crawl_status with max_pages limit."""
|
||||
# Mock responses for multiple pages
|
||||
mock_responses = []
|
||||
for i in range(5): # 5 pages available
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": (i + 1) * 10,
|
||||
"total": 50,
|
||||
"creditsUsed": (i + 1) * 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": f"https://api.firecrawl.dev/v2/crawl/test-crawl-123?page={i+2}" if i < 4 else None,
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
mock_responses.append(mock_response)
|
||||
|
||||
self.mock_client.get.side_effect = mock_responses
|
||||
|
||||
# Test with max_pages=3
|
||||
pagination_config = PaginationConfig(auto_paginate=True, max_pages=3)
|
||||
result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert len(result.data) == 4 # 1 initial + 3 from pages
|
||||
assert self.mock_client.get.call_count == 4 # 1 initial + 3 pagination calls
|
||||
|
||||
def test_get_crawl_status_max_results_limit(self):
|
||||
"""Test get_crawl_status with max_results limit."""
|
||||
# Mock responses with multiple documents per page
|
||||
mock_response1 = Mock()
|
||||
mock_response1.ok = True
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
|
||||
"data": [self.sample_doc, self.sample_doc, self.sample_doc] # 3 docs
|
||||
}
|
||||
|
||||
mock_response2 = Mock()
|
||||
mock_response2.ok = True
|
||||
mock_response2.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 20,
|
||||
"total": 20,
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=3",
|
||||
"data": [self.sample_doc, self.sample_doc] # 2 more docs
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Test with max_results=4
|
||||
pagination_config = PaginationConfig(auto_paginate=True, max_results=4)
|
||||
result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert len(result.data) == 4 # Should stop at 4 results
|
||||
assert self.mock_client.get.call_count == 2 # Should fetch 2 pages
|
||||
|
||||
def test_get_crawl_status_max_wait_time_limit(self):
|
||||
"""Test get_crawl_status with max_wait_time limit."""
|
||||
# Mock slow response
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
# Test with max_wait_time=1 second
|
||||
pagination_config = PaginationConfig(auto_paginate=True, max_wait_time=1)
|
||||
|
||||
with patch('firecrawl.v2.methods.crawl.time.monotonic', side_effect=[0, 2]): # Simulate 2 seconds elapsed
|
||||
result = get_crawl_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert len(result.data) == 1 # Should stop due to timeout
|
||||
assert self.mock_client.get.call_count == 1
|
||||
|
||||
def test_fetch_all_pages_error_handling(self):
|
||||
"""Test _fetch_all_pages with API errors."""
|
||||
# Mock first page success, second page error
|
||||
mock_response1 = Mock()
|
||||
mock_response1.ok = True
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"data": [self.sample_doc],
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2"
|
||||
}
|
||||
|
||||
mock_response2 = Mock()
|
||||
mock_response2.ok = False
|
||||
mock_response2.status_code = 500
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Should continue with what we have
|
||||
result = _fetch_all_pages(self.mock_client, "https://api.firecrawl.dev/v2/crawl/test-crawl-123?page=2", [], None)
|
||||
|
||||
assert len(result) == 1 # Should have the first page data
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
|
||||
class TestBatchScrapePagination:
|
||||
"""Test batch scrape pagination functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.mock_client = Mock()
|
||||
self.job_id = "test-batch-123"
|
||||
|
||||
# Sample document data
|
||||
self.sample_doc = {
|
||||
"url": "https://example.com",
|
||||
"markdown": "# Test Content",
|
||||
"metadata": {
|
||||
"title": "Test Page",
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
|
||||
def test_get_batch_scrape_status_no_pagination(self):
|
||||
"""Test get_batch_scrape_status with auto_paginate=False."""
|
||||
# Mock response with next URL
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 10,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
# Test with auto_paginate=False
|
||||
pagination_config = PaginationConfig(auto_paginate=False)
|
||||
result = get_batch_scrape_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
|
||||
assert len(result.data) == 1
|
||||
assert isinstance(result.data[0], Document)
|
||||
|
||||
def test_get_batch_scrape_status_page(self):
|
||||
"""Test get_batch_scrape_status_page returns a single page."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3",
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
|
||||
|
||||
result = get_batch_scrape_status_page(self.mock_client, next_url)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=3"
|
||||
assert len(result.data) == 1
|
||||
self.mock_client.get.assert_called_with(next_url, timeout=None)
|
||||
|
||||
def test_get_batch_scrape_status_page_propagates_request_timeout(self):
|
||||
"""Ensure request_timeout is forwarded to batch status page requests."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2"
|
||||
timeout_seconds = 2.7
|
||||
result = get_batch_scrape_status_page(
|
||||
self.mock_client,
|
||||
next_url,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_called_with(next_url, timeout=timeout_seconds)
|
||||
|
||||
def test_get_batch_scrape_status_with_pagination(self):
|
||||
"""Test get_batch_scrape_status with auto_paginate=True."""
|
||||
# Mock first page response
|
||||
mock_response1 = Mock()
|
||||
mock_response1.ok = True
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
# Mock second page response
|
||||
mock_response2 = Mock()
|
||||
mock_response2.ok = True
|
||||
mock_response2.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 20,
|
||||
"total": 20,
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Test with auto_paginate=True
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = get_batch_scrape_status(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next is None # Should be None when auto_paginate=True
|
||||
assert len(result.data) == 2
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
def test_fetch_all_batch_pages_limits(self):
|
||||
"""Test _fetch_all_batch_pages with various limits."""
|
||||
# Mock responses for multiple pages
|
||||
mock_responses = []
|
||||
for i in range(5): # 5 pages available
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": [self.sample_doc, self.sample_doc], # 2 docs per page
|
||||
"next": f"https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page={i+2}" if i < 4 else None
|
||||
}
|
||||
mock_responses.append(mock_response)
|
||||
|
||||
self.mock_client.get.side_effect = mock_responses
|
||||
|
||||
# Test with max_pages=2, max_results=4 (total docs we want)
|
||||
pagination_config = PaginationConfig(max_pages=2, max_results=4)
|
||||
result = _fetch_all_batch_pages(
|
||||
self.mock_client,
|
||||
"https://api.firecrawl.dev/v2/batch/scrape/test-batch-123?page=2",
|
||||
[Document(**self.sample_doc)], # 1 initial doc
|
||||
pagination_config
|
||||
)
|
||||
|
||||
# Should have 1 initial + 3 from pages (limited by max_results=4)
|
||||
assert len(result) == 4
|
||||
assert self.mock_client.get.call_count == 2 # Should fetch 2 pages
|
||||
|
||||
|
||||
class TestAsyncPagination:
|
||||
"""Test async pagination functionality."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.mock_client = AsyncMock()
|
||||
self.job_id = "test-async-123"
|
||||
|
||||
# Sample document data
|
||||
self.sample_doc = {
|
||||
"url": "https://example.com",
|
||||
"markdown": "# Test Content",
|
||||
"metadata": {
|
||||
"title": "Test Page",
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_crawl_status_async_with_pagination(self):
|
||||
"""Test async get_crawl_status with pagination."""
|
||||
# Mock first page response
|
||||
mock_response1 = Mock()
|
||||
mock_response1.status_code = 200
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
# Mock second page response
|
||||
mock_response2 = Mock()
|
||||
mock_response2.status_code = 200
|
||||
mock_response2.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 20,
|
||||
"total": 20,
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Test with auto_paginate=True
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = await get_crawl_status_async(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next is None
|
||||
assert len(result.data) == 2
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_crawl_status_async_propagates_request_timeout(self):
|
||||
"""Ensure async request_timeout is forwarded to the HTTP client."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
timeout_seconds = 3.3
|
||||
import firecrawl.v2.methods.aio.crawl as crawl_module_async
|
||||
|
||||
assert crawl_module_async.__file__.endswith("firecrawl/v2/methods/aio/crawl.py")
|
||||
assert crawl_module_async.get_crawl_status.__kwdefaults__ is not None
|
||||
assert "request_timeout" in crawl_module_async.get_crawl_status.__kwdefaults__
|
||||
result = await get_crawl_status_async(
|
||||
self.mock_client,
|
||||
self.job_id,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_awaited_with(
|
||||
f"/v2/crawl/{self.job_id}", timeout=timeout_seconds
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_crawl_status_page_async(self):
|
||||
"""Test async get_crawl_status_page returns a single page."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3",
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"
|
||||
|
||||
result = await get_crawl_status_page_async(self.mock_client, next_url)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/crawl/test-async-123?page=3"
|
||||
assert len(result.data) == 1
|
||||
self.mock_client.get.assert_awaited_with(next_url, timeout=None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_crawl_status_page_async_propagates_request_timeout(self):
|
||||
"""Ensure async request_timeout is forwarded to crawl status page requests."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
next_url = "https://api.firecrawl.dev/v2/crawl/test-async-123?page=2"
|
||||
timeout_seconds = 6.1
|
||||
result = await get_crawl_status_page_async(
|
||||
self.mock_client,
|
||||
next_url,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_batch_scrape_status_async_with_pagination(self):
|
||||
"""Test async get_batch_scrape_status with pagination."""
|
||||
# Mock first page response
|
||||
mock_response1 = Mock()
|
||||
mock_response1.status_code = 200
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2",
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
# Mock second page response
|
||||
mock_response2 = Mock()
|
||||
mock_response2.status_code = 200
|
||||
mock_response2.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 20,
|
||||
"total": 20,
|
||||
"creditsUsed": 10,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc]
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Test with auto_paginate=True
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = await get_batch_scrape_status_async(self.mock_client, self.job_id, pagination_config)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next is None
|
||||
assert len(result.data) == 2
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_batch_scrape_status_page_async(self):
|
||||
"""Test async get_batch_scrape_status_page returns a single page."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 10,
|
||||
"total": 20,
|
||||
"creditsUsed": 5,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3",
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"
|
||||
|
||||
result = await get_batch_scrape_status_page_async(self.mock_client, next_url)
|
||||
|
||||
assert result.status == "completed"
|
||||
assert result.next == "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=3"
|
||||
assert len(result.data) == 1
|
||||
self.mock_client.get.assert_awaited_with(next_url, timeout=None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_batch_scrape_status_page_async_propagates_request_timeout(self):
|
||||
"""Ensure async request_timeout is forwarded to batch status page requests."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 1,
|
||||
"total": 1,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": [self.sample_doc],
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
next_url = "https://api.firecrawl.dev/v2/batch/scrape/test-async-123?page=2"
|
||||
timeout_seconds = 4.4
|
||||
result = await get_batch_scrape_status_page_async(
|
||||
self.mock_client,
|
||||
next_url,
|
||||
request_timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
assert result.status == "completed"
|
||||
self.mock_client.get.assert_awaited_with(next_url, timeout=timeout_seconds)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_pages_async_limits(self):
|
||||
"""Test async _fetch_all_pages_async with limits."""
|
||||
# Mock responses for multiple pages
|
||||
mock_responses = []
|
||||
for i in range(3): # 3 pages available
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": [self.sample_doc],
|
||||
"next": f"https://api.firecrawl.dev/v2/crawl/test-async-123?page={i+2}" if i < 2 else None
|
||||
}
|
||||
mock_responses.append(mock_response)
|
||||
|
||||
self.mock_client.get.side_effect = mock_responses
|
||||
|
||||
# Test with max_pages=2
|
||||
pagination_config = PaginationConfig(max_pages=2)
|
||||
result = await _fetch_all_pages_async(
|
||||
self.mock_client,
|
||||
"https://api.firecrawl.dev/v2/crawl/test-async-123?page=2",
|
||||
[Document(**self.sample_doc)], # 1 initial doc
|
||||
pagination_config
|
||||
)
|
||||
|
||||
assert len(result) == 3 # 1 initial + 2 from pages
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
|
||||
class TestPaginationEdgeCases:
|
||||
"""Test pagination edge cases and error conditions."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Set up test fixtures."""
|
||||
self.mock_client = Mock()
|
||||
self.sample_doc = {
|
||||
"url": "https://example.com",
|
||||
"markdown": "# Test Content",
|
||||
"metadata": {"title": "Test Page"}
|
||||
}
|
||||
|
||||
def test_pagination_with_empty_data(self):
|
||||
"""Test pagination when API returns empty data."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 0,
|
||||
"total": 0,
|
||||
"creditsUsed": 0,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": []
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = get_crawl_status(self.mock_client, "test-123", pagination_config)
|
||||
|
||||
assert len(result.data) == 0
|
||||
assert result.next is None
|
||||
|
||||
def test_pagination_with_string_data(self):
|
||||
"""Test pagination when API returns string data (should be skipped)."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"status": "completed",
|
||||
"completed": 2,
|
||||
"total": 2,
|
||||
"creditsUsed": 1,
|
||||
"expiresAt": "2024-01-01T00:00:00Z",
|
||||
"next": None,
|
||||
"data": ["https://example.com", self.sample_doc] # String + dict
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
result = get_crawl_status(self.mock_client, "test-123", pagination_config)
|
||||
|
||||
assert len(result.data) == 1 # Only the dict should be processed
|
||||
assert isinstance(result.data[0], Document)
|
||||
|
||||
def test_pagination_with_failed_response(self):
|
||||
"""Test pagination when API response indicates failure."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": False,
|
||||
"error": "Job not found"
|
||||
}
|
||||
|
||||
self.mock_client.get.return_value = mock_response
|
||||
|
||||
pagination_config = PaginationConfig(auto_paginate=True)
|
||||
|
||||
with pytest.raises(Exception, match="Job not found"):
|
||||
get_crawl_status(self.mock_client, "test-123", pagination_config)
|
||||
|
||||
def test_pagination_with_unsuccessful_page(self):
|
||||
"""Test pagination when a subsequent page is unsuccessful."""
|
||||
# Mock first page success
|
||||
mock_response1 = Mock()
|
||||
mock_response1.ok = True
|
||||
mock_response1.json.return_value = {
|
||||
"success": True,
|
||||
"data": [self.sample_doc],
|
||||
"next": "https://api.firecrawl.dev/v2/crawl/test-123?page=2"
|
||||
}
|
||||
|
||||
# Mock second page failure
|
||||
mock_response2 = Mock()
|
||||
mock_response2.ok = True
|
||||
mock_response2.json.return_value = {
|
||||
"success": False,
|
||||
"error": "Page not found"
|
||||
}
|
||||
|
||||
self.mock_client.get.side_effect = [mock_response1, mock_response2]
|
||||
|
||||
# Should continue with what we have
|
||||
result = _fetch_all_pages(self.mock_client, "https://api.firecrawl.dev/v2/crawl/test-123?page=2", [], None)
|
||||
|
||||
assert len(result) == 1 # Should have the first page data
|
||||
assert self.mock_client.get.call_count == 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.types import ParseOptions
|
||||
from firecrawl.v2.methods.parse import _prepare_parse_request
|
||||
|
||||
|
||||
class TestParseRequestPreparation:
|
||||
def test_prepare_parse_request_from_bytes(self):
|
||||
options = ParseOptions(
|
||||
formats=["markdown"],
|
||||
only_main_content=False,
|
||||
integration=" parse-unit ",
|
||||
)
|
||||
|
||||
fields, files = _prepare_parse_request(
|
||||
b"<html><body><h1>Parse</h1></body></html>",
|
||||
options,
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
assert "options" in fields
|
||||
payload = json.loads(fields["options"])
|
||||
assert payload["formats"] == ["markdown"]
|
||||
assert payload["onlyMainContent"] is False
|
||||
assert payload["integration"] == "parse-unit"
|
||||
assert payload["origin"].startswith("python-sdk@")
|
||||
assert "maxAge" not in payload
|
||||
assert "storeInCache" not in payload
|
||||
assert "lockdown" not in payload
|
||||
|
||||
assert "file" in files
|
||||
filename, file_bytes, mime_type = files["file"]
|
||||
assert filename == "upload.html"
|
||||
assert file_bytes.startswith(b"<html>")
|
||||
assert mime_type == "text/html"
|
||||
|
||||
def test_prepare_parse_request_from_file_path(self, tmp_path):
|
||||
file_path = tmp_path / "sample.html"
|
||||
file_path.write_text("<html><body>Path Upload</body></html>")
|
||||
|
||||
fields, files = _prepare_parse_request(str(file_path))
|
||||
|
||||
payload = json.loads(fields["options"])
|
||||
assert payload["origin"].startswith("python-sdk@")
|
||||
assert payload.get("formats") is None
|
||||
|
||||
filename, file_bytes, mime_type = files["file"]
|
||||
assert filename == "sample.html"
|
||||
assert b"Path Upload" in file_bytes
|
||||
assert mime_type == "text/html"
|
||||
|
||||
def test_prepare_parse_request_rejects_missing_path(self, tmp_path):
|
||||
missing_file = tmp_path / "missing-upload-file.html"
|
||||
with pytest.raises(ValueError, match="File path does not exist"):
|
||||
_prepare_parse_request(str(missing_file))
|
||||
|
||||
def test_prepare_parse_request_rejects_change_tracking_format(self):
|
||||
options = ParseOptions(formats=["markdown", "changeTracking"])
|
||||
with pytest.raises(ValueError, match="do not support change tracking"):
|
||||
_prepare_parse_request(
|
||||
b"<html><body><h1>Parse</h1></body></html>",
|
||||
options,
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
def test_prepare_parse_request_rejects_video_format(self):
|
||||
options = ParseOptions(formats=["video"])
|
||||
with pytest.raises(ValueError, match="do not support video output"):
|
||||
_prepare_parse_request(
|
||||
b"<html><body><h1>Parse</h1></body></html>",
|
||||
options,
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
def test_prepare_parse_request_strips_lockdown(self):
|
||||
options = ParseOptions(formats=["markdown"], lockdown=True)
|
||||
fields, _ = _prepare_parse_request(
|
||||
b"<html><body><h1>Parse</h1></body></html>",
|
||||
options,
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
payload = json.loads(fields["options"])
|
||||
assert "lockdown" not in payload
|
||||
|
||||
def test_prepare_parse_request_strips_min_age(self):
|
||||
options = ParseOptions(formats=["markdown"], min_age=1000)
|
||||
fields, _ = _prepare_parse_request(
|
||||
b"<html><body><h1>Parse</h1></body></html>",
|
||||
options,
|
||||
filename="upload.html",
|
||||
content_type="text/html",
|
||||
)
|
||||
|
||||
payload = json.loads(fields["options"])
|
||||
assert "minAge" not in payload
|
||||
assert "min_age" not in payload
|
||||
@@ -0,0 +1,171 @@
|
||||
import pytest
|
||||
from unittest.mock import Mock
|
||||
from firecrawl.v2.methods.scrape import scrape
|
||||
from firecrawl.v2.types import ScrapeOptions
|
||||
|
||||
|
||||
class TestProductFormat:
|
||||
"""Unit tests for product format support."""
|
||||
|
||||
def test_scrape_with_product_format_returns_product_data(self):
|
||||
"""Test that scraping with product format returns product data."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example Product",
|
||||
"product": {
|
||||
"title": "Acme Running Shoe",
|
||||
"brand": "Acme",
|
||||
"category": "Footwear",
|
||||
"url": "https://example.com/shoe",
|
||||
"description": "A lightweight running shoe.",
|
||||
"variants": [
|
||||
{
|
||||
"id": "default",
|
||||
"values": {"size": "10"},
|
||||
"price": {"amount": 89.99, "currency": "USD", "formatted": "$89.99"},
|
||||
"sale": {"originalPrice": {"amount": 129.99, "currency": "USD"}},
|
||||
"availability": {"inStock": True, "text": "In stock"},
|
||||
"images": [{"url": "https://example.com/shoe.jpg", "alt": "Acme shoe"}],
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))
|
||||
|
||||
assert result.product is not None
|
||||
assert result.product.title == "Acme Running Shoe"
|
||||
assert result.product.brand == "Acme"
|
||||
assert result.product.category == "Footwear"
|
||||
variant = result.product.variants[0]
|
||||
assert variant.price.amount == 89.99
|
||||
assert variant.price.currency == "USD"
|
||||
assert variant.sale.original_price.amount == 129.99
|
||||
assert variant.sale.original_price.currency == "USD"
|
||||
assert variant.availability.in_stock is True
|
||||
assert variant.availability.text == "In stock"
|
||||
assert variant.images[0].url == "https://example.com/shoe.jpg"
|
||||
|
||||
def test_scrape_with_product_and_markdown_formats_returns_both(self):
|
||||
"""Test that scraping with both product and markdown formats returns both."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example Content",
|
||||
"product": {
|
||||
"title": "Acme Mug",
|
||||
"url": "https://example.com/mug",
|
||||
"variants": [
|
||||
{
|
||||
"price": {"amount": 12.5, "currency": "USD"},
|
||||
"availability": {"inStock": True}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown", "product"]))
|
||||
|
||||
assert result.markdown == "# Example Content"
|
||||
assert result.product is not None
|
||||
assert result.product.title == "Acme Mug"
|
||||
assert result.product.variants[0].price.amount == 12.5
|
||||
|
||||
def test_scrape_without_product_format_does_not_return_product(self):
|
||||
"""Test that scraping without product format does not return product."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Example"
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["markdown"]))
|
||||
|
||||
assert result.markdown == "# Example"
|
||||
assert result.product is None
|
||||
|
||||
def test_non_product_page_yields_warning_and_no_product(self):
|
||||
"""Test that a non-product page scraped with product format yields a warning and no product."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"markdown": "# Blog Post",
|
||||
"warning": "No product found on this page."
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))
|
||||
|
||||
assert result.product is None
|
||||
assert "No product found" in result.warning
|
||||
|
||||
def test_product_format_with_variants_populated(self):
|
||||
"""Test product format with variants populated, including camelCase aliasing."""
|
||||
mock_response = Mock()
|
||||
mock_response.ok = True
|
||||
mock_response.json.return_value = {
|
||||
"success": True,
|
||||
"data": {
|
||||
"product": {
|
||||
"title": "Acme T-Shirt",
|
||||
"brand": "Acme",
|
||||
"url": "https://example.com/tshirt",
|
||||
"variants": [
|
||||
{
|
||||
"id": "v1",
|
||||
"sku": "TSHIRT-S-RED",
|
||||
"title": "Small / Red",
|
||||
"values": {"size": "S", "color": "Red"},
|
||||
"price": {"amount": 24.0, "currency": "USD"},
|
||||
"availability": {"inStock": True},
|
||||
"images": [{"url": "https://example.com/tshirt-red.jpg"}]
|
||||
},
|
||||
{
|
||||
"id": "v2",
|
||||
"sku": "TSHIRT-L-BLUE",
|
||||
"title": "Large / Blue",
|
||||
"values": {"size": "L", "color": "Blue"},
|
||||
"availability": {"inStock": False, "text": "Sold out"}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mock_client = Mock()
|
||||
mock_client.post.return_value = mock_response
|
||||
|
||||
result = scrape(mock_client, "https://example.com", ScrapeOptions(formats=["product"]))
|
||||
|
||||
assert result.product is not None
|
||||
assert len(result.product.variants) == 2
|
||||
assert result.product.variants[0].sku == "TSHIRT-S-RED"
|
||||
assert result.product.variants[0].values["color"] == "Red"
|
||||
assert result.product.variants[0].availability.in_stock is True
|
||||
assert result.product.variants[0].images[0].url == "https://example.com/tshirt-red.jpg"
|
||||
assert result.product.variants[1].availability.in_stock is False
|
||||
assert result.product.variants[1].availability.text == "Sold out"
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import ScrapeOptions, Viewport, ScreenshotAction
|
||||
from firecrawl.v2.methods.scrape import (
|
||||
_prepare_scrape_request,
|
||||
interact,
|
||||
stop_interaction,
|
||||
)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, status_code: int, payload):
|
||||
self.status_code = status_code
|
||||
self._payload = payload
|
||||
self.ok = status_code < 400
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
@property
|
||||
def text(self):
|
||||
return str(self._payload)
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, *, post_response: _FakeResponse, delete_response: _FakeResponse):
|
||||
self.post_response = post_response
|
||||
self.delete_response = delete_response
|
||||
self.last_post = None
|
||||
self.last_delete = None
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
self.last_post = (endpoint, payload)
|
||||
return self.post_response
|
||||
|
||||
def delete(self, endpoint):
|
||||
self.last_delete = endpoint
|
||||
return self.delete_response
|
||||
|
||||
|
||||
class TestScrapeRequestPreparation:
|
||||
"""Unit tests for scrape request preparation."""
|
||||
|
||||
def test_basic_request_preparation(self):
|
||||
"""Test basic request preparation with minimal fields."""
|
||||
data = _prepare_scrape_request("https://example.com")
|
||||
|
||||
# Check basic fields
|
||||
assert data["url"] == "https://example.com"
|
||||
|
||||
# Check that no options are present
|
||||
assert "formats" not in data
|
||||
assert "headers" not in data
|
||||
|
||||
def test_scrape_options_conversion(self):
|
||||
"""Test that ScrapeOptions fields are converted to camelCase."""
|
||||
options = ScrapeOptions(
|
||||
formats=["markdown", "html"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
|
||||
data = _prepare_scrape_request("https://example.com", options)
|
||||
|
||||
# Check basic field
|
||||
assert data["url"] == "https://example.com"
|
||||
|
||||
# Check snake_case to camelCase conversions
|
||||
assert "includeTags" in data
|
||||
assert data["includeTags"] == ["h1", "h2"]
|
||||
assert "excludeTags" in data
|
||||
assert data["excludeTags"] == ["nav"]
|
||||
assert "onlyMainContent" in data
|
||||
assert data["onlyMainContent"] is False
|
||||
assert "waitFor" in data
|
||||
assert data["waitFor"] == 2000
|
||||
assert "skipTlsVerification" in data
|
||||
assert data["skipTlsVerification"] is True
|
||||
assert "removeBase64Images" in data
|
||||
assert data["removeBase64Images"] is False
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "include_tags" not in data
|
||||
assert "exclude_tags" not in data
|
||||
assert "only_main_content" not in data
|
||||
|
||||
def test_actions_conversion(self):
|
||||
"""Test that actions are converted to camelCase."""
|
||||
viewport = Viewport(width=800, height=600)
|
||||
action = ScreenshotAction(full_page=False, quality=80, viewport=viewport)
|
||||
|
||||
options = ScrapeOptions(actions=[action])
|
||||
data = _prepare_scrape_request("https://example.com", options)
|
||||
|
||||
assert "actions" in data
|
||||
assert len(data["actions"]) == 1
|
||||
|
||||
action_data = data["actions"][0]
|
||||
assert action_data["type"] == "screenshot"
|
||||
assert action_data["fullPage"] is False
|
||||
assert action_data["quality"] == 80
|
||||
assert "viewport" in action_data
|
||||
|
||||
def test_none_options_handling(self):
|
||||
"""Test handling of None options."""
|
||||
data = _prepare_scrape_request("https://example.com", None)
|
||||
|
||||
assert data["url"] == "https://example.com"
|
||||
# Should not have any option fields
|
||||
assert len(data) == 1
|
||||
|
||||
def test_empty_url_validation(self):
|
||||
"""Test validation with empty URL."""
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
_prepare_scrape_request("")
|
||||
|
||||
def test_whitespace_url_validation(self):
|
||||
"""Test validation with whitespace-only URL."""
|
||||
with pytest.raises(ValueError, match="URL cannot be empty"):
|
||||
_prepare_scrape_request(" ")
|
||||
|
||||
def test_all_params_including_integration(self):
|
||||
opts = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False,
|
||||
integration=" _unit-test ",
|
||||
)
|
||||
data = _prepare_scrape_request("https://example.com", opts)
|
||||
assert data["integration"] == "_unit-test"
|
||||
|
||||
def test_interact_request_and_response_normalization(self):
|
||||
client = _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"stdout": "ok",
|
||||
"exitCode": 0,
|
||||
},
|
||||
),
|
||||
delete_response=_FakeResponse(200, {"success": True}),
|
||||
)
|
||||
response = interact(
|
||||
client,
|
||||
"job-123",
|
||||
"console.log('ok')",
|
||||
timeout=45,
|
||||
origin="_unit-test",
|
||||
)
|
||||
|
||||
assert client.last_post[0] == "/v2/scrape/job-123/interact"
|
||||
assert client.last_post[1] == {
|
||||
"code": "console.log('ok')",
|
||||
"language": "node",
|
||||
"timeout": 45,
|
||||
"origin": "_unit-test",
|
||||
}
|
||||
assert response.success is True
|
||||
assert response.exit_code == 0
|
||||
|
||||
def test_interact_with_prompt(self):
|
||||
client = _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"output": "Clicked the button",
|
||||
"cdpUrl": "wss://browser.example.com/cdp",
|
||||
"liveViewUrl": "https://live.example.com/view",
|
||||
"interactiveLiveViewUrl": "https://live.example.com/interactive",
|
||||
"stdout": "",
|
||||
"exitCode": 0,
|
||||
},
|
||||
),
|
||||
delete_response=_FakeResponse(200, {"success": True}),
|
||||
)
|
||||
response = interact(
|
||||
client,
|
||||
"job-456",
|
||||
prompt="Click the login button",
|
||||
)
|
||||
|
||||
assert client.last_post[0] == "/v2/scrape/job-456/interact"
|
||||
assert client.last_post[1] == {
|
||||
"language": "node",
|
||||
"prompt": "Click the login button",
|
||||
}
|
||||
assert response.success is True
|
||||
assert response.output == "Clicked the button"
|
||||
assert response.cdp_url == "wss://browser.example.com/cdp"
|
||||
assert response.live_view_url == "https://live.example.com/view"
|
||||
assert response.interactive_live_view_url == "https://live.example.com/interactive"
|
||||
|
||||
def test_interact_validates_required_inputs(self):
|
||||
client = _FakeClient(
|
||||
post_response=_FakeResponse(200, {"success": True}),
|
||||
delete_response=_FakeResponse(200, {"success": True}),
|
||||
)
|
||||
with pytest.raises(ValueError, match="Job ID cannot be empty"):
|
||||
interact(client, "", "console.log('ok')")
|
||||
with pytest.raises(ValueError, match="Either 'code' or 'prompt' must be provided"):
|
||||
interact(client, "job-123")
|
||||
|
||||
def test_interact_raises_when_success_false(self):
|
||||
client = _FakeClient(
|
||||
post_response=_FakeResponse(
|
||||
200,
|
||||
{
|
||||
"success": False,
|
||||
"error": "Replay context is unavailable",
|
||||
},
|
||||
),
|
||||
delete_response=_FakeResponse(200, {"success": True}),
|
||||
)
|
||||
|
||||
with pytest.raises(Exception, match="Replay context is unavailable"):
|
||||
interact(client, "job-123", "console.log('ok')")
|
||||
|
||||
def test_stop_interaction_request_and_response_normalization(self):
|
||||
client = _FakeClient(
|
||||
post_response=_FakeResponse(200, {"success": True}),
|
||||
delete_response=_FakeResponse(
|
||||
200,
|
||||
{
|
||||
"success": True,
|
||||
"sessionDurationMs": 1200,
|
||||
"creditsBilled": 3,
|
||||
},
|
||||
),
|
||||
)
|
||||
response = stop_interaction(client, "job-123")
|
||||
|
||||
assert client.last_delete == "/v2/scrape/job-123/interact"
|
||||
assert response.success is True
|
||||
assert response.session_duration_ms == 1200
|
||||
assert response.credits_billed == 3
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import SearchRequest, ScrapeOptions, Source
|
||||
from firecrawl.v2.methods.search import _prepare_search_request
|
||||
|
||||
|
||||
class TestSearchRequestPreparation:
|
||||
"""Unit tests for search request preparation."""
|
||||
|
||||
def test_basic_request_preparation(self):
|
||||
"""Test basic request preparation with minimal fields."""
|
||||
request = SearchRequest(query="test query")
|
||||
data = _prepare_search_request(request)
|
||||
|
||||
# Check basic fields
|
||||
assert data["query"] == "test query"
|
||||
assert data["limit"] == 5
|
||||
assert data["timeout"] == 300000
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "ignore_invalid_urls" not in data
|
||||
assert "scrape_options" not in data
|
||||
|
||||
def test_all_fields_conversion(self):
|
||||
"""Test request preparation with all possible fields."""
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
|
||||
request = SearchRequest(
|
||||
query="test query",
|
||||
sources=["web", "news"],
|
||||
include_domains=["firecrawl.dev"],
|
||||
limit=10,
|
||||
tbs="qdr:w",
|
||||
location="US",
|
||||
ignore_invalid_urls=False,
|
||||
timeout=30000,
|
||||
highlights=False,
|
||||
scrape_options=scrape_opts,
|
||||
integration=" _e2e-test ",
|
||||
)
|
||||
|
||||
data = _prepare_search_request(request)
|
||||
|
||||
# Check all basic fields
|
||||
assert data["query"] == "test query"
|
||||
assert data["limit"] == 10
|
||||
assert data["tbs"] == "qdr:w"
|
||||
assert data["location"] == "US"
|
||||
assert data["timeout"] == 30000
|
||||
assert data["highlights"] is False
|
||||
|
||||
# Check snake_case to camelCase conversions
|
||||
assert "ignoreInvalidURLs" in data
|
||||
assert data["ignoreInvalidURLs"] is False
|
||||
assert "ignore_invalid_urls" not in data
|
||||
|
||||
assert "scrapeOptions" in data
|
||||
assert "scrape_options" not in data
|
||||
|
||||
# Check sources
|
||||
assert "sources" in data
|
||||
assert len(data["sources"]) == 2
|
||||
assert data["sources"][0]["type"] == "web"
|
||||
assert data["sources"][1]["type"] == "news"
|
||||
assert data["includeDomains"] == ["firecrawl.dev"]
|
||||
assert "include_domains" not in data
|
||||
|
||||
# Check nested scrape options conversions
|
||||
scrape_data = data["scrapeOptions"]
|
||||
assert "includeTags" in scrape_data
|
||||
assert scrape_data["includeTags"] == ["h1", "h2"]
|
||||
assert "excludeTags" in scrape_data
|
||||
assert scrape_data["excludeTags"] == ["nav"]
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert scrape_data["onlyMainContent"] is False
|
||||
assert "waitFor" in scrape_data
|
||||
assert scrape_data["waitFor"] == 2000
|
||||
assert "skipTlsVerification" in scrape_data
|
||||
assert scrape_data["skipTlsVerification"] is True
|
||||
assert "removeBase64Images" in scrape_data
|
||||
assert scrape_data["removeBase64Images"] is False
|
||||
assert data["integration"] == "_e2e-test"
|
||||
|
||||
def test_exclude_none_behavior(self):
|
||||
"""Test that exclude_none=True behavior is working."""
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
sources=None,
|
||||
limit=None,
|
||||
tbs=None,
|
||||
location=None,
|
||||
ignore_invalid_urls=None,
|
||||
timeout=None,
|
||||
scrape_options=None
|
||||
)
|
||||
|
||||
data = _prepare_search_request(request)
|
||||
|
||||
# When limit and timeout are explicitly None, they should be excluded
|
||||
assert "query" in data
|
||||
assert len(data) == 1 # Only query should be present
|
||||
|
||||
def test_domain_filters_are_mutually_exclusive(self):
|
||||
"""Test that include_domains and exclude_domains cannot both be set."""
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="include_domains and exclude_domains cannot both be specified",
|
||||
):
|
||||
SearchRequest(
|
||||
query="test",
|
||||
include_domains=["firecrawl.dev"],
|
||||
exclude_domains=["example.com"],
|
||||
)
|
||||
|
||||
def test_empty_scrape_options(self):
|
||||
"""Test that empty scrape options are handled correctly."""
|
||||
scrape_opts = ScrapeOptions() # All defaults
|
||||
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
data = _prepare_search_request(request)
|
||||
|
||||
assert "scrapeOptions" in data
|
||||
scrape_data = data["scrapeOptions"]
|
||||
|
||||
# Should have default values
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert scrape_data["onlyMainContent"] is True
|
||||
assert "mobile" in scrape_data
|
||||
assert scrape_data["mobile"] is False
|
||||
|
||||
def test_scrape_options_shared_function_integration(self):
|
||||
"""Test that the shared prepare_scrape_options function is being used."""
|
||||
# Test with all snake_case fields to ensure conversion
|
||||
scrape_opts = ScrapeOptions(
|
||||
formats=["markdown", "rawHtml"],
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
wait_for=2000,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
# Note: raw_html should be in formats array, not as a separate field
|
||||
)
|
||||
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
scrape_options=scrape_opts
|
||||
)
|
||||
|
||||
data = _prepare_search_request(request)
|
||||
|
||||
# Check that scrapeOptions is present and converted
|
||||
assert "scrapeOptions" in data
|
||||
scrape_data = data["scrapeOptions"]
|
||||
|
||||
# Check all conversions are working
|
||||
assert "formats" in scrape_data
|
||||
assert scrape_data["formats"] == ["markdown", "rawHtml"]
|
||||
assert "includeTags" in scrape_data
|
||||
assert "excludeTags" in scrape_data
|
||||
assert "onlyMainContent" in scrape_data
|
||||
assert "waitFor" in scrape_data
|
||||
assert "skipTlsVerification" in scrape_data
|
||||
assert "removeBase64Images" in scrape_data
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "include_tags" not in scrape_data
|
||||
assert "exclude_tags" not in scrape_data
|
||||
assert "only_main_content" not in scrape_data
|
||||
assert "wait_for" not in scrape_data
|
||||
assert "skip_tls_verification" not in scrape_data
|
||||
assert "remove_base64_images" not in scrape_data
|
||||
@@ -0,0 +1,73 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
from firecrawl.v2.types import SearchRequest, SearchResultWeb
|
||||
from firecrawl.v2.methods.search import search
|
||||
from firecrawl.v2.methods.aio.search import search as search_async
|
||||
|
||||
|
||||
def _response(data):
|
||||
response = Mock()
|
||||
response.status_code = 200
|
||||
response.json.return_value = {"success": True, "data": data}
|
||||
return response
|
||||
|
||||
|
||||
DEVELOPER_PAYLOAD = {
|
||||
"web": [{"title": "W", "url": "http://a.com", "description": "d"}],
|
||||
"developer": [
|
||||
{
|
||||
"title": "https://github.com/firecrawl/firecrawl/issues/1",
|
||||
"url": "https://github.com/firecrawl/firecrawl/issues/1",
|
||||
"description": "passage",
|
||||
"position": 1,
|
||||
"category": "developer",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestSearchDeveloperResults:
|
||||
"""Unit tests for the developer category results."""
|
||||
|
||||
def test_developer_results_are_parsed(self):
|
||||
client = Mock()
|
||||
client.post.return_value = _response(DEVELOPER_PAYLOAD)
|
||||
|
||||
result = search(client, SearchRequest(query="test", categories=["developer"]))
|
||||
|
||||
assert len(result.web) == 1
|
||||
assert len(result.developer) == 1
|
||||
assert isinstance(result.developer[0], SearchResultWeb)
|
||||
assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
|
||||
assert result.developer[0].position == 1
|
||||
assert result.developer[0].category == "developer"
|
||||
|
||||
def test_developer_absent_when_not_returned(self):
|
||||
client = Mock()
|
||||
client.post.return_value = _response({"web": []})
|
||||
|
||||
result = search(client, SearchRequest(query="test"))
|
||||
|
||||
assert result.developer is None
|
||||
|
||||
def test_data_property_lists_developer(self):
|
||||
client = Mock()
|
||||
client.post.return_value = _response({"developer": DEVELOPER_PAYLOAD["developer"]})
|
||||
|
||||
result = search(client, SearchRequest(query="test", categories=["developer"]))
|
||||
|
||||
with pytest.raises(AttributeError, match=r"\.developer \(1 results\)"):
|
||||
result.data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_developer_results_are_parsed_async(self):
|
||||
client = Mock()
|
||||
client.post = AsyncMock(return_value=_response(DEVELOPER_PAYLOAD))
|
||||
|
||||
result = await search_async(
|
||||
client, SearchRequest(query="test", categories=["developer"])
|
||||
)
|
||||
|
||||
assert len(result.developer) == 1
|
||||
assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
|
||||
@@ -0,0 +1,237 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import Category, SearchRequest, Source, ScrapeOptions, ScrapeFormats
|
||||
from firecrawl.v2.methods.search import _validate_search_request
|
||||
|
||||
|
||||
class TestSearchValidation:
|
||||
"""Unit tests for search request validation."""
|
||||
|
||||
def test_validate_empty_query(self):
|
||||
"""Test validation of empty query."""
|
||||
request = SearchRequest(query="")
|
||||
with pytest.raises(ValueError, match="Query cannot be empty"):
|
||||
_validate_search_request(request)
|
||||
|
||||
request = SearchRequest(query=" ")
|
||||
with pytest.raises(ValueError, match="Query cannot be empty"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_invalid_limit(self):
|
||||
"""Test validation of invalid limits."""
|
||||
# Zero limit
|
||||
request = SearchRequest(query="test", limit=0)
|
||||
with pytest.raises(ValueError, match="Limit must be positive"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Negative limit
|
||||
request = SearchRequest(query="test", limit=-1)
|
||||
with pytest.raises(ValueError, match="Limit must be positive"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Too high limit
|
||||
request = SearchRequest(query="test", limit=101)
|
||||
with pytest.raises(ValueError, match="Limit cannot exceed 100"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_invalid_timeout(self):
|
||||
"""Test validation of invalid timeouts."""
|
||||
# Zero timeout
|
||||
request = SearchRequest(query="test", timeout=0)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Negative timeout
|
||||
request = SearchRequest(query="test", timeout=-1000)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Too high timeout
|
||||
request = SearchRequest(query="test", timeout=300001)
|
||||
with pytest.raises(ValueError, match="Timeout cannot exceed 300000ms"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_invalid_sources(self):
|
||||
"""Test validation of invalid sources."""
|
||||
# Invalid string source
|
||||
request = SearchRequest(query="test", sources=["invalid_source"])
|
||||
with pytest.raises(ValueError, match="Invalid source type"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Invalid object source
|
||||
request = SearchRequest(query="test", sources=[Source(type="invalid_source")])
|
||||
with pytest.raises(ValueError, match="Invalid source type"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Mixed valid/invalid sources
|
||||
request = SearchRequest(query="test", sources=["web", "invalid_source"])
|
||||
with pytest.raises(ValueError, match="Invalid source type"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_categories(self):
|
||||
"""Test validation of search categories."""
|
||||
for category in ["github", "research", "pdf", "developer"]:
|
||||
request = SearchRequest(query="test", categories=[category])
|
||||
assert _validate_search_request(request) == request
|
||||
|
||||
request = SearchRequest(query="test", categories=[Category(type=category)])
|
||||
assert _validate_search_request(request) == request
|
||||
|
||||
request = SearchRequest(query="test", categories=["invalid_category"])
|
||||
with pytest.raises(ValueError, match="Invalid category type"):
|
||||
_validate_search_request(request)
|
||||
|
||||
request = SearchRequest(query="test", categories=[Category(type="invalid_category")])
|
||||
with pytest.raises(ValueError, match="Invalid category type"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_invalid_location(self):
|
||||
"""Test validation of invalid location."""
|
||||
# Empty location
|
||||
request = SearchRequest(query="test", location="")
|
||||
with pytest.raises(ValueError, match="Location must be a non-empty string"):
|
||||
_validate_search_request(request)
|
||||
|
||||
# Whitespace location
|
||||
request = SearchRequest(query="test", location=" ")
|
||||
with pytest.raises(ValueError, match="Location must be a non-empty string"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_empty_tbs(self):
|
||||
"""Test validation rejects empty tbs values."""
|
||||
for empty_tbs in ["", " "]:
|
||||
request = SearchRequest(query="test", tbs=empty_tbs)
|
||||
with pytest.raises(ValueError, match="tbs must be a non-empty string"):
|
||||
_validate_search_request(request)
|
||||
|
||||
def test_validate_tbs_passthrough(self):
|
||||
"""Test that tbs values are passed through without strict validation."""
|
||||
valid_tbs_values = [
|
||||
"qdr:d",
|
||||
"qdr:w",
|
||||
"sbd:1",
|
||||
"sbd:1,qdr:w",
|
||||
"sbd:1,cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",
|
||||
"cdr:1,cd_min:1/1/2024,cd_max:12/31/2024",
|
||||
]
|
||||
|
||||
for tbs in valid_tbs_values:
|
||||
request = SearchRequest(query="test", tbs=tbs)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
def test_validate_valid_requests(self):
|
||||
"""Test that valid requests pass validation."""
|
||||
# Minimal valid request
|
||||
request = SearchRequest(query="test")
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Request with all optional parameters
|
||||
request = SearchRequest(
|
||||
query="test query",
|
||||
sources=["web", "news"],
|
||||
limit=10,
|
||||
tbs="qdr:w",
|
||||
location="US",
|
||||
ignore_invalid_urls=False,
|
||||
timeout=30000
|
||||
)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Request with object sources
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
sources=[Source(type="web"), Source(type="images")]
|
||||
)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
def test_validate_edge_cases(self):
|
||||
"""Test edge cases and boundary values."""
|
||||
# Maximum valid limit
|
||||
request = SearchRequest(query="test", limit=100)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Maximum valid timeout
|
||||
request = SearchRequest(query="test", timeout=300000)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Minimum valid limit
|
||||
request = SearchRequest(query="test", limit=1)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Minimum valid timeout
|
||||
request = SearchRequest(query="test", timeout=1)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
def test_validate_none_values(self):
|
||||
"""Test that None values for optional fields are handled correctly."""
|
||||
request = SearchRequest(
|
||||
query="test",
|
||||
sources=None,
|
||||
limit=None,
|
||||
tbs=None,
|
||||
location=None,
|
||||
ignore_invalid_urls=None,
|
||||
timeout=None
|
||||
)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
def test_validate_scrape_options_integration(self):
|
||||
"""Test that scrape_options validation is integrated."""
|
||||
# Test with valid scrape options
|
||||
scrape_opts = ScrapeOptions(formats=["markdown"], timeout=30000)
|
||||
request = SearchRequest(query="test", scrape_options=scrape_opts)
|
||||
validated = _validate_search_request(request)
|
||||
assert validated == request
|
||||
|
||||
# Test with invalid scrape options (should raise error)
|
||||
invalid_scrape_opts = ScrapeOptions(timeout=-1000)
|
||||
request = SearchRequest(query="test", scrape_options=invalid_scrape_opts)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
_validate_search_request(request)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class TestSearchRequestModel:
|
||||
"""Unit tests for SearchRequest model behavior."""
|
||||
|
||||
def test_default_values(self):
|
||||
"""Test that default values are set correctly."""
|
||||
request = SearchRequest(query="test")
|
||||
assert request.limit == 5
|
||||
assert request.ignore_invalid_urls is None # No default in model
|
||||
assert request.timeout == 300000
|
||||
assert request.sources is None
|
||||
assert request.tbs is None
|
||||
assert request.location is None
|
||||
assert request.scrape_options is None
|
||||
|
||||
def test_field_aliases(self):
|
||||
"""Test that field aliases work correctly for API serialization."""
|
||||
# Test with None value (no default)
|
||||
request1 = SearchRequest(query="test")
|
||||
data1 = request1.model_dump(by_alias=True)
|
||||
assert "ignore_invalid_urls" in data1 # No alias, uses snake_case
|
||||
assert data1["ignore_invalid_urls"] is None
|
||||
|
||||
# Test with explicit False value
|
||||
request2 = SearchRequest(
|
||||
query="test",
|
||||
ignore_invalid_urls=False,
|
||||
scrape_options=ScrapeOptions(formats=["markdown"])
|
||||
)
|
||||
|
||||
# Check that aliases are used in model_dump with by_alias=True
|
||||
data2 = request2.model_dump(by_alias=True)
|
||||
assert "ignore_invalid_urls" in data2 # No alias, uses snake_case
|
||||
assert "scrape_options" in data2 # No alias, uses snake_case
|
||||
assert data2["ignore_invalid_urls"] is False
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
from firecrawl.v2.types import (
|
||||
CrawlRequest,
|
||||
MapOptions,
|
||||
ScrapeOptions,
|
||||
SearchRequest,
|
||||
ThreatProtectionOptions,
|
||||
)
|
||||
from firecrawl.v2.methods.agent import _prepare_agent_request
|
||||
from firecrawl.v2.methods.batch import prepare_batch_scrape_request
|
||||
from firecrawl.v2.methods.crawl import _prepare_crawl_request
|
||||
from firecrawl.v2.methods.extract import _prepare_extract_request
|
||||
from firecrawl.v2.methods.map import _prepare_map_request
|
||||
from firecrawl.v2.methods.scrape import _prepare_scrape_request
|
||||
from firecrawl.v2.methods.search import _prepare_search_request
|
||||
|
||||
|
||||
EXPECTED_WIRE_PAYLOAD = {
|
||||
"mode": "normal",
|
||||
"riskScoreThreshold": 80,
|
||||
"blacklist": ["*.blocked.example.com"],
|
||||
"whitelist": ["allowed.example.com"],
|
||||
"blockedTlds": ["zip"],
|
||||
"failurePolicy": "open",
|
||||
}
|
||||
|
||||
|
||||
def _threat_protection() -> ThreatProtectionOptions:
|
||||
return ThreatProtectionOptions(
|
||||
mode="normal",
|
||||
risk_score_threshold=80,
|
||||
blacklist=["*.blocked.example.com"],
|
||||
whitelist=["allowed.example.com"],
|
||||
blocked_tlds=["zip"],
|
||||
failure_policy="open",
|
||||
)
|
||||
|
||||
|
||||
class TestThreatProtectionRequestPreparation:
|
||||
"""threat_protection must reach the wire body camelCased everywhere."""
|
||||
|
||||
def test_construction_by_alias(self):
|
||||
options = ThreatProtectionOptions(
|
||||
riskScoreThreshold=42,
|
||||
blockedTlds=["zip"],
|
||||
failurePolicy="closed",
|
||||
)
|
||||
assert options.risk_score_threshold == 42
|
||||
assert options.blocked_tlds == ["zip"]
|
||||
assert options.failure_policy == "closed"
|
||||
|
||||
def test_scrape_top_level(self):
|
||||
options = ScrapeOptions(threat_protection=_threat_protection())
|
||||
data = _prepare_scrape_request("https://example.com", options)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_scrape_omitted_when_unset(self):
|
||||
data = _prepare_scrape_request(
|
||||
"https://example.com", ScrapeOptions(only_main_content=True)
|
||||
)
|
||||
assert "threatProtection" not in data
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_batch_scrape_top_level(self):
|
||||
data = prepare_batch_scrape_request(
|
||||
["https://example.com"],
|
||||
options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_crawl_scrape_options(self):
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
data = _prepare_crawl_request(request)
|
||||
assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
|
||||
def test_search_top_level_and_scrape_options(self):
|
||||
request = SearchRequest(
|
||||
query="firecrawl",
|
||||
threat_protection=_threat_protection(),
|
||||
scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
|
||||
)
|
||||
data = _prepare_search_request(request)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_map_top_level(self):
|
||||
options = MapOptions(threat_protection=_threat_protection())
|
||||
data = _prepare_map_request("https://example.com", options)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
assert "threat_protection" not in data
|
||||
|
||||
def test_extract_top_level(self):
|
||||
data = _prepare_extract_request(
|
||||
["https://example.com"],
|
||||
prompt="extract",
|
||||
threat_protection=_threat_protection(),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
|
||||
def test_agent_top_level(self):
|
||||
data = _prepare_agent_request(
|
||||
None,
|
||||
prompt="find pricing",
|
||||
threat_protection=_threat_protection(),
|
||||
)
|
||||
assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
|
||||
|
||||
def test_partial_override_only_sends_provided_fields(self):
|
||||
options = ScrapeOptions(
|
||||
threat_protection=ThreatProtectionOptions(mode="off")
|
||||
)
|
||||
data = _prepare_scrape_request("https://example.com", options)
|
||||
assert data["threatProtection"] == {"mode": "off"}
|
||||
@@ -0,0 +1,18 @@
|
||||
from firecrawl.v2.types import ConcurrencyCheck, CreditUsage, TokenUsage
|
||||
|
||||
|
||||
class TestUsageTypes:
|
||||
def test_concurrency_check_model(self):
|
||||
cc = ConcurrencyCheck(concurrency=3, max_concurrency=10)
|
||||
assert cc.concurrency == 3
|
||||
assert cc.max_concurrency == 10
|
||||
|
||||
def test_credit_usage_model(self):
|
||||
cu = CreditUsage(remaining_credits=123)
|
||||
assert isinstance(cu.remaining_credits, int)
|
||||
assert cu.remaining_credits == 123
|
||||
|
||||
def test_token_usage_model(self):
|
||||
tu = TokenUsage(remaining_tokens=10)
|
||||
assert tu.remaining_tokens == 10
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Unit tests for webhook functionality in Firecrawl v2 SDK.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from firecrawl.v2.types import WebhookConfig, CrawlRequest
|
||||
from firecrawl.v2.methods.crawl import _prepare_crawl_request
|
||||
|
||||
class TestWebhookConfig:
|
||||
"""Test WebhookConfig class functionality."""
|
||||
|
||||
def test_webhook_config_creation_minimal(self):
|
||||
"""Test creating WebhookConfig with minimal parameters."""
|
||||
webhook = WebhookConfig(url="https://example.com/webhook")
|
||||
assert webhook.url == "https://example.com/webhook"
|
||||
assert webhook.headers is None
|
||||
assert webhook.metadata is None
|
||||
assert webhook.events is None
|
||||
|
||||
def test_webhook_config_creation_full(self):
|
||||
"""Test creating WebhookConfig with all parameters."""
|
||||
webhook = WebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
metadata={"project": "test"},
|
||||
events=["completed", "failed"]
|
||||
)
|
||||
assert webhook.url == "https://example.com/webhook"
|
||||
assert webhook.headers == {"Authorization": "Bearer token"}
|
||||
assert webhook.metadata == {"project": "test"}
|
||||
assert webhook.events == ["completed", "failed"]
|
||||
|
||||
def test_webhook_config_validation(self):
|
||||
"""Test WebhookConfig validation."""
|
||||
# URL is required
|
||||
with pytest.raises(Exception): # Pydantic validation error
|
||||
WebhookConfig()
|
||||
|
||||
|
||||
class TestCrawlRequestWebhook:
|
||||
"""Test CrawlRequest webhook functionality."""
|
||||
|
||||
def test_crawl_request_with_string_webhook(self):
|
||||
"""Test CrawlRequest with string webhook."""
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
webhook="https://example.com/webhook"
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
assert data["webhook"] == "https://example.com/webhook"
|
||||
|
||||
def test_crawl_request_with_webhook_config(self):
|
||||
"""Test CrawlRequest with WebhookConfig object."""
|
||||
webhook_config = WebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
events=["completed"]
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
assert data["webhook"]["url"] == "https://example.com/webhook"
|
||||
assert data["webhook"]["headers"] == {"Authorization": "Bearer token"}
|
||||
assert data["webhook"]["events"] == ["completed"]
|
||||
|
||||
def test_crawl_request_without_webhook(self):
|
||||
"""Test CrawlRequest without webhook."""
|
||||
request = CrawlRequest(url="https://example.com")
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
assert "webhook" not in data
|
||||
|
||||
def test_crawl_request_webhook_serialization(self):
|
||||
"""Test that webhook config is properly serialized."""
|
||||
webhook_config = WebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers={"Content-Type": "application/json"},
|
||||
metadata={"test": "value"},
|
||||
events=["page", "completed"]
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
webhook_data = data["webhook"]
|
||||
|
||||
# Check that all fields are properly serialized
|
||||
assert webhook_data["url"] == "https://example.com/webhook"
|
||||
assert webhook_data["headers"] == {"Content-Type": "application/json"}
|
||||
assert webhook_data["metadata"] == {"test": "value"}
|
||||
assert webhook_data["events"] == ["page", "completed"]
|
||||
|
||||
def test_crawl_request_webhook_with_none_values(self):
|
||||
"""Test webhook config with None values are excluded from serialization."""
|
||||
webhook_config = WebhookConfig(
|
||||
url="https://example.com/webhook",
|
||||
headers=None,
|
||||
metadata=None,
|
||||
events=None
|
||||
)
|
||||
|
||||
request = CrawlRequest(
|
||||
url="https://example.com",
|
||||
webhook=webhook_config
|
||||
)
|
||||
|
||||
data = _prepare_crawl_request(request)
|
||||
webhook_data = data["webhook"]
|
||||
|
||||
# Only url should be present, None values should be excluded
|
||||
assert webhook_data["url"] == "https://example.com/webhook"
|
||||
assert "headers" not in webhook_data
|
||||
assert "metadata" not in webhook_data
|
||||
assert "events" not in webhook_data
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
from firecrawl.v2.types import MonitorCreateRequest, MonitorPageJudgment, MonitorTarget
|
||||
|
||||
|
||||
def test_monitor_search_target_parses_camelcase_fields():
|
||||
target = MonitorTarget.model_validate(
|
||||
{
|
||||
"type": "search",
|
||||
"queries": ["firecrawl launch"],
|
||||
"searchWindow": "24h",
|
||||
"includeDomains": ["firecrawl.dev"],
|
||||
"excludeDomains": ["spam.com"],
|
||||
"maxResults": 20,
|
||||
}
|
||||
)
|
||||
|
||||
assert target.type == "search"
|
||||
assert target.queries == ["firecrawl launch"]
|
||||
assert target.search_window == "24h"
|
||||
assert target.include_domains == ["firecrawl.dev"]
|
||||
assert target.exclude_domains == ["spam.com"]
|
||||
assert target.max_results == 20
|
||||
|
||||
|
||||
def test_monitor_create_request_serializes_search_target_to_camelcase():
|
||||
request = MonitorCreateRequest(
|
||||
name="Search monitor",
|
||||
schedule={"text": "every 30 minutes"},
|
||||
goal="Alert when Firecrawl launches a product",
|
||||
targets=[
|
||||
MonitorTarget(
|
||||
type="search",
|
||||
queries=["firecrawl launch"],
|
||||
search_window="24h",
|
||||
max_results=15,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
payload = request.model_dump(exclude_none=True, by_alias=True)
|
||||
target = payload["targets"][0]
|
||||
assert target["type"] == "search"
|
||||
assert target["queries"] == ["firecrawl launch"]
|
||||
assert target["searchWindow"] == "24h"
|
||||
assert target["maxResults"] == 15
|
||||
# internal-only fields are never emitted by the SDK
|
||||
assert "depth" not in target
|
||||
assert "alertMode" not in target
|
||||
|
||||
|
||||
def test_monitor_page_judgment_parses_meaningful_changes():
|
||||
judgment = MonitorPageJudgment.model_validate(
|
||||
{
|
||||
"meaningful": True,
|
||||
"confidence": "high",
|
||||
"reason": "The tracked price changed.",
|
||||
"meaningfulChanges": [
|
||||
{
|
||||
"type": "changed",
|
||||
"before": "$10",
|
||||
"after": "$12",
|
||||
"reason": "Price increased.",
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert judgment.meaningful is True
|
||||
assert judgment.meaningful_changes[0].type == "changed"
|
||||
@@ -0,0 +1,94 @@
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.types import Document, DocumentMetadata
|
||||
from firecrawl.v2.utils.normalize import normalize_document_input
|
||||
|
||||
|
||||
class TestDocumentMetadataExtras:
|
||||
def test_metadata_extras_preserved_in_metadata_dict(self):
|
||||
raw = {
|
||||
"markdown": "# Hello",
|
||||
"metadata": {
|
||||
"title": "Example",
|
||||
# Known camelCase -> snake_case mapping
|
||||
"statusCode": "200",
|
||||
# Unknown keys that should be preserved as-is
|
||||
"twitter:card": "summary",
|
||||
"twitterCard": "summary_large_image",
|
||||
"theme-color": "#fff",
|
||||
},
|
||||
}
|
||||
|
||||
doc = Document(**normalize_document_input(raw))
|
||||
md_dict = doc.metadata_dict
|
||||
# Known key mapped and coerced
|
||||
assert md_dict["status_code"] == 200
|
||||
assert md_dict["title"] == "Example"
|
||||
# Extras are preserved verbatim
|
||||
assert md_dict["twitter:card"] == "summary"
|
||||
assert md_dict["twitterCard"] == "summary_large_image"
|
||||
assert md_dict["theme-color"] == "#fff"
|
||||
|
||||
def test_metadata_typed_from_plain_dict_preserves_extras(self):
|
||||
# Construct Document with raw dict metadata without normalization step
|
||||
doc = Document(
|
||||
markdown="# Hi",
|
||||
metadata={
|
||||
"ogTitle": "Hello", # will be treated as extra without normalization
|
||||
"x-custom": "ok",
|
||||
},
|
||||
)
|
||||
|
||||
md = doc.metadata_typed
|
||||
assert isinstance(md, DocumentMetadata)
|
||||
# Known fields aren't populated without normalization
|
||||
assert md.og_title is None
|
||||
# Extras are available on the underlying pydantic storage and in metadata_dict
|
||||
extras = getattr(md, "__pydantic_extra__", {}) or {}
|
||||
assert extras == {"ogTitle": "Hello", "x-custom": "ok"}
|
||||
assert doc.metadata_dict["ogTitle"] == "Hello"
|
||||
assert doc.metadata_dict["x-custom"] == "ok"
|
||||
|
||||
def test_document_model_dump_includes_metadata_extras(self):
|
||||
raw = {
|
||||
"markdown": "# Body",
|
||||
"metadata": {
|
||||
"title": "Page",
|
||||
"twitter:site": "@site",
|
||||
},
|
||||
}
|
||||
doc = Document(**normalize_document_input(raw))
|
||||
dumped = doc.model_dump(exclude_none=True)
|
||||
assert "metadata" in dumped
|
||||
assert dumped["metadata"]["title"] == "Page"
|
||||
assert dumped["metadata"]["twitter:site"] == "@site"
|
||||
|
||||
def test_concurrency_fields_are_mapped(self):
|
||||
raw = {
|
||||
"markdown": "# Queue info",
|
||||
"metadata": {
|
||||
"concurrencyLimited": True,
|
||||
"concurrencyQueueDurationMs": 1234,
|
||||
},
|
||||
}
|
||||
doc = Document(**normalize_document_input(raw))
|
||||
md = doc.metadata_typed
|
||||
assert md.concurrency_limited is True
|
||||
assert md.concurrency_queue_duration_ms == 1234
|
||||
|
||||
def test_unknown_list_metadata_preserved(self):
|
||||
raw = {
|
||||
"markdown": "# Body",
|
||||
"metadata": {
|
||||
"title": "Page",
|
||||
"x-list": [1, "a", 3],
|
||||
},
|
||||
}
|
||||
doc = Document(**normalize_document_input(raw))
|
||||
md = doc.metadata_dict
|
||||
assert md["x-list"] == [1, "a", 3]
|
||||
|
||||
def test_metadata_typed_extras_property(self):
|
||||
md = DocumentMetadata(title="T", **{"x-foo": "bar"})
|
||||
# extras accessor should expose unknown keys
|
||||
assert md.extras == {"x-foo": "bar"}
|
||||
@@ -0,0 +1,22 @@
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.types import Document
|
||||
from firecrawl.v2.utils.normalize import normalize_document_input
|
||||
|
||||
|
||||
class TestMetadataMultiValue:
|
||||
def test_article_tag_list_coerced_to_string(self):
|
||||
raw = {
|
||||
"markdown": "# Body",
|
||||
"metadata": {
|
||||
"title": "Page",
|
||||
"articleTag": ["one", "two"],
|
||||
},
|
||||
}
|
||||
doc = Document(**normalize_document_input(raw))
|
||||
# typed access works and is joined as string
|
||||
assert doc.metadata is not None
|
||||
assert doc.metadata.article_tag == "one, two"
|
||||
# dict view shows string
|
||||
md = doc.metadata_dict
|
||||
assert md["article_tag"] == "one, two"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,400 @@
|
||||
import pytest
|
||||
from firecrawl.v2.types import (
|
||||
HighlightsFormat,
|
||||
JsonFormat,
|
||||
QueryFormat,
|
||||
QuestionFormat,
|
||||
ScrapeOptions,
|
||||
PDFParser,
|
||||
)
|
||||
from firecrawl.v2.utils.validation import validate_scrape_options, prepare_scrape_options
|
||||
|
||||
|
||||
class TestValidateScrapeOptions:
|
||||
"""Unit tests for validate_scrape_options function."""
|
||||
|
||||
def test_validate_none_options(self):
|
||||
"""Test validation with None options."""
|
||||
result = validate_scrape_options(None)
|
||||
assert result is None
|
||||
|
||||
def test_validate_valid_options(self):
|
||||
"""Test validation with valid options."""
|
||||
options = ScrapeOptions(
|
||||
formats=["markdown"],
|
||||
timeout=30000,
|
||||
wait_for=2000
|
||||
)
|
||||
result = validate_scrape_options(options)
|
||||
assert result == options
|
||||
|
||||
def test_validate_invalid_timeout(self):
|
||||
"""Test validation with invalid timeout."""
|
||||
options = ScrapeOptions(timeout=0)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
validate_scrape_options(options)
|
||||
|
||||
def test_validate_negative_timeout(self):
|
||||
"""Test validation with negative timeout."""
|
||||
options = ScrapeOptions(timeout=-1000)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
validate_scrape_options(options)
|
||||
|
||||
def test_validate_invalid_wait_for(self):
|
||||
"""Test validation with invalid wait_for."""
|
||||
options = ScrapeOptions(wait_for=-500)
|
||||
with pytest.raises(ValueError, match="wait_for must be non-negative"):
|
||||
validate_scrape_options(options)
|
||||
|
||||
def test_validate_zero_wait_for(self):
|
||||
"""Test validation with zero wait_for (should be valid)."""
|
||||
options = ScrapeOptions(wait_for=0)
|
||||
result = validate_scrape_options(options)
|
||||
assert result == options
|
||||
|
||||
def test_validate_complex_options(self):
|
||||
"""Test validation with complex options."""
|
||||
options = ScrapeOptions(
|
||||
formats=["markdown", "html"],
|
||||
headers={"User-Agent": "Test"},
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False,
|
||||
raw_html=True,
|
||||
screenshot_full_page=True
|
||||
)
|
||||
result = validate_scrape_options(options)
|
||||
assert result == options
|
||||
|
||||
def test_validate_multiple_invalid_fields(self):
|
||||
"""Test validation with multiple invalid fields."""
|
||||
options = ScrapeOptions(timeout=-1000, wait_for=-500)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
validate_scrape_options(options)
|
||||
# Should fail on first invalid field (timeout)
|
||||
|
||||
def test_validate_edge_cases(self):
|
||||
"""Test validation with edge case values."""
|
||||
# Test with very large timeout
|
||||
options = ScrapeOptions(timeout=999999)
|
||||
result = validate_scrape_options(options)
|
||||
assert result == options
|
||||
|
||||
# Test with very large wait_for
|
||||
options = ScrapeOptions(wait_for=999999)
|
||||
result = validate_scrape_options(options)
|
||||
assert result == options
|
||||
|
||||
|
||||
class TestPrepareScrapeOptions:
|
||||
"""Unit tests for prepare_scrape_options function."""
|
||||
|
||||
def test_prepare_none_options(self):
|
||||
"""Test preparation with None options."""
|
||||
result = prepare_scrape_options(None)
|
||||
assert result is None
|
||||
|
||||
def test_prepare_basic_options(self):
|
||||
"""Test preparation with basic options."""
|
||||
options = ScrapeOptions(
|
||||
formats=["markdown", "video"],
|
||||
timeout=30000,
|
||||
wait_for=2000
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert "formats" in result
|
||||
assert "timeout" in result
|
||||
assert "waitFor" in result
|
||||
assert result["formats"] == ["markdown", "video"]
|
||||
assert result["timeout"] == 30000
|
||||
assert result["waitFor"] == 2000
|
||||
|
||||
def test_prepare_snake_case_conversion(self):
|
||||
"""Test snake_case to camelCase conversion."""
|
||||
options = ScrapeOptions(
|
||||
include_tags=["h1", "h2"],
|
||||
exclude_tags=["nav"],
|
||||
only_main_content=False,
|
||||
wait_for=2000,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
# Note: raw_html should be in formats array, not as a separate field
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
# Check conversions
|
||||
assert "includeTags" in result
|
||||
assert result["includeTags"] == ["h1", "h2"]
|
||||
assert "excludeTags" in result
|
||||
assert result["excludeTags"] == ["nav"]
|
||||
assert "onlyMainContent" in result
|
||||
assert result["onlyMainContent"] is False
|
||||
assert "waitFor" in result
|
||||
assert result["waitFor"] == 2000
|
||||
assert "skipTlsVerification" in result
|
||||
assert result["skipTlsVerification"] is True
|
||||
assert "removeBase64Images" in result
|
||||
assert result["removeBase64Images"] is False
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "include_tags" not in result
|
||||
assert "exclude_tags" not in result
|
||||
assert "only_main_content" not in result
|
||||
assert "wait_for" not in result
|
||||
assert "skip_tls_verification" not in result
|
||||
assert "remove_base64_images" not in result
|
||||
|
||||
def test_prepare_complex_options(self):
|
||||
"""Test preparation with complex options."""
|
||||
options = ScrapeOptions(
|
||||
formats=["markdown", "html", "rawHtml"],
|
||||
headers={"User-Agent": "Test Bot"},
|
||||
include_tags=["h1", "h2", "h3"],
|
||||
exclude_tags=["nav", "footer"],
|
||||
only_main_content=False,
|
||||
timeout=15000,
|
||||
wait_for=2000,
|
||||
mobile=True,
|
||||
skip_tls_verification=True,
|
||||
remove_base64_images=False
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
# Check all fields are present and converted
|
||||
assert "formats" in result
|
||||
assert "headers" in result
|
||||
assert "includeTags" in result
|
||||
assert "excludeTags" in result
|
||||
assert "onlyMainContent" in result
|
||||
assert "timeout" in result
|
||||
assert "waitFor" in result
|
||||
assert "mobile" in result
|
||||
assert "skipTlsVerification" in result
|
||||
assert "removeBase64Images" in result
|
||||
|
||||
# Check values
|
||||
assert result["formats"] == ["markdown", "html", "rawHtml"]
|
||||
assert result["headers"] == {"User-Agent": "Test Bot"}
|
||||
assert result["includeTags"] == ["h1", "h2", "h3"]
|
||||
assert result["excludeTags"] == ["nav", "footer"]
|
||||
assert result["onlyMainContent"] is False
|
||||
assert result["timeout"] == 15000
|
||||
assert result["waitFor"] == 2000
|
||||
assert result["mobile"] is True
|
||||
assert result["skipTlsVerification"] is True
|
||||
assert result["removeBase64Images"] is False
|
||||
|
||||
def test_prepare_invalid_options(self):
|
||||
"""Test preparation with invalid options (should raise error)."""
|
||||
options = ScrapeOptions(timeout=-1000)
|
||||
with pytest.raises(ValueError, match="Timeout must be positive"):
|
||||
prepare_scrape_options(options)
|
||||
|
||||
def test_prepare_query_format_with_mode(self):
|
||||
"""Test query format mode is preserved."""
|
||||
options = ScrapeOptions(
|
||||
formats=[QueryFormat(prompt="What is Firecrawl?", mode="directQuote")]
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert result["formats"] == [
|
||||
{"type": "query", "prompt": "What is Firecrawl?", "mode": "directQuote"}
|
||||
]
|
||||
|
||||
def test_prepare_question_and_highlights_formats(self):
|
||||
"""Test question and highlights formats are preserved."""
|
||||
options = ScrapeOptions(
|
||||
formats=[
|
||||
QuestionFormat(question="What is Firecrawl?"),
|
||||
HighlightsFormat(query="What is Firecrawl?"),
|
||||
]
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert result["formats"] == [
|
||||
{"type": "question", "question": "What is Firecrawl?"},
|
||||
{"type": "highlights", "query": "What is Firecrawl?"},
|
||||
]
|
||||
|
||||
def test_prepare_question_and_highlights_reject_empty_values(self):
|
||||
"""Test question and highlights validation."""
|
||||
with pytest.raises(ValueError, match="question format requires"):
|
||||
prepare_scrape_options(
|
||||
ScrapeOptions(formats=[{"type": "question", "question": ""}])
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="highlights format requires"):
|
||||
prepare_scrape_options(
|
||||
ScrapeOptions(formats=[{"type": "highlights", "query": ""}])
|
||||
)
|
||||
|
||||
def test_prepare_query_format_rejects_direct_quote_boolean(self):
|
||||
"""Test old query directQuote layout is rejected."""
|
||||
options = ScrapeOptions(
|
||||
formats=[
|
||||
{
|
||||
"type": "query",
|
||||
"prompt": "What is Firecrawl?",
|
||||
"directQuote": True,
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="uses 'mode' instead of 'directQuote'"):
|
||||
prepare_scrape_options(options)
|
||||
|
||||
def test_prepare_query_format_rejects_invalid_mode(self):
|
||||
"""Test query mode validation."""
|
||||
options = ScrapeOptions(
|
||||
formats=[
|
||||
{
|
||||
"type": "query",
|
||||
"prompt": "What is Firecrawl?",
|
||||
"mode": "quoted",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="mode must be 'freeform' or 'directQuote'"):
|
||||
prepare_scrape_options(options)
|
||||
|
||||
def test_prepare_empty_options(self):
|
||||
"""Test preparation with empty options."""
|
||||
options = ScrapeOptions() # All defaults
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
# Should return dict with default values
|
||||
assert isinstance(result, dict)
|
||||
assert "onlyMainContent" in result
|
||||
assert result["onlyMainContent"] is True
|
||||
assert "mobile" in result
|
||||
assert result["mobile"] is False
|
||||
|
||||
def test_prepare_none_values(self):
|
||||
"""Test preparation with None values in options."""
|
||||
options = ScrapeOptions(
|
||||
formats=None,
|
||||
timeout=None,
|
||||
wait_for=None,
|
||||
include_tags=None,
|
||||
exclude_tags=None
|
||||
)
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
# Should only include non-None values
|
||||
assert isinstance(result, dict)
|
||||
# Should have default values for required fields
|
||||
assert "onlyMainContent" in result
|
||||
assert "mobile" in result
|
||||
|
||||
def test_format_schema_conversion(self):
|
||||
"""Test that Format schema is properly handled."""
|
||||
# Create a JsonFormat object with schema
|
||||
format_obj = JsonFormat(
|
||||
type="json",
|
||||
prompt="Extract product info",
|
||||
schema={"type": "object", "properties": {"name": {"type": "string"}}}
|
||||
)
|
||||
|
||||
dumped = format_obj.model_dump()
|
||||
assert "schema" in dumped
|
||||
assert dumped["schema"] == {"type": "object", "properties": {"name": {"type": "string"}}}
|
||||
|
||||
def test_prepare_new_v2_fields(self):
|
||||
"""Test preparation with new v2 fields."""
|
||||
from firecrawl.v2.types import Viewport, ScreenshotAction
|
||||
|
||||
viewport = Viewport(width=1920, height=1080)
|
||||
screenshot_action = ScreenshotAction(
|
||||
type="screenshot",
|
||||
full_page=True,
|
||||
quality=90,
|
||||
viewport=viewport
|
||||
)
|
||||
|
||||
options = ScrapeOptions(
|
||||
fast_mode=True,
|
||||
use_mock="test-mock",
|
||||
block_ads=False,
|
||||
store_in_cache=False,
|
||||
lockdown=True,
|
||||
max_age=7200000, # 2 hours
|
||||
actions=[screenshot_action],
|
||||
parsers=["pdf"]
|
||||
)
|
||||
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
# Check new field conversions
|
||||
assert "fastMode" in result
|
||||
assert result["fastMode"] is True
|
||||
assert "useMock" in result
|
||||
assert result["useMock"] == "test-mock"
|
||||
assert "blockAds" in result
|
||||
assert result["blockAds"] is False
|
||||
assert "storeInCache" in result
|
||||
assert result["storeInCache"] is False
|
||||
assert "lockdown" in result
|
||||
assert result["lockdown"] is True
|
||||
assert "maxAge" in result
|
||||
assert result["maxAge"] == 7200000
|
||||
|
||||
# Check actions conversion
|
||||
assert "actions" in result
|
||||
assert len(result["actions"]) == 1
|
||||
action = result["actions"][0]
|
||||
assert action["type"] == "screenshot"
|
||||
assert action["fullPage"] is True
|
||||
assert action["quality"] == 90
|
||||
assert "viewport" in action
|
||||
assert action["viewport"]["width"] == 1920
|
||||
assert action["viewport"]["height"] == 1080
|
||||
|
||||
# Check parsers
|
||||
assert "parsers" in result
|
||||
assert result["parsers"] == ["pdf"]
|
||||
|
||||
# Check that snake_case fields are not present
|
||||
assert "fast_mode" not in result
|
||||
assert "use_mock" not in result
|
||||
assert "block_ads" not in result
|
||||
assert "store_in_cache" not in result
|
||||
assert "max_age" not in result
|
||||
|
||||
def test_prepare_parsers_max_pages_dict(self):
|
||||
"""Ensure parser dicts convert max_pages to maxPages."""
|
||||
options = ScrapeOptions(
|
||||
parsers=[{"type": "pdf", "max_pages": 3}]
|
||||
)
|
||||
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert "parsers" in result
|
||||
assert result["parsers"][0]["maxPages"] == 3
|
||||
assert "max_pages" not in result["parsers"][0]
|
||||
|
||||
def test_prepare_parsers_max_pages_model(self):
|
||||
"""Ensure parser models convert max_pages to maxPages."""
|
||||
parser = PDFParser(max_pages=5)
|
||||
options = ScrapeOptions(parsers=[parser])
|
||||
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert result["parsers"][0]["maxPages"] == 5
|
||||
|
||||
def test_prepare_min_age_maps_to_camel_case(self):
|
||||
"""min_age must be sent as minAge; the server drops the snake_case key."""
|
||||
options = ScrapeOptions(min_age=1000, max_age=5000)
|
||||
|
||||
result = prepare_scrape_options(options)
|
||||
|
||||
assert result["minAge"] == 1000
|
||||
assert "min_age" not in result
|
||||
assert result["maxAge"] == 5000
|
||||
@@ -0,0 +1,332 @@
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from firecrawl.v2.watcher import Watcher
|
||||
|
||||
|
||||
class DummyHttpClient:
|
||||
def __init__(self, api_url: str = "http://localhost", api_key: str = "TEST"):
|
||||
self.api_url = api_url
|
||||
self.api_key = api_key
|
||||
|
||||
|
||||
class DummyClient:
|
||||
def __init__(self):
|
||||
self.http_client = DummyHttpClient()
|
||||
|
||||
|
||||
class FakeWebSocket:
|
||||
def __init__(self, messages):
|
||||
# messages: list of dicts to be json-dumped
|
||||
self._messages = list(messages)
|
||||
|
||||
async def recv(self):
|
||||
if not self._messages:
|
||||
# No more messages; block a bit to allow loop to end
|
||||
await asyncio.sleep(0.01)
|
||||
# Simulate disconnect
|
||||
raise asyncio.CancelledError()
|
||||
msg = self._messages.pop(0)
|
||||
return json.dumps(msg)
|
||||
|
||||
|
||||
class FakeConnect:
|
||||
def __init__(self, ws: FakeWebSocket):
|
||||
self._ws = ws
|
||||
|
||||
async def __aenter__(self):
|
||||
return self._ws
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["crawl", "batch"])
|
||||
def test_ws_watcher_document_and_done(monkeypatch, kind):
|
||||
# Prepare messages: one document then done
|
||||
messages = [
|
||||
{"type": "document", "data": {"url": "https://example.com", "rawHtml": "<html>"}},
|
||||
{"type": "done", "data": {"status": "completed", "data": []}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind=kind)
|
||||
|
||||
events = {"document": 0, "done": 0}
|
||||
statuses = []
|
||||
|
||||
watcher.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
|
||||
watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
|
||||
watcher.add_listener(lambda s: statuses.append(s.status))
|
||||
|
||||
watcher.start()
|
||||
|
||||
# Wait for thread to finish
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert events["document"] >= 1
|
||||
assert events["done"] == 1
|
||||
assert statuses[-1] in ("completed", "failed", "cancelled")
|
||||
|
||||
|
||||
def test_ws_watcher_error_event(monkeypatch):
|
||||
messages = [
|
||||
{"type": "error", "error": "boom", "data": {"status": "failed"}},
|
||||
]
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind="crawl")
|
||||
|
||||
seen_error = {"count": 0}
|
||||
watcher.add_event_listener("error", lambda d: seen_error.__setitem__("count", seen_error["count"] + 1))
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert seen_error["count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["crawl", "batch"])
|
||||
def test_ws_watcher_catchup_dispatches_documents_and_updates_status(monkeypatch, kind):
|
||||
messages = [
|
||||
{
|
||||
"type": "catchup",
|
||||
"data": {
|
||||
"status": "scraping",
|
||||
"data": [
|
||||
{"url": "https://example.com/1", "rawHtml": "<html>1</html>"},
|
||||
{"url": "https://example.com/2", "rawHtml": "<html>2</html>"},
|
||||
],
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind=kind)
|
||||
|
||||
events = {"document": 0}
|
||||
statuses = []
|
||||
|
||||
watcher.add_event_listener("document", lambda d: events.__setitem__("document", events["document"] + 1))
|
||||
watcher.add_listener(lambda s: statuses.append(s.status))
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert events["document"] == 2
|
||||
assert statuses[-1] == "scraping"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["crawl", "batch"])
|
||||
def test_ws_watcher_status_only_terminal_snapshot_triggers_done(monkeypatch, kind):
|
||||
# No explicit type, only a terminal status snapshot
|
||||
messages = [
|
||||
{"data": {"status": "completed", "data": []}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind=kind)
|
||||
|
||||
events = {"done": 0}
|
||||
statuses = []
|
||||
|
||||
watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
|
||||
watcher.add_listener(lambda s: statuses.append(s.status))
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert events["done"] == 1
|
||||
assert statuses[-1] == "completed"
|
||||
|
||||
|
||||
def test_ws_watcher_batch_cancelled_snapshot_no_done_event(monkeypatch):
|
||||
# Batch-only: cancelled snapshot should end without a 'done' event
|
||||
messages = [
|
||||
{"data": {"status": "cancelled", "data": []}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind="batch")
|
||||
|
||||
events = {"done": 0}
|
||||
statuses = []
|
||||
|
||||
watcher.add_event_listener("done", lambda d: events.__setitem__("done", events["done"] + 1))
|
||||
watcher.add_listener(lambda s: statuses.append(s.status))
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert events["done"] == 0
|
||||
assert statuses[-1] == "cancelled"
|
||||
|
||||
|
||||
def test_ws_watcher_propagates_authorization_header(monkeypatch):
|
||||
# Ensure Authorization header is forwarded to websockets.connect
|
||||
messages = [
|
||||
{"type": "done", "data": {"status": "completed", "data": []}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
captured_headers = {"headers": None}
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
captured_headers["headers"] = kwargs.get("additional_headers")
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind="crawl")
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert captured_headers["headers"] is not None
|
||||
# Expect an Authorization header with Bearer token
|
||||
assert any(h[0] == "Authorization" and "Bearer" in h[1] for h in captured_headers["headers"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["crawl", "batch"])
|
||||
def test_ws_watcher_normalizes_document_fields_in_snapshot(monkeypatch, kind):
|
||||
# Status-only snapshot with camelCase fields should be normalized in emitted job
|
||||
messages = [
|
||||
{"data": {"status": "completed", "data": [
|
||||
{"url": "https://example.com/x", "rawHtml": "<x>", "changeTracking": {"modes": ["git-diff"]}}
|
||||
]}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind=kind)
|
||||
|
||||
jobs = []
|
||||
watcher.add_listener(lambda j: jobs.append(j))
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert jobs, "No job snapshots emitted"
|
||||
last_job = jobs[-1]
|
||||
assert last_job.status == "completed"
|
||||
assert last_job.data and last_job.data[0].raw_html == "<x>"
|
||||
assert last_job.data[0].change_tracking is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["crawl", "batch"])
|
||||
def test_ws_watcher_uses_correct_ws_uri(monkeypatch, kind):
|
||||
# Verify WS URI uses the correct path per kind and http->ws scheme
|
||||
messages = [
|
||||
{"type": "done", "data": {"status": "completed", "data": []}},
|
||||
]
|
||||
|
||||
ws = FakeWebSocket(messages)
|
||||
|
||||
captured_uri = {"uri": None}
|
||||
|
||||
def fake_connect(uri, *args, **kwargs):
|
||||
captured_uri["uri"] = uri
|
||||
return FakeConnect(ws)
|
||||
|
||||
import websockets
|
||||
monkeypatch.setattr(websockets, "connect", fake_connect)
|
||||
|
||||
client = DummyClient()
|
||||
watcher = Watcher(client, job_id="jid", kind=kind)
|
||||
|
||||
watcher.start()
|
||||
|
||||
deadline = time.time() + 2
|
||||
while watcher._thread and watcher._thread.is_alive() and time.time() < deadline:
|
||||
time.sleep(0.01)
|
||||
|
||||
watcher.stop()
|
||||
|
||||
assert captured_uri["uri"] is not None
|
||||
expected = "ws://localhost/v2/crawl/jid" if kind == "crawl" else "ws://localhost/v2/batch/scrape/jid"
|
||||
assert captured_uri["uri"] == expected
|
||||
@@ -0,0 +1,449 @@
|
||||
"""
|
||||
Firecrawl Client
|
||||
|
||||
A Firecrawl client that enables you to scrape content from websites, crawl entire sites, search the web, and extract structured data using AI.
|
||||
|
||||
The client supports both v1 and v2 API versions, providing access to features like:
|
||||
- Web scraping with advanced options (screenshots, markdown conversion, etc.)
|
||||
- Site crawling with configurable depth and limits
|
||||
- Web search with content extraction
|
||||
- Structured data extraction using AI models
|
||||
- Deep research capabilities
|
||||
|
||||
Usage:
|
||||
from firecrawl import Firecrawl
|
||||
firecrawl = Firecrawl(api_key="your-api-key")
|
||||
result = firecrawl.scrape("https://example.com")
|
||||
|
||||
Check example.py for other usage examples.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, List, Union, BinaryIO
|
||||
import logging
|
||||
|
||||
|
||||
from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp
|
||||
from .v2 import FirecrawlClient as V2FirecrawlClient
|
||||
from .v2.client_async import AsyncFirecrawlClient
|
||||
from .v2.types import Document, ParseOptions, ScrapeOptions
|
||||
|
||||
logger = logging.getLogger("firecrawl")
|
||||
|
||||
class V1Proxy:
|
||||
"""Type-annotated proxy for v1 client methods."""
|
||||
_client: Optional[V1FirecrawlApp]
|
||||
|
||||
def __init__(self, client_instance: Optional[V1FirecrawlApp]):
|
||||
self._client = client_instance
|
||||
|
||||
if client_instance:
|
||||
self.scrape_url = client_instance.scrape_url
|
||||
self.crawl_url = client_instance.crawl_url
|
||||
self.batch_scrape_urls = client_instance.batch_scrape_urls
|
||||
self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
|
||||
self.async_crawl_url = client_instance.async_crawl_url
|
||||
self.check_crawl_status = client_instance.check_crawl_status
|
||||
self.map_url = client_instance.map_url
|
||||
self.extract = client_instance.extract
|
||||
self.deep_research = client_instance.deep_research
|
||||
self.generate_llms_text = client_instance.generate_llms_text
|
||||
|
||||
class V2Proxy:
|
||||
"""Proxy class that forwards method calls to the appropriate version client."""
|
||||
_client: Optional[V2FirecrawlClient]
|
||||
|
||||
def __init__(self, client_instance: Optional[V2FirecrawlClient]):
|
||||
self._client = client_instance
|
||||
|
||||
if client_instance:
|
||||
self.scrape = client_instance.scrape
|
||||
self.interact = client_instance.interact
|
||||
self.stop_interaction = client_instance.stop_interaction
|
||||
self.stop_interactive_browser = client_instance.stop_interactive_browser
|
||||
self.scrape_execute = self.interact
|
||||
self.delete_scrape_browser = self.stop_interaction
|
||||
self.parse = client_instance.parse
|
||||
self.search = client_instance.search
|
||||
self.crawl = client_instance.crawl
|
||||
self.start_crawl = client_instance.start_crawl
|
||||
self.get_crawl_status = client_instance.get_crawl_status
|
||||
self.get_crawl_status_page = client_instance.get_crawl_status_page
|
||||
self.cancel_crawl = client_instance.cancel_crawl
|
||||
self.get_crawl_errors = client_instance.get_crawl_errors
|
||||
self.get_active_crawls = client_instance.get_active_crawls
|
||||
self.active_crawls = client_instance.active_crawls
|
||||
self.crawl_params_preview = client_instance.crawl_params_preview
|
||||
|
||||
self.extract = client_instance.extract
|
||||
self.start_extract = client_instance.start_extract
|
||||
self.get_extract_status = client_instance.get_extract_status
|
||||
|
||||
self.agent = client_instance.agent
|
||||
self.start_agent = client_instance.start_agent
|
||||
self.get_agent_status = client_instance.get_agent_status
|
||||
self.cancel_agent = client_instance.cancel_agent
|
||||
|
||||
self.start_batch_scrape = client_instance.start_batch_scrape
|
||||
self.get_batch_scrape_status = client_instance.get_batch_scrape_status
|
||||
self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_page
|
||||
self.cancel_batch_scrape = client_instance.cancel_batch_scrape
|
||||
self.batch_scrape = client_instance.batch_scrape
|
||||
self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
|
||||
|
||||
self.map = client_instance.map
|
||||
self.create_monitor = client_instance.create_monitor
|
||||
self.list_monitors = client_instance.list_monitors
|
||||
self.get_monitor = client_instance.get_monitor
|
||||
self.update_monitor = client_instance.update_monitor
|
||||
self.delete_monitor = client_instance.delete_monitor
|
||||
self.run_monitor = client_instance.run_monitor
|
||||
self.list_monitor_checks = client_instance.list_monitor_checks
|
||||
self.get_monitor_check = client_instance.get_monitor_check
|
||||
self.get_concurrency = client_instance.get_concurrency
|
||||
self.get_credit_usage = client_instance.get_credit_usage
|
||||
self.get_token_usage = client_instance.get_token_usage
|
||||
self.get_queue_status = client_instance.get_queue_status
|
||||
|
||||
self.browser = client_instance.browser
|
||||
self.browser_execute = client_instance.browser_execute
|
||||
self.delete_browser = client_instance.delete_browser
|
||||
self.list_browsers = client_instance.list_browsers
|
||||
|
||||
self.watcher = client_instance.watcher
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Forward attribute access to the underlying client."""
|
||||
return getattr(self._client, name)
|
||||
|
||||
class AsyncV1Proxy:
|
||||
"""Type-annotated proxy for v1 client methods."""
|
||||
_client: Optional[AsyncV1FirecrawlApp]
|
||||
|
||||
def __init__(self, client_instance: Optional[AsyncV1FirecrawlApp]):
|
||||
self._client = client_instance
|
||||
|
||||
if client_instance:
|
||||
self.scrape_url = client_instance.scrape_url
|
||||
self.crawl_url = client_instance.crawl_url
|
||||
self.batch_scrape_urls = client_instance.batch_scrape_urls
|
||||
self.async_batch_scrape_urls = client_instance.async_batch_scrape_urls
|
||||
self.async_crawl_url = client_instance.async_crawl_url
|
||||
self.check_crawl_status = client_instance.check_crawl_status
|
||||
self.map_url = client_instance.map_url
|
||||
self.extract = client_instance.extract
|
||||
self.deep_research = client_instance.deep_research
|
||||
self.generate_llms_text = client_instance.generate_llms_text
|
||||
|
||||
class AsyncV2Proxy:
|
||||
"""Proxy class that forwards method calls to the appropriate version client."""
|
||||
_client: Optional[AsyncFirecrawlClient] = None
|
||||
|
||||
def __init__(self, client_instance: Optional[AsyncFirecrawlClient] = None):
|
||||
self._client = client_instance
|
||||
|
||||
if client_instance:
|
||||
self.scrape = client_instance.scrape
|
||||
self.interact = client_instance.interact
|
||||
self.stop_interaction = client_instance.stop_interaction
|
||||
self.stop_interactive_browser = client_instance.stop_interactive_browser
|
||||
self.scrape_execute = self.interact
|
||||
self.delete_scrape_browser = self.stop_interaction
|
||||
self.parse = client_instance.parse
|
||||
self.search = client_instance.search
|
||||
self.crawl = client_instance.crawl
|
||||
self.start_crawl = client_instance.start_crawl
|
||||
self.wait_crawl = client_instance.wait_crawl
|
||||
self.get_crawl_status = client_instance.get_crawl_status
|
||||
self.get_crawl_status_page = client_instance.get_crawl_status_page
|
||||
self.cancel_crawl = client_instance.cancel_crawl
|
||||
self.get_crawl_errors = client_instance.get_crawl_errors
|
||||
self.get_active_crawls = client_instance.get_active_crawls
|
||||
self.active_crawls = client_instance.active_crawls
|
||||
self.crawl_params_preview = client_instance.crawl_params_preview
|
||||
|
||||
self.extract = client_instance.extract
|
||||
self.start_extract = client_instance.start_extract
|
||||
self.get_extract_status = client_instance.get_extract_status
|
||||
|
||||
self.agent = client_instance.agent
|
||||
self.start_agent = client_instance.start_agent
|
||||
self.get_agent_status = client_instance.get_agent_status
|
||||
self.cancel_agent = client_instance.cancel_agent
|
||||
|
||||
self.start_batch_scrape = client_instance.start_batch_scrape
|
||||
self.get_batch_scrape_status = client_instance.get_batch_scrape_status
|
||||
self.get_batch_scrape_status_page = client_instance.get_batch_scrape_status_page
|
||||
self.cancel_batch_scrape = client_instance.cancel_batch_scrape
|
||||
self.wait_batch_scrape = client_instance.wait_batch_scrape
|
||||
self.batch_scrape = client_instance.batch_scrape
|
||||
self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
|
||||
|
||||
self.map = client_instance.map
|
||||
self.create_monitor = client_instance.create_monitor
|
||||
self.list_monitors = client_instance.list_monitors
|
||||
self.get_monitor = client_instance.get_monitor
|
||||
self.update_monitor = client_instance.update_monitor
|
||||
self.delete_monitor = client_instance.delete_monitor
|
||||
self.run_monitor = client_instance.run_monitor
|
||||
self.list_monitor_checks = client_instance.list_monitor_checks
|
||||
self.get_monitor_check = client_instance.get_monitor_check
|
||||
self.get_concurrency = client_instance.get_concurrency
|
||||
self.get_credit_usage = client_instance.get_credit_usage
|
||||
self.get_token_usage = client_instance.get_token_usage
|
||||
self.get_queue_status = client_instance.get_queue_status
|
||||
|
||||
self.browser = client_instance.browser
|
||||
self.browser_execute = client_instance.browser_execute
|
||||
self.delete_browser = client_instance.delete_browser
|
||||
self.list_browsers = client_instance.list_browsers
|
||||
|
||||
self.watcher = client_instance.watcher
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Forward attribute access to the underlying client."""
|
||||
if self._client:
|
||||
return getattr(self._client, name)
|
||||
raise AttributeError(f"Async v2 client not implemented yet: {name}")
|
||||
|
||||
|
||||
class Firecrawl:
|
||||
"""
|
||||
Unified Firecrawl client (v2 by default, v1 under ``.v1``).
|
||||
|
||||
Provides a single entrypoint that exposes the latest API directly while
|
||||
keeping a feature-frozen v1 available for incremental migration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str = None,
|
||||
api_url: str = "https://api.firecrawl.dev",
|
||||
timeout: float = None,
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
):
|
||||
"""Initialize the unified client.
|
||||
|
||||
Args:
|
||||
api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``)
|
||||
api_url: Base API URL (defaults to production)
|
||||
timeout: Default request timeout in seconds for all HTTP requests
|
||||
max_retries: Maximum number of retries for failed requests (default: 3)
|
||||
backoff_factor: Exponential backoff factor for retries (default: 0.5)
|
||||
"""
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
|
||||
# Initialize version-specific clients
|
||||
self._v1_client = V1FirecrawlApp(api_key=api_key, api_url=api_url) if V1FirecrawlApp else None
|
||||
self._v2_client = V2FirecrawlClient(
|
||||
api_key=api_key,
|
||||
api_url=api_url,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
backoff_factor=backoff_factor,
|
||||
) if V2FirecrawlClient else None
|
||||
|
||||
# Create version-specific proxies
|
||||
self.v1 = V1Proxy(self._v1_client) if self._v1_client else None
|
||||
self.v2 = V2Proxy(self._v2_client)
|
||||
|
||||
self.scrape = self._v2_client.scrape
|
||||
self.interact = self._v2_client.interact
|
||||
self.stop_interaction = self._v2_client.stop_interaction
|
||||
self.stop_interactive_browser = self._v2_client.stop_interactive_browser
|
||||
self.scrape_execute = self.interact
|
||||
self.delete_scrape_browser = self.stop_interaction
|
||||
self.parse = self._v2_client.parse
|
||||
self.search = self._v2_client.search
|
||||
self.map = self._v2_client.map
|
||||
self.create_monitor = self._v2_client.create_monitor
|
||||
self.list_monitors = self._v2_client.list_monitors
|
||||
self.get_monitor = self._v2_client.get_monitor
|
||||
self.update_monitor = self._v2_client.update_monitor
|
||||
self.delete_monitor = self._v2_client.delete_monitor
|
||||
self.run_monitor = self._v2_client.run_monitor
|
||||
self.list_monitor_checks = self._v2_client.list_monitor_checks
|
||||
self.get_monitor_check = self._v2_client.get_monitor_check
|
||||
|
||||
self.crawl = self._v2_client.crawl
|
||||
self.start_crawl = self._v2_client.start_crawl
|
||||
self.crawl_params_preview = self._v2_client.crawl_params_preview
|
||||
self.get_crawl_status = self._v2_client.get_crawl_status
|
||||
self.get_crawl_status_page = self._v2_client.get_crawl_status_page
|
||||
self.cancel_crawl = self._v2_client.cancel_crawl
|
||||
self.get_crawl_errors = self._v2_client.get_crawl_errors
|
||||
self.get_active_crawls = self._v2_client.get_active_crawls
|
||||
self.active_crawls = self._v2_client.active_crawls
|
||||
|
||||
self.start_batch_scrape = self._v2_client.start_batch_scrape
|
||||
self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
|
||||
self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_page
|
||||
self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
|
||||
self.batch_scrape = self._v2_client.batch_scrape
|
||||
self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
|
||||
|
||||
self.start_extract = self._v2_client.start_extract
|
||||
self.get_extract_status = self._v2_client.get_extract_status
|
||||
self.extract = self._v2_client.extract
|
||||
|
||||
self.start_agent = self._v2_client.start_agent
|
||||
self.get_agent_status = self._v2_client.get_agent_status
|
||||
self.cancel_agent = self._v2_client.cancel_agent
|
||||
self.agent = self._v2_client.agent
|
||||
|
||||
self.get_concurrency = self._v2_client.get_concurrency
|
||||
self.get_credit_usage = self._v2_client.get_credit_usage
|
||||
self.get_token_usage = self._v2_client.get_token_usage
|
||||
self.get_queue_status = self._v2_client.get_queue_status
|
||||
|
||||
self.browser = self._v2_client.browser
|
||||
self.browser_execute = self._v2_client.browser_execute
|
||||
self.delete_browser = self._v2_client.delete_browser
|
||||
self.list_browsers = self._v2_client.list_browsers
|
||||
|
||||
self.watcher = self._v2_client.watcher
|
||||
|
||||
self.scrape_url = self._v2_client.scrape_url
|
||||
self.crawl_url = self._v2_client.crawl_url
|
||||
self.map_url = self._v2_client.map_url
|
||||
self.async_crawl_url = self._v2_client.async_crawl_url
|
||||
self.check_crawl_status = self._v2_client.check_crawl_status
|
||||
self.check_crawl_errors = self._v2_client.check_crawl_errors
|
||||
self.batch_scrape_urls = self._v2_client.batch_scrape_urls
|
||||
self.async_batch_scrape_urls = self._v2_client.async_batch_scrape_urls
|
||||
self.check_batch_scrape_status = self._v2_client.check_batch_scrape_status
|
||||
self.check_batch_scrape_errors = self._v2_client.check_batch_scrape_errors
|
||||
|
||||
def parse(
|
||||
self,
|
||||
file: Union[str, Path, bytes, bytearray, BinaryIO],
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
options: Optional[ParseOptions] = None,
|
||||
) -> Document:
|
||||
"""Parse an uploaded file via the v2 parse endpoint."""
|
||||
return self._v2_client.parse(
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
options=options,
|
||||
)
|
||||
|
||||
class AsyncFirecrawl:
|
||||
"""Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str = None,
|
||||
api_url: str = "https://api.firecrawl.dev",
|
||||
timeout: float = None,
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
):
|
||||
self.api_key = api_key
|
||||
self.api_url = api_url
|
||||
|
||||
# Initialize version-specific clients
|
||||
self._v1_client = AsyncV1FirecrawlApp(api_key=api_key, api_url=api_url) if AsyncV1FirecrawlApp else None
|
||||
self._v2_client = AsyncFirecrawlClient(
|
||||
api_key=api_key,
|
||||
api_url=api_url,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
backoff_factor=backoff_factor,
|
||||
) if AsyncFirecrawlClient else None
|
||||
|
||||
# Create version-specific proxies
|
||||
self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None
|
||||
self.v2 = AsyncV2Proxy(self._v2_client)
|
||||
|
||||
# Expose v2 async surface directly on the top-level client for ergonomic access
|
||||
# Keep method names aligned with the sync client
|
||||
self.scrape = self._v2_client.scrape
|
||||
self.interact = self._v2_client.interact
|
||||
self.stop_interaction = self._v2_client.stop_interaction
|
||||
self.stop_interactive_browser = self._v2_client.stop_interactive_browser
|
||||
self.scrape_execute = self.interact
|
||||
self.delete_scrape_browser = self.stop_interaction
|
||||
self.parse = self._v2_client.parse
|
||||
self.search = self._v2_client.search
|
||||
self.map = self._v2_client.map
|
||||
self.create_monitor = self._v2_client.create_monitor
|
||||
self.list_monitors = self._v2_client.list_monitors
|
||||
self.get_monitor = self._v2_client.get_monitor
|
||||
self.update_monitor = self._v2_client.update_monitor
|
||||
self.delete_monitor = self._v2_client.delete_monitor
|
||||
self.run_monitor = self._v2_client.run_monitor
|
||||
self.list_monitor_checks = self._v2_client.list_monitor_checks
|
||||
self.get_monitor_check = self._v2_client.get_monitor_check
|
||||
|
||||
self.start_crawl = self._v2_client.start_crawl
|
||||
self.get_crawl_status = self._v2_client.get_crawl_status
|
||||
self.get_crawl_status_page = self._v2_client.get_crawl_status_page
|
||||
self.cancel_crawl = self._v2_client.cancel_crawl
|
||||
self.crawl = self._v2_client.crawl
|
||||
self.get_crawl_errors = self._v2_client.get_crawl_errors
|
||||
self.active_crawls = self._v2_client.active_crawls
|
||||
self.crawl_params_preview = self._v2_client.crawl_params_preview
|
||||
|
||||
self.start_batch_scrape = self._v2_client.start_batch_scrape
|
||||
self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
|
||||
self.get_batch_scrape_status_page = self._v2_client.get_batch_scrape_status_page
|
||||
self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
|
||||
self.batch_scrape = self._v2_client.batch_scrape
|
||||
self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
|
||||
|
||||
self.start_extract = self._v2_client.start_extract
|
||||
self.get_extract_status = self._v2_client.get_extract_status
|
||||
self.extract = self._v2_client.extract
|
||||
|
||||
self.start_agent = self._v2_client.start_agent
|
||||
self.get_agent_status = self._v2_client.get_agent_status
|
||||
self.cancel_agent = self._v2_client.cancel_agent
|
||||
self.agent = self._v2_client.agent
|
||||
|
||||
self.get_concurrency = self._v2_client.get_concurrency
|
||||
self.get_credit_usage = self._v2_client.get_credit_usage
|
||||
self.get_token_usage = self._v2_client.get_token_usage
|
||||
self.get_queue_status = self._v2_client.get_queue_status
|
||||
|
||||
self.browser = self._v2_client.browser
|
||||
self.browser_execute = self._v2_client.browser_execute
|
||||
self.delete_browser = self._v2_client.delete_browser
|
||||
self.list_browsers = self._v2_client.list_browsers
|
||||
|
||||
self.watcher = self._v2_client.watcher
|
||||
|
||||
self.scrape_url = self._v2_client.scrape_url
|
||||
self.crawl_url = self._v2_client.crawl_url
|
||||
self.map_url = self._v2_client.map_url
|
||||
self.async_crawl_url = self._v2_client.async_crawl_url
|
||||
self.check_crawl_status = self._v2_client.check_crawl_status
|
||||
self.check_crawl_errors = self._v2_client.check_crawl_errors
|
||||
self.batch_scrape_urls = self._v2_client.batch_scrape_urls
|
||||
self.async_batch_scrape_urls = self._v2_client.async_batch_scrape_urls
|
||||
self.check_batch_scrape_status = self._v2_client.check_batch_scrape_status
|
||||
self.check_batch_scrape_errors = self._v2_client.check_batch_scrape_errors
|
||||
|
||||
async def parse(
|
||||
self,
|
||||
file: Union[str, Path, bytes, bytearray, BinaryIO],
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
options: Optional[ParseOptions] = None,
|
||||
) -> Document:
|
||||
"""Parse an uploaded file via the v2 parse endpoint."""
|
||||
return await self._v2_client.parse(
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
options=options,
|
||||
)
|
||||
|
||||
# Export Firecrawl as an alias for FirecrawlApp
|
||||
FirecrawlApp = Firecrawl
|
||||
AsyncFirecrawlApp = AsyncFirecrawl
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Unified Firecrawl Types
|
||||
|
||||
This module provides unified access to Firecrawl types across all API versions.
|
||||
Currently exports v2 types as the primary interface.
|
||||
"""
|
||||
|
||||
from .v2.types import (
|
||||
# Base types
|
||||
BaseResponse,
|
||||
|
||||
# Document types
|
||||
Document,
|
||||
DocumentMetadata,
|
||||
|
||||
# Scrape types
|
||||
ScrapeFormats,
|
||||
ScrapeOptions,
|
||||
ScrapeRequest,
|
||||
ScrapeData,
|
||||
ScrapeResponse,
|
||||
ThreatProtectionOptions,
|
||||
|
||||
# Crawl types
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse,
|
||||
CrawlParamsRequest,
|
||||
CrawlParamsData,
|
||||
CrawlParamsResponse,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
|
||||
# Batch scrape types
|
||||
BatchScrapeRequest,
|
||||
BatchScrapeJob,
|
||||
BatchScrapeResponse,
|
||||
|
||||
# Map types
|
||||
MapOptions,
|
||||
MapRequest,
|
||||
MapData,
|
||||
MapResponse,
|
||||
|
||||
# Search types
|
||||
Source,
|
||||
SourceOption,
|
||||
Format,
|
||||
JsonFormat,
|
||||
QuestionFormat,
|
||||
HighlightsFormat,
|
||||
QueryFormat,
|
||||
FormatOption,
|
||||
SearchRequest,
|
||||
SearchResultWeb,
|
||||
SearchResultNews,
|
||||
SearchResultImages,
|
||||
SearchData,
|
||||
|
||||
# Action types
|
||||
WaitAction,
|
||||
ScreenshotAction,
|
||||
ClickAction,
|
||||
WriteAction,
|
||||
PressAction,
|
||||
ScrollAction,
|
||||
ScrapeAction,
|
||||
ExecuteJavascriptAction,
|
||||
PDFAction,
|
||||
|
||||
# Usage types
|
||||
QueueStatusResponse,
|
||||
|
||||
# Location and format types
|
||||
Location,
|
||||
|
||||
# Error types
|
||||
ErrorDetails,
|
||||
ErrorResponse,
|
||||
|
||||
# Job management types
|
||||
JobStatus,
|
||||
|
||||
# Webhook types
|
||||
WebhookData,
|
||||
|
||||
# Configuration types
|
||||
ClientConfig,
|
||||
AgentOptions,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Base types
|
||||
'BaseResponse',
|
||||
|
||||
# Document types
|
||||
'Document',
|
||||
'DocumentMetadata',
|
||||
|
||||
# Scrape types
|
||||
'ScrapeFormats',
|
||||
'ScrapeOptions',
|
||||
'ScrapeRequest',
|
||||
'ScrapeData',
|
||||
'ScrapeResponse',
|
||||
'ThreatProtectionOptions',
|
||||
|
||||
# Crawl types
|
||||
'CrawlRequest',
|
||||
'CrawlJob',
|
||||
'CrawlJobData',
|
||||
'CrawlResponse',
|
||||
'CrawlParamsRequest',
|
||||
'CrawlParamsData',
|
||||
'CrawlParamsResponse',
|
||||
'CrawlErrorsResponse',
|
||||
'ActiveCrawlsResponse',
|
||||
|
||||
# Batch scrape types
|
||||
'BatchScrapeRequest',
|
||||
'BatchScrapeJob',
|
||||
'BatchScrapeResponse',
|
||||
|
||||
# Map types
|
||||
'MapOptions',
|
||||
'MapRequest',
|
||||
'MapData',
|
||||
'MapResponse',
|
||||
|
||||
# Search types
|
||||
'Source',
|
||||
'SourceOption',
|
||||
'Format',
|
||||
'JsonFormat',
|
||||
'QuestionFormat',
|
||||
'HighlightsFormat',
|
||||
'QueryFormat',
|
||||
'FormatOption',
|
||||
'SearchRequest',
|
||||
'SearchResultWeb',
|
||||
'SearchResultNews',
|
||||
'SearchResultImages',
|
||||
'SearchData',
|
||||
|
||||
# Action types
|
||||
'WaitAction',
|
||||
'ScreenshotAction',
|
||||
'ClickAction',
|
||||
'WriteAction',
|
||||
'PressAction',
|
||||
'ScrollAction',
|
||||
'ScrapeAction',
|
||||
'ExecuteJavascriptAction',
|
||||
'PDFAction',
|
||||
|
||||
# Usage types
|
||||
'QueueStatusResponse',
|
||||
|
||||
# Location and format types
|
||||
'Location',
|
||||
|
||||
# Error types
|
||||
'ErrorDetails',
|
||||
'ErrorResponse',
|
||||
|
||||
# Job management types
|
||||
'JobStatus',
|
||||
|
||||
# Webhook types
|
||||
'WebhookData',
|
||||
|
||||
# Configuration types
|
||||
'ClientConfig',
|
||||
'AgentOptions',
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
"""
|
||||
Firecrawl v1 API (Legacy)
|
||||
|
||||
This module provides the legacy v1 API for backward compatibility.
|
||||
|
||||
Usage:
|
||||
from firecrawl.v1 import V1FirecrawlApp
|
||||
app = V1FirecrawlApp(api_key="your-api-key")
|
||||
result = app.scrape_url("https://example.com")
|
||||
"""
|
||||
|
||||
from .client import V1FirecrawlApp, AsyncV1FirecrawlApp, V1JsonConfig, V1ScrapeOptions, V1ChangeTrackingOptions
|
||||
|
||||
__all__ = ['V1FirecrawlApp', 'AsyncV1FirecrawlApp', 'V1JsonConfig', 'V1ScrapeOptions', 'V1ChangeTrackingOptions']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,4 @@
|
||||
from .client import FirecrawlClient
|
||||
from .client_async import AsyncFirecrawlClient
|
||||
|
||||
__all__ = ["FirecrawlClient", "AsyncFirecrawlClient"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,867 @@
|
||||
"""
|
||||
Async v2 client mirroring the regular client surface using true async HTTP transport.
|
||||
"""
|
||||
|
||||
import os
|
||||
import asyncio
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any, Union, Callable, Literal, BinaryIO
|
||||
from .types import (
|
||||
ParseOptions,
|
||||
ScrapeOptions,
|
||||
CrawlRequest,
|
||||
WebhookConfig,
|
||||
AgentWebhookConfig,
|
||||
MonitorWebhookConfig,
|
||||
SearchRequest,
|
||||
SearchData,
|
||||
SourceOption,
|
||||
CrawlResponse,
|
||||
CrawlJob,
|
||||
CrawlParamsRequest,
|
||||
CrawlParamsData,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
MapOptions,
|
||||
MapData,
|
||||
FormatOption,
|
||||
WaitAction,
|
||||
ScreenshotAction,
|
||||
ClickAction,
|
||||
WriteAction,
|
||||
PressAction,
|
||||
ScrollAction,
|
||||
ScrapeAction,
|
||||
ExecuteJavascriptAction,
|
||||
PDFAction,
|
||||
Location,
|
||||
PaginationConfig,
|
||||
ThreatProtectionOptions,
|
||||
AuditMetadata,
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
MonitorCreateRequest,
|
||||
MonitorNotification,
|
||||
MonitorSchedule,
|
||||
MonitorTarget,
|
||||
MonitorUpdateRequest,
|
||||
)
|
||||
from .utils.http_client import HttpClient
|
||||
from .utils.http_client_async import AsyncHttpClient
|
||||
|
||||
from .methods.aio import scrape as async_scrape # type: ignore[attr-defined]
|
||||
from .methods.aio import parse as async_parse # type: ignore[attr-defined]
|
||||
from .methods.aio import batch as async_batch # type: ignore[attr-defined]
|
||||
from .methods.aio import crawl as async_crawl # type: ignore[attr-defined]
|
||||
from .methods.aio import search as async_search # type: ignore[attr-defined]
|
||||
from .methods.aio import map as async_map # type: ignore[attr-defined]
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
from .methods.aio import extract as async_extract # type: ignore[attr-defined]
|
||||
from .methods.aio import agent as async_agent # type: ignore[attr-defined]
|
||||
from .methods.aio import browser as async_browser # type: ignore[attr-defined]
|
||||
from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]
|
||||
from .methods.aio import research as async_research # type: ignore[attr-defined]
|
||||
|
||||
from .client import _SCRAPE_OPTION_KEYS
|
||||
from .watcher_async import AsyncWatcher
|
||||
|
||||
class AsyncFirecrawlClient:
|
||||
@staticmethod
|
||||
def _is_cloud_service(url: str) -> bool:
|
||||
return "api.firecrawl.dev" in url.lower()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str] = None,
|
||||
api_url: str = "https://api.firecrawl.dev",
|
||||
timeout: Optional[float] = None,
|
||||
max_retries: int = 3,
|
||||
backoff_factor: float = 0.5,
|
||||
):
|
||||
if api_key is None:
|
||||
api_key = os.getenv("FIRECRAWL_API_KEY")
|
||||
if self._is_cloud_service(api_url) and not api_key:
|
||||
raise ValueError("API key is required for the cloud API. Set FIRECRAWL_API_KEY or pass api_key.")
|
||||
self.http_client = HttpClient(
|
||||
api_key,
|
||||
api_url,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
backoff_factor=backoff_factor,
|
||||
)
|
||||
self.async_http_client = AsyncHttpClient(
|
||||
api_key,
|
||||
api_url,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
backoff_factor=backoff_factor,
|
||||
)
|
||||
|
||||
# Scrape
|
||||
async def scrape(
|
||||
self,
|
||||
url: str,
|
||||
**kwargs,
|
||||
):
|
||||
options = ScrapeOptions(**{k: v for k, v in kwargs.items() if v is not None}) if kwargs else None
|
||||
return await async_scrape.scrape(self.async_http_client, url, options)
|
||||
|
||||
async def search_papers(self, query: str, **kwargs):
|
||||
return await async_research.search_papers(self.async_http_client, query, **kwargs)
|
||||
|
||||
async def inspect_paper(self, paper_id: str):
|
||||
return await async_research.inspect_paper(self.async_http_client, paper_id)
|
||||
|
||||
async def read_paper(self, paper_id: str, query: str, **kwargs):
|
||||
return await async_research.read_paper(self.async_http_client, paper_id, query, **kwargs)
|
||||
|
||||
async def related_papers(self, paper_id: str, intent: str, **kwargs):
|
||||
return await async_research.related_papers(self.async_http_client, paper_id, intent, **kwargs)
|
||||
|
||||
async def search_github(self, query: str, **kwargs):
|
||||
return await async_research.search_github(self.async_http_client, query, **kwargs)
|
||||
|
||||
async def interact(
|
||||
self,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
):
|
||||
return await async_scrape.interact(
|
||||
self.async_http_client,
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
async def stop_interaction(self, job_id: str):
|
||||
return await async_scrape.stop_interaction(
|
||||
self.async_http_client,
|
||||
job_id,
|
||||
)
|
||||
|
||||
async def stop_interactive_browser(self, job_id: str):
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await self.stop_interaction(job_id)
|
||||
|
||||
async def scrape_execute(
|
||||
self,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
):
|
||||
"""Deprecated alias for interact()."""
|
||||
return await self.interact(
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
async def delete_scrape_browser(self, job_id: str):
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await self.stop_interaction(job_id)
|
||||
|
||||
async def scrape_url(self, url: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer scrape()."""
|
||||
return await self.scrape(url, **kwargs)
|
||||
|
||||
async def crawl_url(self, url: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer crawl()."""
|
||||
return await self.crawl(url=url, **kwargs)
|
||||
|
||||
async def map_url(self, url: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer map()."""
|
||||
return await self.map(url, **kwargs)
|
||||
|
||||
async def async_crawl_url(self, url: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer start_crawl()."""
|
||||
return await self.start_crawl(url, **kwargs)
|
||||
|
||||
async def check_crawl_status(self, id: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer get_crawl_status()."""
|
||||
return await self.get_crawl_status(id, **kwargs)
|
||||
|
||||
async def check_crawl_errors(self, id: str):
|
||||
"""V1 compatibility alias for agent recovery. Prefer get_crawl_errors()."""
|
||||
return await self.get_crawl_errors(id)
|
||||
|
||||
async def batch_scrape_urls(self, urls, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer batch_scrape()."""
|
||||
return await self.batch_scrape(urls, **kwargs)
|
||||
|
||||
async def async_batch_scrape_urls(self, urls, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer start_batch_scrape()."""
|
||||
return await self.start_batch_scrape(urls, **kwargs)
|
||||
|
||||
async def check_batch_scrape_status(self, id: str, **kwargs):
|
||||
"""V1 compatibility alias for agent recovery. Prefer get_batch_scrape_status()."""
|
||||
return await self.get_batch_scrape_status(id, **kwargs)
|
||||
|
||||
async def check_batch_scrape_errors(self, id: str):
|
||||
"""V1 compatibility alias for agent recovery. Prefer get_batch_scrape_errors()."""
|
||||
return await self.get_batch_scrape_errors(id)
|
||||
|
||||
async def parse(
|
||||
self,
|
||||
file: Union[str, Path, bytes, bytearray, BinaryIO],
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
options: Optional[ParseOptions] = None,
|
||||
):
|
||||
return await async_parse.parse(
|
||||
self.async_http_client,
|
||||
file,
|
||||
options=options,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
|
||||
# Search
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
**kwargs,
|
||||
) -> SearchData:
|
||||
request = SearchRequest(query=query, **{k: v for k, v in kwargs.items() if v is not None})
|
||||
return await async_search.search(self.async_http_client, request)
|
||||
|
||||
async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
|
||||
if kwargs.get("scrape_options") is None:
|
||||
scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _SCRAPE_OPTION_KEYS and kwargs[k] is not None}
|
||||
if scrape_kwargs:
|
||||
kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)
|
||||
else:
|
||||
for k in list(kwargs):
|
||||
if k in _SCRAPE_OPTION_KEYS:
|
||||
kwargs.pop(k)
|
||||
|
||||
sitemap = kwargs.pop("sitemap", None)
|
||||
ignore_sitemap = kwargs.pop("ignore_sitemap", None)
|
||||
if sitemap is None and ignore_sitemap is not None:
|
||||
sitemap = "skip" if ignore_sitemap else "include"
|
||||
if sitemap is not None:
|
||||
kwargs["sitemap"] = sitemap
|
||||
|
||||
request = CrawlRequest(url=url, **kwargs)
|
||||
return await async_crawl.start_crawl(self.async_http_client, request)
|
||||
|
||||
async def wait_crawl(
|
||||
self,
|
||||
job_id: str,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Polls the status of a crawl job until it reaches a terminal state.
|
||||
|
||||
Args:
|
||||
job_id (str): The ID of the crawl job to poll.
|
||||
poll_interval (int, optional): Number of seconds to wait between polling attempts. Defaults to 2.
|
||||
timeout (Optional[int], optional): Maximum number of seconds to wait for the entire crawl job to complete before timing out. If None, waits indefinitely. Defaults to None.
|
||||
request_timeout (Optional[float], optional): Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout. If None, no per-request timeout is set. Defaults to None.
|
||||
|
||||
Returns:
|
||||
CrawlJob: The final status of the crawl job when it reaches a terminal state.
|
||||
|
||||
Raises:
|
||||
TimeoutError: If the crawl does not reach a terminal state within the specified timeout.
|
||||
|
||||
Terminal states:
|
||||
- "completed": The crawl finished successfully.
|
||||
- "failed": The crawl finished with an error.
|
||||
- "cancelled": The crawl was cancelled.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
while True:
|
||||
status = await async_crawl.get_crawl_status(
|
||||
self.async_http_client,
|
||||
job_id,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
if status.status in ["completed", "failed", "cancelled"]:
|
||||
return status
|
||||
if timeout and (time.monotonic() - start) > timeout:
|
||||
raise TimeoutError("Crawl wait timed out")
|
||||
await asyncio.sleep(poll_interval)
|
||||
|
||||
async def crawl(self, **kwargs) -> CrawlJob:
|
||||
# wrapper combining start and wait
|
||||
resp = await self.start_crawl(
|
||||
**{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout", "request_timeout")}
|
||||
)
|
||||
poll_interval = kwargs.get("poll_interval", 2)
|
||||
timeout = kwargs.get("timeout")
|
||||
request_timeout = kwargs.get("request_timeout")
|
||||
effective_request_timeout = request_timeout if request_timeout is not None else timeout
|
||||
return await self.wait_crawl(
|
||||
resp.id,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
request_timeout=effective_request_timeout,
|
||||
)
|
||||
|
||||
async def get_crawl_status(
|
||||
self,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with current status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
return await async_crawl.get_crawl_status(
|
||||
self.async_http_client,
|
||||
job_id,
|
||||
pagination_config=pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
async def get_crawl_status_page(
|
||||
self,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using a next URL.
|
||||
|
||||
Args:
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
"""
|
||||
return await async_crawl.get_crawl_status_page(
|
||||
self.async_http_client,
|
||||
next_url,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
async def cancel_crawl(self, job_id: str) -> bool:
|
||||
return await async_crawl.cancel_crawl(self.async_http_client, job_id)
|
||||
|
||||
async def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:
|
||||
req = CrawlParamsRequest(url=url, prompt=prompt)
|
||||
return await async_crawl.crawl_params_preview(self.async_http_client, req)
|
||||
|
||||
async def get_crawl_errors(self, crawl_id: str) -> CrawlErrorsResponse:
|
||||
return await async_crawl.get_crawl_errors(self.async_http_client, crawl_id)
|
||||
|
||||
async def get_active_crawls(self) -> ActiveCrawlsResponse:
|
||||
return await async_crawl.get_active_crawls(self.async_http_client)
|
||||
|
||||
async def active_crawls(self) -> ActiveCrawlsResponse:
|
||||
return await self.get_active_crawls()
|
||||
|
||||
# Map
|
||||
async def map(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
search: Optional[str] = None,
|
||||
include_subdomains: Optional[bool] = None,
|
||||
limit: Optional[int] = None,
|
||||
sitemap: Optional[Literal["only", "include", "skip"]] = None,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> MapData:
|
||||
options = MapOptions(
|
||||
search=search,
|
||||
include_subdomains=include_subdomains,
|
||||
limit=limit,
|
||||
sitemap=sitemap if sitemap is not None else "include",
|
||||
timeout=timeout,
|
||||
integration=integration,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection, audit_metadata]) else None
|
||||
return await async_map.map(self.async_http_client, url, options)
|
||||
|
||||
async def create_monitor(
|
||||
self,
|
||||
name: str,
|
||||
schedule: Union[MonitorSchedule, Dict[str, Any]],
|
||||
targets: List[Union[MonitorTarget, Dict[str, Any]]],
|
||||
*,
|
||||
webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
|
||||
notification: Optional[MonitorNotification] = None,
|
||||
retention_days: Optional[int] = None,
|
||||
goal: Optional[str] = None,
|
||||
judge_enabled: Optional[bool] = None,
|
||||
) -> Monitor:
|
||||
if isinstance(schedule, dict):
|
||||
schedule = MonitorSchedule(**schedule)
|
||||
request = MonitorCreateRequest(
|
||||
name=name,
|
||||
schedule=schedule,
|
||||
targets=targets,
|
||||
webhook=webhook,
|
||||
notification=notification,
|
||||
retention_days=retention_days,
|
||||
goal=goal,
|
||||
judge_enabled=judge_enabled,
|
||||
)
|
||||
return await async_monitor.create_monitor(self.async_http_client, request)
|
||||
|
||||
async def list_monitors(
|
||||
self,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[Monitor]:
|
||||
return await async_monitor.list_monitors(
|
||||
self.async_http_client,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
async def get_monitor(self, monitor_id: str) -> Monitor:
|
||||
return await async_monitor.get_monitor(self.async_http_client, monitor_id)
|
||||
|
||||
async def update_monitor(
|
||||
self,
|
||||
monitor_id: str,
|
||||
*,
|
||||
name: Optional[str] = None,
|
||||
status: Optional[Literal["active", "paused"]] = None,
|
||||
schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,
|
||||
webhook: Optional[Union[MonitorWebhookConfig, Dict[str, Any]]] = None,
|
||||
notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,
|
||||
targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,
|
||||
retention_days: Optional[int] = None,
|
||||
goal: Optional[str] = None,
|
||||
judge_enabled: Optional[bool] = None,
|
||||
) -> Monitor:
|
||||
if isinstance(schedule, dict):
|
||||
schedule = MonitorSchedule(**schedule)
|
||||
request = MonitorUpdateRequest(
|
||||
name=name,
|
||||
status=status,
|
||||
schedule=schedule,
|
||||
webhook=webhook,
|
||||
notification=notification,
|
||||
targets=targets,
|
||||
retention_days=retention_days,
|
||||
goal=goal,
|
||||
judge_enabled=judge_enabled,
|
||||
)
|
||||
return await async_monitor.update_monitor(
|
||||
self.async_http_client,
|
||||
monitor_id,
|
||||
request,
|
||||
)
|
||||
|
||||
async def delete_monitor(self, monitor_id: str) -> bool:
|
||||
return await async_monitor.delete_monitor(self.async_http_client, monitor_id)
|
||||
|
||||
async def run_monitor(self, monitor_id: str) -> MonitorCheck:
|
||||
return await async_monitor.run_monitor(self.async_http_client, monitor_id)
|
||||
|
||||
async def list_monitor_checks(
|
||||
self,
|
||||
monitor_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[MonitorCheck]:
|
||||
return await async_monitor.list_monitor_checks(
|
||||
self.async_http_client,
|
||||
monitor_id,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
async def get_monitor_check(
|
||||
self,
|
||||
monitor_id: str,
|
||||
check_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
skip: Optional[int] = None,
|
||||
status: Optional[Literal["same", "new", "changed", "removed", "error"]] = None,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> MonitorCheckDetail:
|
||||
return await async_monitor.get_monitor_check(
|
||||
self.async_http_client,
|
||||
monitor_id,
|
||||
check_id,
|
||||
limit=limit,
|
||||
skip=skip,
|
||||
status=status,
|
||||
pagination_config=pagination_config,
|
||||
)
|
||||
|
||||
async def start_batch_scrape(self, urls: List[str], **kwargs) -> Any:
|
||||
return await async_batch.start_batch_scrape(self.async_http_client, urls, **kwargs)
|
||||
|
||||
async def wait_batch_scrape(self, job_id: str, poll_interval: int = 2, timeout: Optional[int] = None) -> Any:
|
||||
start = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await async_batch.get_batch_scrape_status(self.async_http_client, job_id)
|
||||
if status.status in ["completed", "failed", "cancelled"]:
|
||||
return status
|
||||
if timeout and (asyncio.get_event_loop().time() - start) > timeout:
|
||||
raise TimeoutError("Batch wait timed out")
|
||||
await asyncio.sleep(poll_interval)
|
||||
|
||||
async def batch_scrape(self, urls: List[str], **kwargs) -> Any:
|
||||
# waiter wrapper
|
||||
start = await self.start_batch_scrape(urls, **{k: v for k, v in kwargs.items() if k not in ("poll_interval", "timeout")})
|
||||
job_id = start.id
|
||||
poll_interval = kwargs.get("poll_interval", 2)
|
||||
timeout = kwargs.get("timeout")
|
||||
return await self.wait_batch_scrape(job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
async def get_batch_scrape_status(
|
||||
self,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
):
|
||||
return await async_batch.get_batch_scrape_status(
|
||||
self.async_http_client,
|
||||
job_id,
|
||||
pagination_config=pagination_config
|
||||
)
|
||||
|
||||
async def get_batch_scrape_status_page(
|
||||
self,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
):
|
||||
return await async_batch.get_batch_scrape_status_page(
|
||||
self.async_http_client,
|
||||
next_url,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
async def cancel_batch_scrape(self, job_id: str) -> bool:
|
||||
return await async_batch.cancel_batch_scrape(self.async_http_client, job_id)
|
||||
|
||||
async def get_batch_scrape_errors(self, job_id: str) -> CrawlErrorsResponse:
|
||||
# Returns v2 errors structure; typed as CrawlErrorsResponse for parity
|
||||
return await async_batch.get_batch_scrape_errors(self.async_http_client, job_id) # type: ignore[return-value]
|
||||
|
||||
# Extract (proxy to v1 async) — deprecated
|
||||
async def extract(
|
||||
self,
|
||||
urls: Optional[List[str]] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional['ScrapeOptions'] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
):
|
||||
"""Extract structured data and wait until completion (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
return await async_extract.extract(
|
||||
self.async_http_client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
integration=integration,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
|
||||
async def get_extract_status(self, job_id: str):
|
||||
"""Get the current status (and data if completed) of an extract job (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
return await async_extract.get_extract_status(self.async_http_client, job_id)
|
||||
|
||||
async def start_extract(
|
||||
self,
|
||||
urls: Optional[List[str]] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional['ScrapeOptions'] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
):
|
||||
"""Start an extract job (non-blocking, async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
return await async_extract.start_extract(
|
||||
self.async_http_client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
|
||||
# Agent
|
||||
async def agent(
|
||||
self,
|
||||
urls: Optional[List[str]] = None,
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
):
|
||||
return await async_agent.agent(
|
||||
self.async_http_client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
|
||||
async def get_agent_status(self, job_id: str):
|
||||
return await async_agent.get_agent_status(self.async_http_client, job_id)
|
||||
|
||||
async def start_agent(
|
||||
self,
|
||||
urls: Optional[List[str]] = None,
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
):
|
||||
return await async_agent.start_agent(
|
||||
self.async_http_client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
|
||||
async def cancel_agent(self, job_id: str) -> bool:
|
||||
"""Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
job_id: Agent job ID
|
||||
|
||||
Returns:
|
||||
True if the agent was cancelled
|
||||
"""
|
||||
return await async_agent.cancel_agent(self.async_http_client, job_id)
|
||||
|
||||
# Browser
|
||||
async def browser(
|
||||
self,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
return await async_browser.browser(
|
||||
self.async_http_client,
|
||||
ttl=ttl,
|
||||
activity_ttl=activity_ttl,
|
||||
stream_web_view=stream_web_view,
|
||||
profile=profile,
|
||||
)
|
||||
|
||||
async def browser_execute(
|
||||
self,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
):
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
return await async_browser.browser_execute(
|
||||
self.async_http_client,
|
||||
session_id,
|
||||
code,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
async def delete_browser(self, session_id: str):
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
return await async_browser.delete_browser(
|
||||
self.async_http_client, session_id
|
||||
)
|
||||
|
||||
async def list_browsers(
|
||||
self,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
):
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
return await async_browser.list_browsers(
|
||||
self.async_http_client,
|
||||
status=status,
|
||||
)
|
||||
|
||||
# Usage endpoints
|
||||
async def get_concurrency(self):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_concurrency(self.async_http_client)
|
||||
|
||||
async def get_credit_usage(self):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_credit_usage(self.async_http_client)
|
||||
|
||||
async def get_token_usage(self):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_token_usage(self.async_http_client)
|
||||
|
||||
async def get_credit_usage_historical(self, by_api_key: bool = False):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_credit_usage_historical(self.async_http_client, by_api_key)
|
||||
|
||||
async def get_token_usage_historical(self, by_api_key: bool = False):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_token_usage_historical(self.async_http_client, by_api_key)
|
||||
|
||||
async def get_queue_status(self):
|
||||
from .methods.aio import usage as async_usage # type: ignore[attr-defined]
|
||||
return await async_usage.get_queue_status(self.async_http_client)
|
||||
|
||||
# Watcher (sync object usable from async contexts)
|
||||
def watcher(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
kind: Literal["crawl", "batch"] = "crawl",
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AsyncWatcher:
|
||||
return AsyncWatcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)
|
||||
@@ -0,0 +1,177 @@
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
import time
|
||||
|
||||
from ..types import AgentResponse, AgentWebhookConfig, AuditMetadata, ThreatProtectionOptions
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.error_handler import handle_response_error
|
||||
from ..utils.validation import _normalize_schema
|
||||
|
||||
|
||||
def _prepare_agent_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
body["schema"] = normalized_schema
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid schema type: {type(schema).__name__}. "
|
||||
"Schema must be a dict, Pydantic BaseModel class, or Pydantic model instance."
|
||||
)
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if max_credits is not None and max_credits > 0:
|
||||
body["maxCredits"] = max_credits
|
||||
if strict_constrain_to_urls is not None and strict_constrain_to_urls:
|
||||
body["strictConstrainToURLs"] = strict_constrain_to_urls
|
||||
if model is not None:
|
||||
body["model"] = model
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
body["webhook"] = webhook
|
||||
else:
|
||||
body["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
if threat_protection is not None:
|
||||
body["threatProtection"] = threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
if audit_metadata is not None:
|
||||
body["auditMetadata"] = audit_metadata.model_dump()
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_agent_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
return out
|
||||
|
||||
|
||||
def start_agent(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> AgentResponse:
|
||||
body = _prepare_agent_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
resp = client.post("/v2/agent", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
def get_agent_status(client: HttpClient, job_id: str) -> AgentResponse:
|
||||
resp = client.get(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent-status")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
def wait_agent(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AgentResponse:
|
||||
start_ts = time.time()
|
||||
while True:
|
||||
status = get_agent_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (time.time() - start_ts) > timeout:
|
||||
return status
|
||||
time.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
def agent(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> AgentResponse:
|
||||
started = start_agent(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return wait_agent(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
|
||||
def cancel_agent(client: HttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the agent job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the agent was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
resp = client.delete(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "cancel agent")
|
||||
return resp.json().get("success", False)
|
||||
@@ -0,0 +1 @@
|
||||
# Async (aio) method modules for v2
|
||||
@@ -0,0 +1,177 @@
|
||||
from typing import Any, Dict, List, Literal, Optional, Union
|
||||
import asyncio
|
||||
|
||||
from ...types import AgentResponse, AgentWebhookConfig, AuditMetadata, ThreatProtectionOptions
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import _normalize_schema
|
||||
|
||||
|
||||
def _prepare_agent_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
normalized_schema = _normalize_schema(schema)
|
||||
if normalized_schema is not None:
|
||||
body["schema"] = normalized_schema
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid schema type: {type(schema).__name__}. "
|
||||
"Schema must be a dict, Pydantic BaseModel class, or Pydantic model instance."
|
||||
)
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if max_credits is not None and max_credits > 0:
|
||||
body["maxCredits"] = max_credits
|
||||
if strict_constrain_to_urls is not None and strict_constrain_to_urls:
|
||||
body["strictConstrainToURLs"] = strict_constrain_to_urls
|
||||
if model is not None:
|
||||
body["model"] = model
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
body["webhook"] = webhook
|
||||
else:
|
||||
body["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
if threat_protection is not None:
|
||||
body["threatProtection"] = threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
if audit_metadata is not None:
|
||||
body["auditMetadata"] = audit_metadata.model_dump()
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_agent_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
return out
|
||||
|
||||
|
||||
async def start_agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> AgentResponse:
|
||||
body = _prepare_agent_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
resp = await client.post("/v2/agent", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def get_agent_status(client: AsyncHttpClient, job_id: str) -> AgentResponse:
|
||||
resp = await client.get(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "agent-status")
|
||||
payload = _normalize_agent_response_payload(resp.json())
|
||||
return AgentResponse(**payload)
|
||||
|
||||
|
||||
async def wait_agent(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> AgentResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_agent_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def agent(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: str,
|
||||
schema: Optional[Any] = None,
|
||||
integration: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
max_credits: Optional[int] = None,
|
||||
strict_constrain_to_urls: Optional[bool] = None,
|
||||
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
|
||||
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> AgentResponse:
|
||||
started = await start_agent(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
integration=integration,
|
||||
max_credits=max_credits,
|
||||
strict_constrain_to_urls=strict_constrain_to_urls,
|
||||
model=model,
|
||||
webhook=webhook,
|
||||
threat_protection=threat_protection,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_agent(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
|
||||
async def cancel_agent(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running agent job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the agent job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the agent was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
resp = await client.delete(f"/v2/agent/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "cancel agent")
|
||||
return resp.json().get("success", False)
|
||||
@@ -0,0 +1,242 @@
|
||||
from typing import Optional, List, Dict, Any
|
||||
from ...types import ScrapeOptions, WebhookConfig, Document, BatchScrapeResponse, BatchScrapeJob, PaginationConfig
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...methods.batch import validate_batch_urls
|
||||
import time
|
||||
|
||||
def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc in data_list or []:
|
||||
if isinstance(doc, dict):
|
||||
normalized = normalize_document_input(doc)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed"),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_batch_scrape_documents(body.get("data", []) or []),
|
||||
}
|
||||
|
||||
def _prepare(urls: List[str], *, options: Optional[ScrapeOptions] = None, **kwargs) -> Dict[str, Any]:
|
||||
if not urls:
|
||||
raise ValueError("URLs list cannot be empty")
|
||||
|
||||
validated_urls = validate_batch_urls([u.strip() if isinstance(u, str) else u for u in urls])
|
||||
payload: Dict[str, Any] = {"urls": validated_urls}
|
||||
if options:
|
||||
opts = prepare_scrape_options(options)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
if (v := kwargs.get("audit_metadata")) is not None:
|
||||
payload["auditMetadata"] = v.model_dump()
|
||||
if (w := kwargs.get("webhook")) is not None:
|
||||
payload["webhook"] = w if isinstance(w, str) else w.model_dump(exclude_none=True)
|
||||
if (v := kwargs.get("append_to_id")) is not None:
|
||||
payload["appendToId"] = v
|
||||
if (v := kwargs.get("ignore_invalid_urls")) is not None:
|
||||
payload["ignoreInvalidURLs"] = v
|
||||
if (v := kwargs.get("max_concurrency")) is not None:
|
||||
payload["maxConcurrency"] = v
|
||||
if (v := kwargs.get("zero_data_retention")) is not None:
|
||||
payload["zeroDataRetention"] = v
|
||||
if (v := kwargs.get("integration")) is not None:
|
||||
trimmed_integration = str(v).strip()
|
||||
if trimmed_integration:
|
||||
payload["integration"] = trimmed_integration
|
||||
return payload
|
||||
|
||||
|
||||
async def start_batch_scrape(client: AsyncHttpClient, urls: List[str], **kwargs) -> BatchScrapeResponse:
|
||||
payload = _prepare(urls, **kwargs)
|
||||
response = await client.post("/v2/batch/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start batch scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return BatchScrapeResponse(id=body.get("id"), url=body.get("url"), invalid_urls=body.get("invalidURLs"))
|
||||
|
||||
|
||||
async def get_batch_scrape_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Get the status of a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob containing job status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
docs = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
docs = await _fetch_all_batch_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
docs,
|
||||
pagination_config
|
||||
)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=docs,
|
||||
)
|
||||
|
||||
|
||||
async def get_batch_scrape_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Fetch a single page of batch scrape results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior batch scrape status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape status page")
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_batch_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of batch scrape results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits
|
||||
if (max_pages is not None) and (page_count >= max_pages):
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning(f"Failed to fetch next page: {response.status_code}")
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_batch_scrape_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_batch_scrape(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/batch/scrape/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel batch scrape")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def get_batch_scrape_errors(client: AsyncHttpClient, job_id: str) -> Dict[str, Any]:
|
||||
response = await client.get(f"/v2/batch/scrape/{job_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get batch scrape errors")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body
|
||||
@@ -0,0 +1,179 @@
|
||||
"""
|
||||
Async browser session methods for Firecrawl v2 API.
|
||||
|
||||
Provides async create, execute, delete, and list operations for browser sessions.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
from ...types import (
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
def _normalize_browser_create_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "cdpUrl" in out and "cdp_url" not in out:
|
||||
out["cdp_url"] = out["cdpUrl"]
|
||||
if "liveViewUrl" in out and "live_view_url" not in out:
|
||||
out["live_view_url"] = out["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in out and "interactive_live_view_url" not in out:
|
||||
out["interactive_live_view_url"] = out["interactiveLiveViewUrl"]
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_list_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessions" in out and isinstance(out["sessions"], list):
|
||||
normalized_sessions = []
|
||||
for s in out["sessions"]:
|
||||
ns = dict(s)
|
||||
if "cdpUrl" in ns and "cdp_url" not in ns:
|
||||
ns["cdp_url"] = ns["cdpUrl"]
|
||||
if "liveViewUrl" in ns and "live_view_url" not in ns:
|
||||
ns["live_view_url"] = ns["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in ns and "interactive_live_view_url" not in ns:
|
||||
ns["interactive_live_view_url"] = ns["interactiveLiveViewUrl"]
|
||||
if "streamWebView" in ns and "stream_web_view" not in ns:
|
||||
ns["stream_web_view"] = ns["streamWebView"]
|
||||
if "createdAt" in ns and "created_at" not in ns:
|
||||
ns["created_at"] = ns["createdAt"]
|
||||
if "lastActivity" in ns and "last_activity" not in ns:
|
||||
ns["last_activity"] = ns["lastActivity"]
|
||||
normalized_sessions.append(ns)
|
||||
out["sessions"] = normalized_sessions
|
||||
return out
|
||||
|
||||
|
||||
async def browser(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
) -> BrowserCreateResponse:
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
body: Dict[str, Any] = {}
|
||||
if ttl is not None:
|
||||
body["ttl"] = ttl
|
||||
if activity_ttl is not None:
|
||||
body["activityTtl"] = activity_ttl
|
||||
if stream_web_view is not None:
|
||||
body["streamWebView"] = stream_web_view
|
||||
if profile is not None:
|
||||
body["profile"] = {
|
||||
"name": profile["name"],
|
||||
"saveChanges": profile.get("save_changes", True),
|
||||
}
|
||||
|
||||
resp = await client.post("/v2/browser", body)
|
||||
payload = _normalize_browser_create_response(resp.json())
|
||||
return BrowserCreateResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_execute_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "exitCode" in out and "exit_code" not in out:
|
||||
out["exit_code"] = out["exitCode"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_delete_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessionDurationMs" in out and "session_duration_ms" not in out:
|
||||
out["session_duration_ms"] = out["sessionDurationMs"]
|
||||
if "creditsBilled" in out and "credits_billed" not in out:
|
||||
out["credits_billed"] = out["creditsBilled"]
|
||||
return out
|
||||
|
||||
|
||||
async def browser_execute(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
body: Dict[str, Any] = {
|
||||
"code": code,
|
||||
"language": language,
|
||||
}
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
|
||||
resp = await client.post(f"/v2/browser/{session_id}/execute", body)
|
||||
payload = _normalize_browser_execute_response(resp.json())
|
||||
return BrowserExecuteResponse(**payload)
|
||||
|
||||
|
||||
async def delete_browser(
|
||||
client: AsyncHttpClient,
|
||||
session_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
resp = await client.delete(f"/v2/browser/{session_id}")
|
||||
payload = _normalize_browser_delete_response(resp.json())
|
||||
return BrowserDeleteResponse(**payload)
|
||||
|
||||
|
||||
async def list_browsers(
|
||||
client: AsyncHttpClient,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
) -> BrowserListResponse:
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
endpoint = "/v2/browser"
|
||||
if status is not None:
|
||||
endpoint = f"{endpoint}?status={status}"
|
||||
|
||||
resp = await client.get(endpoint)
|
||||
payload = _normalize_browser_list_response(resp.json())
|
||||
return BrowserListResponse(**payload)
|
||||
@@ -0,0 +1,414 @@
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ...types import (
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse,
|
||||
Document,
|
||||
CrawlParamsRequest,
|
||||
CrawlParamsData,
|
||||
WebhookConfig,
|
||||
CrawlErrorsResponse,
|
||||
ActiveCrawlsResponse,
|
||||
ActiveCrawl,
|
||||
PaginationConfig,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.normalize import normalize_document_input
|
||||
import time
|
||||
|
||||
|
||||
def _prepare_crawl_request(request: CrawlRequest) -> dict:
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if request.limit is not None and request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data = {"url": request.url}
|
||||
if request.prompt:
|
||||
data["prompt"] = request.prompt
|
||||
if request.scrape_options is not None:
|
||||
opts = prepare_scrape_options(request.scrape_options)
|
||||
if opts:
|
||||
data["scrapeOptions"] = opts
|
||||
# Webhook conversion
|
||||
if request.webhook is not None:
|
||||
if isinstance(request.webhook, str):
|
||||
data["webhook"] = request.webhook
|
||||
else:
|
||||
data["webhook"] = request.webhook.model_dump(exclude_none=True)
|
||||
request_data = request.model_dump(exclude_none=True, exclude_unset=True)
|
||||
request_data.pop("url", None)
|
||||
request_data.pop("prompt", None)
|
||||
request_data.pop("scrape_options", None)
|
||||
field_mappings = {
|
||||
"include_paths": "includePaths",
|
||||
"exclude_paths": "excludePaths",
|
||||
"max_discovery_depth": "maxDiscoveryDepth",
|
||||
"sitemap": "sitemap",
|
||||
"ignore_query_parameters": "ignoreQueryParameters",
|
||||
"deduplicate_similar_urls": "deduplicateSimilarURLs",
|
||||
"crawl_entire_domain": "crawlEntireDomain",
|
||||
"allow_external_links": "allowExternalLinks",
|
||||
"allow_subdomains": "allowSubdomains",
|
||||
"ignore_robots_txt": "ignoreRobotsTxt",
|
||||
"robots_user_agent": "robotsUserAgent",
|
||||
"delay": "delay",
|
||||
"max_concurrency": "maxConcurrency",
|
||||
"regex_on_full_url": "regexOnFullURL",
|
||||
"zero_data_retention": "zeroDataRetention",
|
||||
}
|
||||
for snake, camel in field_mappings.items():
|
||||
if snake in request_data:
|
||||
data[camel] = request_data.pop(snake)
|
||||
data.update(request_data)
|
||||
if getattr(request, "integration", None) is not None:
|
||||
data["integration"] = str(getattr(request, "integration")).strip()
|
||||
return data
|
||||
|
||||
|
||||
def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc_data in data_list or []:
|
||||
if isinstance(doc_data, dict):
|
||||
normalized = normalize_document_input(doc_data)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_crawl_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed", 0),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_crawl_documents(body.get("data", [])),
|
||||
}
|
||||
|
||||
|
||||
async def start_crawl(client: AsyncHttpClient, request: CrawlRequest) -> CrawlResponse:
|
||||
"""
|
||||
Start a crawl job for a website.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
|
||||
Returns:
|
||||
CrawlResponse with job information
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl operation fails to start
|
||||
"""
|
||||
payload = _prepare_crawl_request(request)
|
||||
response = await client.post("/v2/crawl", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "start crawl")
|
||||
body = response.json()
|
||||
if body.get("success"):
|
||||
return CrawlResponse(id=body.get("id"), url=body.get("url"))
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
async def get_crawl_status(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with job information
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
documents = await _fetch_all_pages_async(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
async def get_crawl_status_page(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = await client.get(next_url, timeout=request_timeout)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get crawl status page")
|
||||
body = response.json()
|
||||
payload = _parse_crawl_status_response(body)
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_all_pages_async(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of crawl results asynchronously.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Optional timeout (in seconds) for the underlying HTTP request
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = await client.get(current_url, timeout=request_timeout)
|
||||
|
||||
if response.status_code >= 400:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_crawl_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if (max_results is not None) and (len(documents) >= max_results):
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
async def cancel_crawl(client: AsyncHttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
True if cancellation was successful
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation operation fails
|
||||
"""
|
||||
response = await client.delete(f"/v2/crawl/{job_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "cancel crawl")
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
|
||||
"""
|
||||
Preview crawl parameters before starting a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: CrawlParamsRequest containing URL and prompt
|
||||
|
||||
Returns:
|
||||
CrawlParamsData containing crawl configuration
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the parameter preview fails
|
||||
"""
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
if not request.prompt or not request.prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
payload = {"url": request.url, "prompt": request.prompt}
|
||||
response = await client.post("/v2/crawl/params-preview", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "crawl params preview")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
params_data = body.get("data", {})
|
||||
converted: Dict[str, Any] = {}
|
||||
mapping = {
|
||||
"includePaths": "include_paths",
|
||||
"excludePaths": "exclude_paths",
|
||||
"maxDiscoveryDepth": "max_discovery_depth",
|
||||
"sitemap": "sitemap",
|
||||
"ignoreQueryParameters": "ignore_query_parameters",
|
||||
"deduplicateSimilarURLs": "deduplicate_similar_urls",
|
||||
"crawlEntireDomain": "crawl_entire_domain",
|
||||
"allowExternalLinks": "allow_external_links",
|
||||
"allowSubdomains": "allow_subdomains",
|
||||
"ignoreRobotsTxt": "ignore_robots_txt",
|
||||
"robotsUserAgent": "robots_user_agent",
|
||||
"maxConcurrency": "max_concurrency",
|
||||
"scrapeOptions": "scrape_options",
|
||||
"zeroDataRetention": "zero_data_retention",
|
||||
}
|
||||
for camel, snake in mapping.items():
|
||||
if camel in params_data:
|
||||
converted[snake] = params_data[camel]
|
||||
if "webhook" in params_data:
|
||||
wk = params_data["webhook"]
|
||||
converted["webhook"] = wk
|
||||
if "warning" in body:
|
||||
converted["warning"] = body["warning"]
|
||||
return CrawlParamsData(**converted)
|
||||
|
||||
|
||||
async def get_crawl_errors(client: AsyncHttpClient, crawl_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors from a crawl job.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
crawl_id: ID of the crawl job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse with errors and robots blocked
|
||||
|
||||
Raises:
|
||||
Exception: If the error check operation fails
|
||||
"""
|
||||
response = await client.get(f"/v2/crawl/{crawl_id}/errors")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "check crawl errors")
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
|
||||
|
||||
async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
|
||||
"""
|
||||
Get active crawl jobs.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
|
||||
Returns:
|
||||
ActiveCrawlsResponse with active crawl jobs
|
||||
|
||||
Raises:
|
||||
Exception: If the active crawl jobs operation fails
|
||||
"""
|
||||
response = await client.get("/v2/crawl/active")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "get active crawls")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
crawls_in = body.get("crawls", [])
|
||||
normalized = []
|
||||
for c in crawls_in:
|
||||
if isinstance(c, dict):
|
||||
normalized.append({
|
||||
"id": c.get("id"),
|
||||
"team_id": c.get("teamId", c.get("team_id")),
|
||||
"url": c.get("url"),
|
||||
"options": c.get("options"),
|
||||
})
|
||||
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
|
||||
@@ -0,0 +1,173 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from ...types import ExtractResponse, ScrapeOptions, ThreatProtectionOptions
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
_EXTRACT_DEPRECATION_MSG = (
|
||||
"The extract endpoint is in maintenance mode and its use is discouraged. "
|
||||
"Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
|
||||
"to find a replacement."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_extract_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
if prompt is not None:
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
body["schema"] = schema
|
||||
if system_prompt is not None:
|
||||
body["systemPrompt"] = system_prompt
|
||||
if allow_external_links is not None:
|
||||
body["allowExternalLinks"] = allow_external_links
|
||||
if enable_web_search is not None:
|
||||
body["enableWebSearch"] = enable_web_search
|
||||
if show_sources is not None:
|
||||
body["showSources"] = show_sources
|
||||
if ignore_invalid_urls is not None:
|
||||
body["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if scrape_options is not None:
|
||||
prepared = prepare_scrape_options(scrape_options)
|
||||
if prepared:
|
||||
body["scrapeOptions"] = prepared
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if threat_protection is not None:
|
||||
body["threatProtection"] = threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
return body
|
||||
|
||||
|
||||
async def start_extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Start an extract job (non-blocking, async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
body = _prepare_extract_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
resp = await client.post("/v2/extract", body)
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def get_extract_status(client: AsyncHttpClient, job_id: str) -> ExtractResponse:
|
||||
"""Get the current status of an extract job (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
resp = await client.get(f"/v2/extract/{job_id}")
|
||||
return ExtractResponse(**resp.json())
|
||||
|
||||
|
||||
async def wait_extract(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> ExtractResponse:
|
||||
start_ts = asyncio.get_event_loop().time()
|
||||
while True:
|
||||
status = await get_extract_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (asyncio.get_event_loop().time() - start_ts) > timeout:
|
||||
return status
|
||||
await asyncio.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
async def extract(
|
||||
client: AsyncHttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Extract structured data and wait until completion (async).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
started = await start_extract(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return await wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
from typing import Optional, Dict, Any
|
||||
from ...types import MapOptions, MapData, LinkResult
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
|
||||
|
||||
def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
data: Dict[str, Any] = {}
|
||||
if getattr(options, "sitemap", None) is not None:
|
||||
data["sitemap"] = options.sitemap
|
||||
if options.search is not None:
|
||||
data["search"] = options.search
|
||||
if options.include_subdomains is not None:
|
||||
data["includeSubdomains"] = options.include_subdomains
|
||||
if options.ignore_query_parameters is not None:
|
||||
data["ignoreQueryParameters"] = options.ignore_query_parameters
|
||||
if options.limit is not None:
|
||||
if options.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data["limit"] = options.limit
|
||||
if options.timeout is not None:
|
||||
data["timeout"] = options.timeout
|
||||
if options.integration is not None:
|
||||
data["integration"] = options.integration.strip()
|
||||
if options.location is not None:
|
||||
data["location"] = options.location.model_dump(exclude_none=True)
|
||||
if options.threat_protection is not None:
|
||||
data["threatProtection"] = options.threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
if options.audit_metadata is not None:
|
||||
data["auditMetadata"] = options.audit_metadata.model_dump()
|
||||
payload.update(data)
|
||||
return payload
|
||||
|
||||
|
||||
async def map(client: AsyncHttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
|
||||
request_data = _prepare_map_request(url, options)
|
||||
response = await client.post("/v2/map", request_data)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "map")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
# data = body.get("data", {})
|
||||
# result_links: list[LinkResult] = []
|
||||
# for item in data.get("links", []):
|
||||
# if isinstance(item, dict):
|
||||
# result_links.append(
|
||||
# LinkResult(
|
||||
# url=item.get("url", ""),
|
||||
# title=item.get("title"),
|
||||
# description=item.get("description"),
|
||||
# )
|
||||
# )
|
||||
# elif isinstance(item, str):
|
||||
# result_links.append(LinkResult(url=item))
|
||||
|
||||
result_links: list[LinkResult] = []
|
||||
for item in body.get("links", []):
|
||||
if isinstance(item, dict):
|
||||
result_links.append(LinkResult(url=item.get("url", ""), title=item.get("title"), description=item.get("description")))
|
||||
elif isinstance(item, str):
|
||||
result_links.append(LinkResult(url=item))
|
||||
|
||||
return MapData(links=result_links)
|
||||
@@ -0,0 +1,215 @@
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ...types import (
|
||||
Monitor,
|
||||
MonitorCheck,
|
||||
MonitorCheckDetail,
|
||||
MonitorCheckPage,
|
||||
MonitorCreateRequest,
|
||||
PaginationConfig,
|
||||
MonitorTarget,
|
||||
MonitorUpdateRequest,
|
||||
ScrapeOptions,
|
||||
)
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.validation import prepare_scrape_options
|
||||
|
||||
|
||||
def _dump(value: Any) -> Any:
|
||||
if isinstance(value, ScrapeOptions):
|
||||
return prepare_scrape_options(value)
|
||||
if isinstance(value, MonitorTarget):
|
||||
data = value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value.scrape_options, ScrapeOptions):
|
||||
data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
|
||||
return _prepare_target(data)
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(exclude_none=True, by_alias=True)
|
||||
if isinstance(value, list):
|
||||
return [_dump(item) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {key: _dump(item) for key, item in value.items() if item is not None}
|
||||
return value
|
||||
|
||||
|
||||
def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prepared = dict(target)
|
||||
if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
|
||||
prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
|
||||
if "crawlOptions" in prepared:
|
||||
prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
|
||||
return prepared
|
||||
|
||||
|
||||
def _prepare_payload(request: Any) -> Dict[str, Any]:
|
||||
payload = _dump(request)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Monitor request must be an object")
|
||||
if "targets" in payload:
|
||||
payload["targets"] = [
|
||||
_prepare_target(_dump(target))
|
||||
for target in payload.get("targets", [])
|
||||
]
|
||||
return payload
|
||||
|
||||
|
||||
async def _data_or_error(response, action: str) -> Any:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return body.get("data")
|
||||
|
||||
|
||||
async def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, action)
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
data = body.get("data") or {}
|
||||
if body.get("next") is not None:
|
||||
data["next"] = body.get("next")
|
||||
return data
|
||||
|
||||
|
||||
async def _fetch_all_monitor_check_pages(
|
||||
client: AsyncHttpClient,
|
||||
next_url: str,
|
||||
initial_pages: List[MonitorCheckPage],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> List[MonitorCheckPage]:
|
||||
pages = initial_pages.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
if max_pages is not None and page_count >= max_pages:
|
||||
break
|
||||
if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
response = await client.get(current_url)
|
||||
if response.status_code >= 400:
|
||||
break
|
||||
try:
|
||||
data = await _monitor_check_data_or_error(response, "get monitor check page")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
for page in data.get("pages") or []:
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
pages.append(MonitorCheckPage(**page))
|
||||
|
||||
if max_results is not None and len(pages) >= max_results:
|
||||
break
|
||||
|
||||
current_url = data.get("next")
|
||||
page_count += 1
|
||||
|
||||
return pages
|
||||
|
||||
|
||||
async def create_monitor(client: AsyncHttpClient, request: MonitorCreateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def list_monitors(client: AsyncHttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor{suffix}"), "list monitors")
|
||||
return [Monitor(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor(client: AsyncHttpClient, monitor_id: str) -> Monitor:
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def update_monitor(client: AsyncHttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
|
||||
data = await _data_or_error(await client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
|
||||
return Monitor(**data)
|
||||
|
||||
|
||||
async def delete_monitor(client: AsyncHttpClient, monitor_id: str) -> bool:
|
||||
response = await client.delete(f"/v2/monitor/{monitor_id}")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "delete monitor")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return True
|
||||
|
||||
|
||||
async def run_monitor(client: AsyncHttpClient, monitor_id: str) -> MonitorCheck:
|
||||
data = await _data_or_error(await client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
|
||||
return MonitorCheck(**data)
|
||||
|
||||
|
||||
async def list_monitor_checks(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> List[MonitorCheck]:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if offset is not None:
|
||||
params.append(f"offset={offset}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
|
||||
return [MonitorCheck(**item) for item in data or []]
|
||||
|
||||
|
||||
async def get_monitor_check(
|
||||
client: AsyncHttpClient,
|
||||
monitor_id: str,
|
||||
check_id: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
skip: Optional[int] = None,
|
||||
status: Optional[str] = None,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
) -> MonitorCheckDetail:
|
||||
params = []
|
||||
if limit is not None:
|
||||
params.append(f"limit={limit}")
|
||||
if skip is not None:
|
||||
params.append(f"skip={skip}")
|
||||
if status is not None:
|
||||
params.append(f"status={status}")
|
||||
suffix = f"?{'&'.join(params)}" if params else ""
|
||||
data = await _monitor_check_data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
|
||||
detail = MonitorCheckDetail(**data)
|
||||
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and detail.next and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(detail.pages) >= pagination_config.max_results
|
||||
):
|
||||
detail.pages = await _fetch_all_monitor_check_pages(
|
||||
client,
|
||||
detail.next,
|
||||
detail.pages,
|
||||
pagination_config,
|
||||
)
|
||||
detail.next = None
|
||||
|
||||
return detail
|
||||
@@ -0,0 +1,63 @@
|
||||
import asyncio
|
||||
from functools import partial
|
||||
import json
|
||||
from typing import Optional, Dict, Any, Tuple
|
||||
|
||||
from ...types import Document, ParseOptions
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ..parse import (
|
||||
ParseFileInput,
|
||||
_prepare_file_payload,
|
||||
_prepare_parse_options_payload,
|
||||
)
|
||||
|
||||
async def _prepare_parse_request(
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:
|
||||
request_data = _prepare_parse_options_payload(options)
|
||||
multipart_fields = {"options": json.dumps(request_data)}
|
||||
loop = asyncio.get_running_loop()
|
||||
multipart_files = await loop.run_in_executor(
|
||||
None,
|
||||
partial(
|
||||
_prepare_file_payload,
|
||||
file,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
),
|
||||
)
|
||||
return multipart_fields, multipart_files
|
||||
|
||||
|
||||
async def parse(
|
||||
client: AsyncHttpClient,
|
||||
file: ParseFileInput,
|
||||
options: Optional[ParseOptions] = None,
|
||||
*,
|
||||
filename: Optional[str] = None,
|
||||
content_type: Optional[str] = None,
|
||||
) -> Document:
|
||||
fields, files = await _prepare_parse_request(
|
||||
file,
|
||||
options,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
response = await client.post_multipart("/v2/parse", data=fields, files=files)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "parse")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Async research functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
from ...utils import handle_response_error
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.get_version import get_version
|
||||
|
||||
|
||||
BASE = "/v2/search/research"
|
||||
ORIGIN = f"python-sdk@{get_version()}"
|
||||
|
||||
|
||||
def _query(params: Dict[str, Any]) -> str:
|
||||
pairs: List[str] = []
|
||||
for key, value in params.items():
|
||||
if value is None:
|
||||
continue
|
||||
values = value if isinstance(value, list) else [value]
|
||||
for item in values:
|
||||
if item is not None:
|
||||
pairs.append(f"{quote(str(key), safe='')}={quote(str(item), safe='')}")
|
||||
return ("?" + "&".join(pairs)) if pairs else ""
|
||||
|
||||
|
||||
async def _get(client: AsyncHttpClient, path: str) -> Dict[str, Any]:
|
||||
response = await client.get(path)
|
||||
if response.status_code != 200:
|
||||
handle_response_error(response, "research")
|
||||
return response.json()
|
||||
|
||||
|
||||
async def search_papers(
|
||||
client: AsyncHttpClient,
|
||||
query: str,
|
||||
*,
|
||||
k: Optional[int] = None,
|
||||
authors: Optional[List[str]] = None,
|
||||
categories: Optional[List[str]] = None,
|
||||
from_date: Optional[str] = None,
|
||||
to_date: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
return await _get(
|
||||
client,
|
||||
BASE
|
||||
+ "/papers"
|
||||
+ _query(
|
||||
{
|
||||
"query": query,
|
||||
"k": k,
|
||||
"authors": authors,
|
||||
"categories": categories,
|
||||
"from": from_date,
|
||||
"to": to_date,
|
||||
"origin": ORIGIN,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def inspect_paper(client: AsyncHttpClient, paper_id: str) -> Dict[str, Any]:
|
||||
return await _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}")
|
||||
|
||||
|
||||
async def read_paper(
|
||||
client: AsyncHttpClient,
|
||||
paper_id: str,
|
||||
query: str,
|
||||
*,
|
||||
k: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
return await _get(
|
||||
client,
|
||||
f"{BASE}/papers/{quote(paper_id, safe='')}"
|
||||
+ _query({"query": query, "k": k, "origin": ORIGIN}),
|
||||
)
|
||||
|
||||
|
||||
async def related_papers(
|
||||
client: AsyncHttpClient,
|
||||
paper_id: str,
|
||||
intent: str,
|
||||
*,
|
||||
mode: Optional[str] = None,
|
||||
k: Optional[int] = None,
|
||||
rerank: Optional[bool] = None,
|
||||
anchor: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
return await _get(
|
||||
client,
|
||||
f"{BASE}/papers/{quote(paper_id, safe='')}/similar"
|
||||
+ _query(
|
||||
{
|
||||
"intent": intent,
|
||||
"mode": mode,
|
||||
"k": k,
|
||||
"rerank": None if rerank is None else str(rerank).lower(),
|
||||
"anchor": anchor,
|
||||
"origin": ORIGIN,
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def search_github(
|
||||
client: AsyncHttpClient,
|
||||
query: str,
|
||||
*,
|
||||
k: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
return await _get(
|
||||
client,
|
||||
BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}),
|
||||
)
|
||||
@@ -0,0 +1,146 @@
|
||||
from typing import Optional, Dict, Any, Literal
|
||||
from ...types import (
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
)
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.validation import prepare_scrape_options, validate_scrape_options
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
|
||||
|
||||
async def _prepare_scrape_request(url: str, options: Optional[ScrapeOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
if options is not None:
|
||||
validated = validate_scrape_options(options)
|
||||
if validated is not None:
|
||||
opts = prepare_scrape_options(validated)
|
||||
if opts:
|
||||
payload.update(opts)
|
||||
return payload
|
||||
|
||||
|
||||
async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOptions] = None) -> Document:
|
||||
payload = await _prepare_scrape_request(url, options)
|
||||
response = await client.post("/v2/scrape", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "scrape")
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
document_data = body.get("data", {})
|
||||
normalized = normalize_document_input(document_data)
|
||||
return Document(**normalized)
|
||||
|
||||
|
||||
async def interact(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
has_code = code and code.strip()
|
||||
has_prompt = prompt and prompt.strip()
|
||||
if not has_code and not has_prompt:
|
||||
raise ValueError("Either 'code' or 'prompt' must be provided")
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"language": language,
|
||||
}
|
||||
if has_code:
|
||||
payload["code"] = code
|
||||
if has_prompt:
|
||||
payload["prompt"] = prompt
|
||||
if timeout is not None:
|
||||
payload["timeout"] = timeout
|
||||
if origin is not None:
|
||||
payload["origin"] = origin
|
||||
|
||||
response = await client.post(f"/v2/scrape/{job_id}/interact", payload)
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "interact with scrape browser")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
normalized = dict(body)
|
||||
if "exitCode" in normalized and "exit_code" not in normalized:
|
||||
normalized["exit_code"] = normalized["exitCode"]
|
||||
if "cdpUrl" in normalized and "cdp_url" not in normalized:
|
||||
normalized["cdp_url"] = normalized["cdpUrl"]
|
||||
if "liveViewUrl" in normalized and "live_view_url" not in normalized:
|
||||
normalized["live_view_url"] = normalized["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in normalized and "interactive_live_view_url" not in normalized:
|
||||
normalized["interactive_live_view_url"] = normalized["interactiveLiveViewUrl"]
|
||||
return BrowserExecuteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interaction(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
if not job_id or not job_id.strip():
|
||||
raise ValueError("Job ID cannot be empty")
|
||||
|
||||
response = await client.delete(f"/v2/scrape/{job_id}/interact")
|
||||
if response.status_code >= 400:
|
||||
handle_response_error(response, "stop interaction")
|
||||
|
||||
body = response.json()
|
||||
normalized = dict(body)
|
||||
if "sessionDurationMs" in normalized and "session_duration_ms" not in normalized:
|
||||
normalized["session_duration_ms"] = normalized["sessionDurationMs"]
|
||||
if "creditsBilled" in normalized and "credits_billed" not in normalized:
|
||||
normalized["credits_billed"] = normalized["creditsBilled"]
|
||||
|
||||
return BrowserDeleteResponse(**normalized)
|
||||
|
||||
|
||||
async def stop_interactive_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
|
||||
async def scrape_execute(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
code: Optional[str] = None,
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
language: Literal["python", "node", "bash"] = "node",
|
||||
timeout: Optional[int] = None,
|
||||
origin: Optional[str] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Deprecated alias for interact()."""
|
||||
return await interact(
|
||||
client,
|
||||
job_id,
|
||||
code,
|
||||
prompt=prompt,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
origin=origin,
|
||||
)
|
||||
|
||||
|
||||
async def delete_scrape_browser(
|
||||
client: AsyncHttpClient,
|
||||
job_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Deprecated alias for stop_interaction()."""
|
||||
return await stop_interaction(client, job_id)
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
from typing import Dict, Any, Union, List, TypeVar, Type
|
||||
from ...types import (
|
||||
SearchRequest,
|
||||
SearchData,
|
||||
Document,
|
||||
SearchResultWeb,
|
||||
SearchResultNews,
|
||||
SearchResultImages,
|
||||
)
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...utils.normalize import normalize_document_input
|
||||
from ...utils.validation import validate_scrape_options, prepare_scrape_options
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
async def search(
|
||||
client: AsyncHttpClient,
|
||||
request: SearchRequest
|
||||
) -> SearchData:
|
||||
"""
|
||||
Async search for documents.
|
||||
|
||||
Args:
|
||||
client: Async HTTP client instance
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
SearchData with search results grouped by source type
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the search operation fails
|
||||
"""
|
||||
request_data = _prepare_search_request(request)
|
||||
try:
|
||||
response = await client.post("/v2/search", request_data)
|
||||
if response.status_code != 200:
|
||||
handle_response_error(response, "search")
|
||||
response_data = response.json()
|
||||
if not response_data.get("success"):
|
||||
handle_response_error(response, "search")
|
||||
data = response_data.get("data", {}) or {}
|
||||
out = SearchData()
|
||||
if "web" in data:
|
||||
out.web = _transform_array(data["web"], SearchResultWeb)
|
||||
if "news" in data:
|
||||
out.news = _transform_array(data["news"], SearchResultNews)
|
||||
if "images" in data:
|
||||
out.images = _transform_array(data["images"], SearchResultImages)
|
||||
if "developer" in data:
|
||||
out.developer = _transform_array(data["developer"], SearchResultWeb)
|
||||
return out
|
||||
except Exception as err:
|
||||
if hasattr(err, "response"):
|
||||
handle_response_error(getattr(err, "response"), "search")
|
||||
raise err
|
||||
|
||||
def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:
|
||||
"""
|
||||
Transforms an array of items into a list of result_type or Document.
|
||||
If the item dict contains any of the special keys, it is treated as a Document.
|
||||
Otherwise, it is treated as result_type.
|
||||
If the item is not a dict, it is wrapped as result_type with url=item.
|
||||
"""
|
||||
results: List[Union[T, Document]] = []
|
||||
for item in arr:
|
||||
if item and isinstance(item, dict):
|
||||
if (
|
||||
"markdown" in item or
|
||||
"html" in item or
|
||||
"rawHtml" in item or
|
||||
"links" in item or
|
||||
"screenshot" in item or
|
||||
"changeTracking" in item or
|
||||
"summary" in item or
|
||||
"json" in item
|
||||
):
|
||||
results.append(Document(**normalize_document_input(item)))
|
||||
else:
|
||||
results.append(result_type(**item))
|
||||
else:
|
||||
results.append(result_type(url=item))
|
||||
return results
|
||||
|
||||
def _validate_search_request(request: SearchRequest) -> SearchRequest:
|
||||
"""
|
||||
Validate and normalize search request.
|
||||
|
||||
Args:
|
||||
request: Search request to validate
|
||||
|
||||
Returns:
|
||||
Validated request
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
if not request.query or not request.query.strip():
|
||||
raise ValueError("Query cannot be empty")
|
||||
|
||||
if request.limit is not None:
|
||||
if request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
if request.limit > 100:
|
||||
raise ValueError("Limit cannot exceed 100")
|
||||
|
||||
if request.timeout is not None:
|
||||
if request.timeout <= 0:
|
||||
raise ValueError("Timeout must be positive")
|
||||
if request.timeout > 300000:
|
||||
raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
|
||||
|
||||
if request.sources is not None:
|
||||
valid_sources = {"web", "news", "images"}
|
||||
for source in request.sources:
|
||||
if isinstance(source, str):
|
||||
if source not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
|
||||
elif hasattr(source, 'type'):
|
||||
if source.type not in valid_sources:
|
||||
raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
|
||||
|
||||
if request.include_domains and request.exclude_domains:
|
||||
raise ValueError(
|
||||
"include_domains and exclude_domains cannot both be specified"
|
||||
)
|
||||
|
||||
if request.location is not None:
|
||||
if not isinstance(request.location, str) or len(request.location.strip()) == 0:
|
||||
raise ValueError("Location must be a non-empty string")
|
||||
|
||||
if request.tbs is not None:
|
||||
if not isinstance(request.tbs, str) or len(request.tbs.strip()) == 0:
|
||||
raise ValueError("tbs must be a non-empty string")
|
||||
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
return request
|
||||
|
||||
def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
|
||||
"""
|
||||
Prepare a search request payload.
|
||||
|
||||
Args:
|
||||
request: Search request
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_request = _validate_search_request(request)
|
||||
data = validated_request.model_dump(exclude_none=True, by_alias=True)
|
||||
|
||||
if "limit" not in data and validated_request.limit is not None:
|
||||
data["limit"] = validated_request.limit
|
||||
if "timeout" not in data and validated_request.timeout is not None:
|
||||
data["timeout"] = validated_request.timeout
|
||||
|
||||
if validated_request.ignore_invalid_urls is not None:
|
||||
data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
|
||||
data.pop("ignore_invalid_urls", None)
|
||||
|
||||
if validated_request.include_domains is not None:
|
||||
data["includeDomains"] = validated_request.include_domains
|
||||
data.pop("include_domains", None)
|
||||
|
||||
if validated_request.exclude_domains is not None:
|
||||
data["excludeDomains"] = validated_request.exclude_domains
|
||||
data.pop("exclude_domains", None)
|
||||
|
||||
if validated_request.threat_protection is not None:
|
||||
data["threatProtection"] = validated_request.threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
data.pop("threat_protection", None)
|
||||
|
||||
if validated_request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(validated_request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
data.pop("scrape_options", None)
|
||||
|
||||
if (v := getattr(validated_request, "integration", None)) is not None and str(v).strip():
|
||||
data["integration"] = str(validated_request.integration).strip()
|
||||
|
||||
return data
|
||||
@@ -0,0 +1,89 @@
|
||||
from ...utils.http_client_async import AsyncHttpClient
|
||||
from ...utils.error_handler import handle_response_error
|
||||
from ...types import ConcurrencyCheck, CreditUsage, TokenUsage, CreditUsageHistoricalResponse, TokenUsageHistoricalResponse, QueueStatusResponse
|
||||
|
||||
|
||||
async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck:
|
||||
resp = await client.get("/v2/concurrency-check")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get concurrency")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return ConcurrencyCheck(
|
||||
concurrency=data.get("concurrency"),
|
||||
max_concurrency=data.get("maxConcurrency", data.get("max_concurrency")),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:
|
||||
resp = await client.get("/v2/team/credit-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return CreditUsage(
|
||||
remaining_credits=data.get("remainingCredits", data.get("remaining_credits", 0)),
|
||||
plan_credits=data.get("planCredits", data.get("plan_credits")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:
|
||||
resp = await client.get("/v2/team/token-usage")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return TokenUsage(
|
||||
remaining_tokens=data.get("remainingTokens", data.get("remaining_tokens", 0)),
|
||||
plan_tokens=data.get("planTokens", data.get("plan_tokens")),
|
||||
billing_period_start=data.get("billingPeriodStart", data.get("billing_period_start")),
|
||||
billing_period_end=data.get("billingPeriodEnd", data.get("billing_period_end")),
|
||||
)
|
||||
|
||||
|
||||
async def get_queue_status(client: AsyncHttpClient) -> QueueStatusResponse:
|
||||
resp = await client.get("/v2/team/queue-status")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get queue status")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
data = body.get("data", body)
|
||||
return QueueStatusResponse(
|
||||
jobs_in_queue=data.get("jobsInQueue", 0),
|
||||
active_jobs_in_queue=data.get("activeJobsInQueue", 0),
|
||||
waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
|
||||
max_concurrency=data.get("maxConcurrency", 0),
|
||||
most_recent_success=data.get("mostRecentSuccess", None),
|
||||
)
|
||||
|
||||
|
||||
async def get_credit_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> CreditUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/credit-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get credit usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return CreditUsageHistoricalResponse(**body)
|
||||
|
||||
|
||||
async def get_token_usage_historical(client: AsyncHttpClient, by_api_key: bool = False) -> TokenUsageHistoricalResponse:
|
||||
query = "?byApiKey=true" if by_api_key else ""
|
||||
resp = await client.get(f"/v2/team/token-usage/historical{query}")
|
||||
if resp.status_code >= 400:
|
||||
handle_response_error(resp, "get token usage historical")
|
||||
body = resp.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error"))
|
||||
return TokenUsageHistoricalResponse(**body)
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
"""
|
||||
Batch scraping functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Optional, List, Callable, Dict, Any, Union
|
||||
from ..types import (
|
||||
BatchScrapeRequest,
|
||||
BatchScrapeResponse,
|
||||
BatchScrapeJob,
|
||||
ScrapeOptions,
|
||||
Document,
|
||||
WebhookConfig,
|
||||
PaginationConfig,
|
||||
AuditMetadata,
|
||||
)
|
||||
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
|
||||
from ..utils.normalize import normalize_document_input
|
||||
from ..types import CrawlErrorsResponse
|
||||
|
||||
|
||||
def _parse_batch_scrape_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc in data_list or []:
|
||||
if isinstance(doc, dict):
|
||||
normalized = normalize_document_input(doc)
|
||||
documents.append(Document(**normalized))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_batch_scrape_status_response(body: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": body.get("status"),
|
||||
"completed": body.get("completed", 0),
|
||||
"total": body.get("total", 0),
|
||||
"credits_used": body.get("creditsUsed"),
|
||||
"expires_at": body.get("expiresAt"),
|
||||
"next": body.get("next"),
|
||||
"data": _parse_batch_scrape_documents(body.get("data", []) or []),
|
||||
}
|
||||
|
||||
|
||||
def start_batch_scrape(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
idempotency_key: Optional[str] = None,
|
||||
) -> BatchScrapeResponse:
|
||||
"""
|
||||
Start a batch scrape job for multiple URLs.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
|
||||
Returns:
|
||||
BatchScrapeResponse containing job information
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the batch scrape operation fails to start
|
||||
"""
|
||||
# Prepare request data
|
||||
request_data = prepare_batch_scrape_request(
|
||||
urls,
|
||||
options=options,
|
||||
webhook=webhook,
|
||||
append_to_id=append_to_id,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
max_concurrency=max_concurrency,
|
||||
zero_data_retention=zero_data_retention,
|
||||
integration=integration,
|
||||
audit_metadata=audit_metadata,
|
||||
)
|
||||
|
||||
# Make the API request
|
||||
headers = client._prepare_headers(idempotency_key) # type: ignore[attr-defined]
|
||||
response = client.post("/v2/batch/scrape", request_data, headers=headers)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "start batch scrape")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
return BatchScrapeResponse(
|
||||
id=body.get("id"),
|
||||
url=body.get("url"),
|
||||
invalid_urls=body.get("invalidURLs") or None,
|
||||
)
|
||||
|
||||
|
||||
def get_batch_scrape_status(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Get the status of a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob containing job status and data
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the status check fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.get(f"/v2/batch/scrape/{job_id}")
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape status")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"]:
|
||||
documents = _fetch_all_batch_pages(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config
|
||||
)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
def get_batch_scrape_status_page(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Fetch a single page of batch scrape results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: Opaque next URL from a prior batch scrape status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
BatchScrapeJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = client.get(next_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape status page")
|
||||
|
||||
body = response.json()
|
||||
payload = _parse_batch_scrape_status_response(body)
|
||||
|
||||
return BatchScrapeJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
def _fetch_all_batch_pages(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of batch scrape results.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = client.get(current_url)
|
||||
|
||||
if not response.ok:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
try:
|
||||
page_payload = _parse_batch_scrape_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit after adding all docs from this page
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def cancel_batch_scrape(
|
||||
client: HttpClient,
|
||||
job_id: str
|
||||
) -> bool:
|
||||
"""
|
||||
Cancel a running batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job to cancel
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse with updated status
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the cancellation fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.delete(f"/v2/batch/scrape/{job_id}")
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "cancel batch scrape")
|
||||
|
||||
# Parse response
|
||||
body = response.json()
|
||||
return body.get("status") == "cancelled"
|
||||
|
||||
|
||||
def wait_for_batch_completion(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Wait for a batch scrape job to complete, polling for status updates.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse when job completes
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the job fails or timeout is reached
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
|
||||
while True:
|
||||
status_job = get_batch_scrape_status(client, job_id)
|
||||
|
||||
# Check if job is complete
|
||||
if status_job.status in ["completed", "failed", "cancelled"]:
|
||||
return status_job
|
||||
|
||||
# Check timeout
|
||||
if timeout and (time.monotonic() - start_time) > timeout:
|
||||
raise TimeoutError(f"Batch scrape job {job_id} did not complete within {timeout} seconds")
|
||||
|
||||
# Wait before next poll
|
||||
time.sleep(poll_interval)
|
||||
|
||||
|
||||
def batch_scrape(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
idempotency_key: Optional[str] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> BatchScrapeJob:
|
||||
"""
|
||||
Start a batch scrape job and wait for it to complete.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
|
||||
Returns:
|
||||
BatchScrapeStatusResponse when job completes
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If the batch scrape fails to start or complete
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
# Start the batch scrape
|
||||
start = start_batch_scrape(
|
||||
client,
|
||||
urls,
|
||||
options=options,
|
||||
webhook=webhook,
|
||||
append_to_id=append_to_id,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
max_concurrency=max_concurrency,
|
||||
zero_data_retention=zero_data_retention,
|
||||
integration=integration,
|
||||
audit_metadata=audit_metadata,
|
||||
idempotency_key=idempotency_key,
|
||||
)
|
||||
|
||||
job_id = start.id
|
||||
|
||||
# Wait for completion
|
||||
return wait_for_batch_completion(
|
||||
client, job_id, poll_interval, timeout
|
||||
)
|
||||
|
||||
|
||||
def validate_batch_urls(urls: List[str]) -> List[str]:
|
||||
"""
|
||||
Validate and normalize a list of URLs for batch scraping.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to validate
|
||||
|
||||
Returns:
|
||||
Validated list of URLs
|
||||
|
||||
Raises:
|
||||
ValueError: If URLs are invalid
|
||||
"""
|
||||
if not urls:
|
||||
raise ValueError("URLs list cannot be empty")
|
||||
|
||||
validated_urls = []
|
||||
for url in urls:
|
||||
if not url or not isinstance(url, str):
|
||||
raise ValueError(f"Invalid URL: {url}")
|
||||
|
||||
# Basic URL validation
|
||||
if not (url.startswith("http://") or url.startswith("https://")):
|
||||
raise ValueError(f"URL must start with http:// or https://: {url}")
|
||||
|
||||
validated_urls.append(url.strip())
|
||||
|
||||
return validated_urls
|
||||
|
||||
|
||||
def prepare_batch_scrape_request(
|
||||
urls: List[str],
|
||||
*,
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
webhook: Optional[Union[str, WebhookConfig]] = None,
|
||||
append_to_id: Optional[str] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
max_concurrency: Optional[int] = None,
|
||||
zero_data_retention: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
audit_metadata: Optional[AuditMetadata] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Prepare a batch scrape request payload.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
|
||||
Returns:
|
||||
Request payload dictionary
|
||||
"""
|
||||
validated_urls = validate_batch_urls(urls)
|
||||
request_data: Dict[str, Any] = {"urls": validated_urls}
|
||||
|
||||
# Flatten scrape options at the top level (v2 behavior)
|
||||
if options:
|
||||
scrape_data = prepare_scrape_options(options)
|
||||
if scrape_data:
|
||||
request_data.update(scrape_data)
|
||||
if audit_metadata is not None:
|
||||
request_data["auditMetadata"] = audit_metadata.model_dump()
|
||||
|
||||
# Batch-specific fields
|
||||
if webhook is not None:
|
||||
if isinstance(webhook, str):
|
||||
request_data["webhook"] = webhook
|
||||
else:
|
||||
request_data["webhook"] = webhook.model_dump(exclude_none=True)
|
||||
if append_to_id is not None:
|
||||
request_data["appendToId"] = append_to_id
|
||||
if ignore_invalid_urls is not None:
|
||||
request_data["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if max_concurrency is not None:
|
||||
request_data["maxConcurrency"] = max_concurrency
|
||||
if zero_data_retention is not None:
|
||||
request_data["zeroDataRetention"] = zero_data_retention
|
||||
if integration is not None:
|
||||
request_data["integration"] = str(integration).strip()
|
||||
|
||||
return request_data
|
||||
|
||||
|
||||
def chunk_urls(urls: List[str], chunk_size: int = 100) -> List[List[str]]:
|
||||
"""
|
||||
Split a large list of URLs into smaller chunks for batch processing.
|
||||
|
||||
Args:
|
||||
urls: List of URLs to chunk
|
||||
chunk_size: Maximum size of each chunk
|
||||
|
||||
Returns:
|
||||
List of URL chunks
|
||||
"""
|
||||
chunks = []
|
||||
for i in range(0, len(urls), chunk_size):
|
||||
chunks.append(urls[i:i + chunk_size])
|
||||
return chunks
|
||||
|
||||
|
||||
def process_large_batch(
|
||||
client: HttpClient,
|
||||
urls: List[str],
|
||||
options: Optional[ScrapeOptions] = None,
|
||||
chunk_size: int = 100,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Process a large batch of URLs by splitting into smaller chunks.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
urls: List of URLs to scrape
|
||||
options: Scraping options
|
||||
chunk_size: Size of each batch chunk
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait per chunk
|
||||
|
||||
Returns:
|
||||
List of all scraped documents
|
||||
|
||||
Raises:
|
||||
FirecrawlError: If any chunk fails
|
||||
"""
|
||||
url_chunks = chunk_urls(urls, chunk_size)
|
||||
all_documents = []
|
||||
completed_chunks = 0
|
||||
|
||||
for chunk in url_chunks:
|
||||
# Process this chunk
|
||||
result = batch_scrape(
|
||||
client,
|
||||
chunk,
|
||||
options=options,
|
||||
poll_interval=poll_interval,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# Add documents from this chunk
|
||||
if result.data:
|
||||
all_documents.extend(result.data)
|
||||
|
||||
completed_chunks += 1
|
||||
|
||||
return all_documents
|
||||
|
||||
|
||||
def get_batch_scrape_errors(client: HttpClient, job_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors for a batch scrape job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the batch scrape job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse with errors and robots-blocked URLs
|
||||
"""
|
||||
response = client.get(f"/v2/batch/scrape/{job_id}/errors")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get batch scrape errors")
|
||||
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
Browser session methods for Firecrawl v2 API.
|
||||
|
||||
Provides create, execute, delete, and list operations for browser sessions.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
from ..types import (
|
||||
BrowserCreateResponse,
|
||||
BrowserExecuteResponse,
|
||||
BrowserDeleteResponse,
|
||||
BrowserListResponse,
|
||||
)
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.error_handler import handle_response_error
|
||||
|
||||
|
||||
def _normalize_browser_create_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "cdpUrl" in out and "cdp_url" not in out:
|
||||
out["cdp_url"] = out["cdpUrl"]
|
||||
if "liveViewUrl" in out and "live_view_url" not in out:
|
||||
out["live_view_url"] = out["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in out and "interactive_live_view_url" not in out:
|
||||
out["interactive_live_view_url"] = out["interactiveLiveViewUrl"]
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_browser_list_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessions" in out and isinstance(out["sessions"], list):
|
||||
normalized_sessions = []
|
||||
for s in out["sessions"]:
|
||||
ns = dict(s)
|
||||
if "cdpUrl" in ns and "cdp_url" not in ns:
|
||||
ns["cdp_url"] = ns["cdpUrl"]
|
||||
if "liveViewUrl" in ns and "live_view_url" not in ns:
|
||||
ns["live_view_url"] = ns["liveViewUrl"]
|
||||
if "interactiveLiveViewUrl" in ns and "interactive_live_view_url" not in ns:
|
||||
ns["interactive_live_view_url"] = ns["interactiveLiveViewUrl"]
|
||||
if "streamWebView" in ns and "stream_web_view" not in ns:
|
||||
ns["stream_web_view"] = ns["streamWebView"]
|
||||
if "createdAt" in ns and "created_at" not in ns:
|
||||
ns["created_at"] = ns["createdAt"]
|
||||
if "lastActivity" in ns and "last_activity" not in ns:
|
||||
ns["last_activity"] = ns["lastActivity"]
|
||||
normalized_sessions.append(ns)
|
||||
out["sessions"] = normalized_sessions
|
||||
return out
|
||||
|
||||
|
||||
def browser(
|
||||
client: HttpClient,
|
||||
*,
|
||||
ttl: Optional[int] = None,
|
||||
activity_ttl: Optional[int] = None,
|
||||
stream_web_view: Optional[bool] = None,
|
||||
profile: Optional[Dict[str, Any]] = None,
|
||||
) -> BrowserCreateResponse:
|
||||
"""Create a new browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
ttl: Total time-to-live in seconds (30-3600, default 300)
|
||||
activity_ttl: Inactivity TTL in seconds (10-3600)
|
||||
stream_web_view: Whether to enable webview streaming
|
||||
profile: Profile config with ``name`` (str) and
|
||||
optional ``save_changes`` (bool, default ``True``)
|
||||
|
||||
Returns:
|
||||
BrowserCreateResponse with session id and CDP URL
|
||||
"""
|
||||
body: Dict[str, Any] = {}
|
||||
if ttl is not None:
|
||||
body["ttl"] = ttl
|
||||
if activity_ttl is not None:
|
||||
body["activityTtl"] = activity_ttl
|
||||
if stream_web_view is not None:
|
||||
body["streamWebView"] = stream_web_view
|
||||
if profile is not None:
|
||||
body["profile"] = {
|
||||
"name": profile["name"],
|
||||
"saveChanges": profile.get("save_changes", True),
|
||||
}
|
||||
|
||||
resp = client.post("/v2/browser", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "create browser session")
|
||||
payload = _normalize_browser_create_response(resp.json())
|
||||
return BrowserCreateResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_execute_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "exitCode" in out and "exit_code" not in out:
|
||||
out["exit_code"] = out["exitCode"]
|
||||
return out
|
||||
|
||||
|
||||
def browser_execute(
|
||||
client: HttpClient,
|
||||
session_id: str,
|
||||
code: str,
|
||||
*,
|
||||
language: Literal["python", "node", "bash"] = "bash",
|
||||
timeout: Optional[int] = None,
|
||||
) -> BrowserExecuteResponse:
|
||||
"""Execute code in a browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
session_id: Browser session ID
|
||||
code: Code to execute
|
||||
language: Programming language ("python", "node", or "bash")
|
||||
timeout: Execution timeout in seconds (1-300, default 30)
|
||||
|
||||
Returns:
|
||||
BrowserExecuteResponse with execution result
|
||||
"""
|
||||
body: Dict[str, Any] = {
|
||||
"code": code,
|
||||
"language": language,
|
||||
}
|
||||
if timeout is not None:
|
||||
body["timeout"] = timeout
|
||||
|
||||
resp = client.post(f"/v2/browser/{session_id}/execute", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "execute browser code")
|
||||
payload = _normalize_browser_execute_response(resp.json())
|
||||
return BrowserExecuteResponse(**payload)
|
||||
|
||||
|
||||
def _normalize_browser_delete_response(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "sessionDurationMs" in out and "session_duration_ms" not in out:
|
||||
out["session_duration_ms"] = out["sessionDurationMs"]
|
||||
if "creditsBilled" in out and "credits_billed" not in out:
|
||||
out["credits_billed"] = out["creditsBilled"]
|
||||
return out
|
||||
|
||||
|
||||
def delete_browser(
|
||||
client: HttpClient,
|
||||
session_id: str,
|
||||
) -> BrowserDeleteResponse:
|
||||
"""Delete a browser session.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
session_id: Browser session ID
|
||||
|
||||
Returns:
|
||||
BrowserDeleteResponse
|
||||
"""
|
||||
resp = client.delete(f"/v2/browser/{session_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "delete browser session")
|
||||
payload = _normalize_browser_delete_response(resp.json())
|
||||
return BrowserDeleteResponse(**payload)
|
||||
|
||||
|
||||
def list_browsers(
|
||||
client: HttpClient,
|
||||
*,
|
||||
status: Optional[Literal["active", "destroyed"]] = None,
|
||||
) -> BrowserListResponse:
|
||||
"""List browser sessions.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
status: Filter by session status ("active" or "destroyed")
|
||||
|
||||
Returns:
|
||||
BrowserListResponse with list of sessions
|
||||
"""
|
||||
endpoint = "/v2/browser"
|
||||
if status is not None:
|
||||
endpoint = f"{endpoint}?status={status}"
|
||||
|
||||
resp = client.get(endpoint)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "list browser sessions")
|
||||
payload = _normalize_browser_list_response(resp.json())
|
||||
return BrowserListResponse(**payload)
|
||||
@@ -0,0 +1,647 @@
|
||||
"""
|
||||
Crawling functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Optional, Dict, Any, List
|
||||
from ..types import (
|
||||
CrawlRequest,
|
||||
CrawlJob,
|
||||
CrawlResponse, Document, CrawlParamsRequest, CrawlParamsResponse, CrawlParamsData,
|
||||
WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl, PaginationConfig
|
||||
)
|
||||
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
|
||||
from ..utils.normalize import normalize_document_input
|
||||
|
||||
|
||||
def _validate_crawl_request(request: CrawlRequest) -> None:
|
||||
"""
|
||||
Validate crawl request parameters.
|
||||
|
||||
Args:
|
||||
request: CrawlRequest to validate
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
"""
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
if request.limit is not None and request.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
|
||||
# Validate scrape_options (if provided)
|
||||
if request.scrape_options is not None:
|
||||
validate_scrape_options(request.scrape_options)
|
||||
|
||||
|
||||
def _prepare_crawl_request(request: CrawlRequest) -> dict:
|
||||
"""
|
||||
Prepare crawl request for API submission.
|
||||
|
||||
Args:
|
||||
request: CrawlRequest to prepare
|
||||
|
||||
Returns:
|
||||
Dictionary ready for API submission
|
||||
"""
|
||||
# Validate request
|
||||
_validate_crawl_request(request)
|
||||
|
||||
# Start with basic data
|
||||
data = {"url": request.url}
|
||||
|
||||
# Add prompt if present
|
||||
if request.prompt:
|
||||
data["prompt"] = request.prompt
|
||||
|
||||
# Handle scrape_options conversion first (before model_dump)
|
||||
if request.scrape_options is not None:
|
||||
scrape_data = prepare_scrape_options(request.scrape_options)
|
||||
if scrape_data:
|
||||
data["scrapeOptions"] = scrape_data
|
||||
|
||||
# Convert request to dict
|
||||
request_data = request.model_dump(exclude_none=True, exclude_unset=True)
|
||||
|
||||
# Remove url, prompt, and scrape_options (already handled)
|
||||
request_data.pop("url", None)
|
||||
request_data.pop("prompt", None)
|
||||
request_data.pop("scrape_options", None)
|
||||
|
||||
# Handle webhook conversion first (before model_dump)
|
||||
if request.webhook is not None:
|
||||
if isinstance(request.webhook, str):
|
||||
data["webhook"] = request.webhook
|
||||
else:
|
||||
# Convert WebhookConfig to dict
|
||||
data["webhook"] = request.webhook.model_dump(exclude_none=True)
|
||||
|
||||
# Convert other snake_case fields to camelCase
|
||||
field_mappings = {
|
||||
"include_paths": "includePaths",
|
||||
"exclude_paths": "excludePaths",
|
||||
"max_discovery_depth": "maxDiscoveryDepth",
|
||||
"sitemap": "sitemap",
|
||||
"ignore_query_parameters": "ignoreQueryParameters",
|
||||
"deduplicate_similar_urls": "deduplicateSimilarURLs",
|
||||
"crawl_entire_domain": "crawlEntireDomain",
|
||||
"allow_external_links": "allowExternalLinks",
|
||||
"allow_subdomains": "allowSubdomains",
|
||||
"ignore_robots_txt": "ignoreRobotsTxt",
|
||||
"robots_user_agent": "robotsUserAgent",
|
||||
"delay": "delay",
|
||||
"max_concurrency": "maxConcurrency",
|
||||
"regex_on_full_url": "regexOnFullURL",
|
||||
"zero_data_retention": "zeroDataRetention"
|
||||
}
|
||||
|
||||
# Apply field mappings
|
||||
for snake_case, camel_case in field_mappings.items():
|
||||
if snake_case in request_data:
|
||||
data[camel_case] = request_data.pop(snake_case)
|
||||
|
||||
# Add any remaining fields that don't need conversion (like limit)
|
||||
data.update(request_data)
|
||||
# Trim integration if present
|
||||
if "integration" in data and isinstance(data["integration"], str):
|
||||
data["integration"] = data["integration"].strip()
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _parse_crawl_documents(data_list: Optional[List[Any]]) -> List[Document]:
|
||||
documents: List[Document] = []
|
||||
for doc_data in data_list or []:
|
||||
if isinstance(doc_data, dict):
|
||||
documents.append(Document(**normalize_document_input(doc_data)))
|
||||
return documents
|
||||
|
||||
|
||||
def _parse_crawl_status_response(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not response_data.get("success"):
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
return {
|
||||
"status": response_data.get("status"),
|
||||
"completed": response_data.get("completed", 0),
|
||||
"total": response_data.get("total", 0),
|
||||
"credits_used": response_data.get("creditsUsed", 0),
|
||||
"expires_at": response_data.get("expiresAt"),
|
||||
"next": response_data.get("next"),
|
||||
"data": _parse_crawl_documents(response_data.get("data", [])),
|
||||
}
|
||||
|
||||
|
||||
def start_crawl(client: HttpClient, request: CrawlRequest) -> CrawlResponse:
|
||||
"""
|
||||
Start a crawl job for a website.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
|
||||
Returns:
|
||||
CrawlResponse with job information
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl operation fails to start
|
||||
"""
|
||||
request_data = _prepare_crawl_request(request)
|
||||
|
||||
response = client.post("/v2/crawl", request_data)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "start crawl")
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
if response_data.get("success"):
|
||||
job_data = {
|
||||
"id": response_data.get("id"),
|
||||
"url": response_data.get("url")
|
||||
}
|
||||
|
||||
return CrawlResponse(**job_data)
|
||||
else:
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
def get_crawl_status(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Get the status of a crawl job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
pagination_config: Optional configuration for pagination behavior
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination
|
||||
is enabled (default) and there are multiple pages of results, this timeout applies to
|
||||
each page request separately, not to the entire operation
|
||||
|
||||
Returns:
|
||||
CrawlJob with current status and data
|
||||
|
||||
Raises:
|
||||
Exception: If the status check fails
|
||||
"""
|
||||
# Make the API request
|
||||
response = client.get(f"/v2/crawl/{job_id}", timeout=request_timeout)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get crawl status")
|
||||
|
||||
# Parse response
|
||||
response_data = response.json()
|
||||
|
||||
payload = _parse_crawl_status_response(response_data)
|
||||
|
||||
documents = payload["data"]
|
||||
|
||||
# Handle pagination if requested
|
||||
auto_paginate = pagination_config.auto_paginate if pagination_config else True
|
||||
if auto_paginate and payload["next"] and not (
|
||||
pagination_config
|
||||
and pagination_config.max_results is not None
|
||||
and len(documents) >= pagination_config.max_results
|
||||
):
|
||||
documents = _fetch_all_pages(
|
||||
client,
|
||||
payload["next"],
|
||||
documents,
|
||||
pagination_config,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
# Create CrawlJob with current status and data
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"] if not auto_paginate else None,
|
||||
data=documents,
|
||||
)
|
||||
|
||||
|
||||
def get_crawl_status_page(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Fetch a single page of crawl results using the provided next URL.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: Opaque next URL from a prior crawl status response
|
||||
request_timeout: Timeout (in seconds) for the HTTP request
|
||||
|
||||
Returns:
|
||||
CrawlJob with the page data and next URL (if any)
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails or returns an error response
|
||||
"""
|
||||
response = client.get(next_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get crawl status page")
|
||||
|
||||
response_data = response.json()
|
||||
payload = _parse_crawl_status_response(response_data)
|
||||
|
||||
return CrawlJob(
|
||||
status=payload["status"],
|
||||
completed=payload["completed"],
|
||||
total=payload["total"],
|
||||
credits_used=payload["credits_used"],
|
||||
expires_at=payload["expires_at"],
|
||||
next=payload["next"],
|
||||
data=payload["data"],
|
||||
)
|
||||
|
||||
|
||||
def _fetch_all_pages(
|
||||
client: HttpClient,
|
||||
next_url: str,
|
||||
initial_documents: List[Document],
|
||||
pagination_config: Optional[PaginationConfig] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> List[Document]:
|
||||
"""
|
||||
Fetch all pages of crawl results.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
next_url: URL for the next page
|
||||
initial_documents: Documents from the first page
|
||||
pagination_config: Optional configuration for pagination limits
|
||||
request_timeout: Optional timeout (in seconds) for the underlying HTTP request
|
||||
|
||||
Returns:
|
||||
List of all documents from all pages
|
||||
"""
|
||||
documents = initial_documents.copy()
|
||||
current_url = next_url
|
||||
page_count = 0
|
||||
|
||||
# Apply pagination limits
|
||||
max_pages = pagination_config.max_pages if pagination_config else None
|
||||
max_results = pagination_config.max_results if pagination_config else None
|
||||
max_wait_time = pagination_config.max_wait_time if pagination_config else None
|
||||
|
||||
start_time = time.monotonic()
|
||||
|
||||
while current_url:
|
||||
# Check pagination limits (treat 0 as a valid limit)
|
||||
if (max_pages is not None) and page_count >= max_pages:
|
||||
break
|
||||
|
||||
if (max_wait_time is not None) and (time.monotonic() - start_time) > max_wait_time:
|
||||
break
|
||||
|
||||
# Fetch next page
|
||||
response = client.get(current_url, timeout=request_timeout)
|
||||
|
||||
if not response.ok:
|
||||
# Log error but continue with what we have
|
||||
import logging
|
||||
logger = logging.getLogger("firecrawl")
|
||||
logger.warning("Failed to fetch next page", extra={"status_code": response.status_code})
|
||||
break
|
||||
|
||||
page_data = response.json()
|
||||
|
||||
try:
|
||||
page_payload = _parse_crawl_status_response(page_data)
|
||||
except Exception:
|
||||
break
|
||||
|
||||
# Add documents from this page
|
||||
for document in page_payload["data"]:
|
||||
# Check max_results limit BEFORE adding each document
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
documents.append(document)
|
||||
|
||||
# Check if we hit max_results limit
|
||||
if max_results is not None and len(documents) >= max_results:
|
||||
break
|
||||
|
||||
# Get next URL
|
||||
current_url = page_payload["next"]
|
||||
page_count += 1
|
||||
|
||||
return documents
|
||||
|
||||
|
||||
def cancel_crawl(client: HttpClient, job_id: str) -> bool:
|
||||
"""
|
||||
Cancel a running crawl job.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job to cancel
|
||||
|
||||
Returns:
|
||||
bool: True if the crawl was cancelled, False otherwise
|
||||
|
||||
Raises:
|
||||
Exception: If the cancellation fails
|
||||
"""
|
||||
response = client.delete(f"/v2/crawl/{job_id}")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "cancel crawl")
|
||||
|
||||
response_data = response.json()
|
||||
|
||||
return response_data.get("status") == "cancelled"
|
||||
|
||||
def wait_for_crawl_completion(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Wait for a crawl job to complete, polling for status updates.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
job_id: ID of the crawl job
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait (None for no timeout)
|
||||
request_timeout: Optional timeout (in seconds) for each status request
|
||||
|
||||
Returns:
|
||||
CrawlJob when job completes
|
||||
|
||||
Raises:
|
||||
Exception: If the job fails
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
|
||||
while True:
|
||||
crawl_job = get_crawl_status(
|
||||
client,
|
||||
job_id,
|
||||
request_timeout=request_timeout,
|
||||
)
|
||||
|
||||
# Check if job is complete
|
||||
if crawl_job.status in ["completed", "failed", "cancelled"]:
|
||||
return crawl_job
|
||||
|
||||
# Check timeout
|
||||
if timeout is not None and (time.monotonic() - start_time) > timeout:
|
||||
raise TimeoutError(f"Crawl job {job_id} did not complete within {timeout} seconds")
|
||||
|
||||
# Wait before next poll
|
||||
time.sleep(poll_interval)
|
||||
|
||||
|
||||
def crawl(
|
||||
client: HttpClient,
|
||||
request: CrawlRequest,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
*,
|
||||
request_timeout: Optional[float] = None,
|
||||
) -> CrawlJob:
|
||||
"""
|
||||
Start a crawl job and wait for it to complete.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlRequest containing URL and options
|
||||
poll_interval: Seconds between status checks
|
||||
timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
|
||||
request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination
|
||||
requests when fetching results. If there are multiple pages, each page request gets this timeout
|
||||
|
||||
Returns:
|
||||
CrawlJob when job completes
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the crawl fails to start or complete
|
||||
TimeoutError: If timeout is reached
|
||||
"""
|
||||
# Start the crawl
|
||||
crawl_job = start_crawl(client, request)
|
||||
job_id = crawl_job.id
|
||||
|
||||
# Determine the per-request timeout. If not provided, reuse the overall timeout value.
|
||||
effective_request_timeout = request_timeout if request_timeout is not None else timeout
|
||||
|
||||
# Wait for completion
|
||||
return wait_for_crawl_completion(
|
||||
client,
|
||||
job_id,
|
||||
poll_interval,
|
||||
timeout,
|
||||
request_timeout=effective_request_timeout,
|
||||
)
|
||||
|
||||
|
||||
def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> CrawlParamsData:
|
||||
"""
|
||||
Get crawl parameters from LLM based on URL and prompt.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
request: CrawlParamsRequest containing URL and prompt
|
||||
|
||||
Returns:
|
||||
CrawlParamsData containing suggested crawl options
|
||||
|
||||
Raises:
|
||||
ValueError: If request is invalid
|
||||
Exception: If the operation fails
|
||||
"""
|
||||
# Validate request
|
||||
if not request.url or not request.url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
if not request.prompt or not request.prompt.strip():
|
||||
raise ValueError("Prompt cannot be empty")
|
||||
|
||||
# Prepare request data
|
||||
request_data = {
|
||||
"url": request.url,
|
||||
"prompt": request.prompt
|
||||
}
|
||||
|
||||
# Make the API request
|
||||
response = client.post("/v2/crawl/params-preview", request_data)
|
||||
|
||||
# Handle errors
|
||||
if not response.ok:
|
||||
handle_response_error(response, "crawl params preview")
|
||||
|
||||
# Parse response
|
||||
response_data = response.json()
|
||||
|
||||
if response_data.get("success"):
|
||||
params_data = response_data.get("data", {})
|
||||
|
||||
# Convert camelCase to snake_case for CrawlParamsData
|
||||
converted_params = {}
|
||||
field_mappings = {
|
||||
"includePaths": "include_paths",
|
||||
"excludePaths": "exclude_paths",
|
||||
"maxDiscoveryDepth": "max_discovery_depth",
|
||||
"sitemap": "sitemap",
|
||||
"ignoreQueryParameters": "ignore_query_parameters",
|
||||
"deduplicateSimilarURLs": "deduplicate_similar_urls",
|
||||
"crawlEntireDomain": "crawl_entire_domain",
|
||||
"allowExternalLinks": "allow_external_links",
|
||||
"allowSubdomains": "allow_subdomains",
|
||||
"ignoreRobotsTxt": "ignore_robots_txt",
|
||||
"robotsUserAgent": "robots_user_agent",
|
||||
"maxConcurrency": "max_concurrency",
|
||||
"scrapeOptions": "scrape_options",
|
||||
"zeroDataRetention": "zero_data_retention"
|
||||
}
|
||||
|
||||
# Handle webhook conversion
|
||||
if "webhook" in params_data:
|
||||
webhook_data = params_data["webhook"]
|
||||
if isinstance(webhook_data, dict):
|
||||
converted_params["webhook"] = WebhookConfig(**webhook_data)
|
||||
else:
|
||||
converted_params["webhook"] = webhook_data
|
||||
|
||||
for camel_case, snake_case in field_mappings.items():
|
||||
if camel_case in params_data:
|
||||
if camel_case == "scrapeOptions" and params_data[camel_case] is not None:
|
||||
# Handle nested scrapeOptions conversion
|
||||
scrape_opts_data = params_data[camel_case]
|
||||
converted_scrape_opts = {}
|
||||
scrape_field_mappings = {
|
||||
"includeTags": "include_tags",
|
||||
"excludeTags": "exclude_tags",
|
||||
"onlyMainContent": "only_main_content",
|
||||
"waitFor": "wait_for",
|
||||
"skipTlsVerification": "skip_tls_verification",
|
||||
"removeBase64Images": "remove_base64_images"
|
||||
}
|
||||
|
||||
for scrape_camel, scrape_snake in scrape_field_mappings.items():
|
||||
if scrape_camel in scrape_opts_data:
|
||||
converted_scrape_opts[scrape_snake] = scrape_opts_data[scrape_camel]
|
||||
|
||||
# Handle formats field - if it's a list, convert to ScrapeFormats
|
||||
if "formats" in scrape_opts_data:
|
||||
formats_data = scrape_opts_data["formats"]
|
||||
if isinstance(formats_data, list):
|
||||
# Convert list to ScrapeFormats object
|
||||
from ..types import ScrapeFormats
|
||||
converted_scrape_opts["formats"] = ScrapeFormats(formats=formats_data)
|
||||
else:
|
||||
converted_scrape_opts["formats"] = formats_data
|
||||
|
||||
# Add fields that don't need conversion
|
||||
for key, value in scrape_opts_data.items():
|
||||
if key not in scrape_field_mappings and key != "formats":
|
||||
converted_scrape_opts[key] = value
|
||||
|
||||
converted_params[snake_case] = converted_scrape_opts
|
||||
else:
|
||||
converted_params[snake_case] = params_data[camel_case]
|
||||
|
||||
# Add fields that don't need conversion
|
||||
for key, value in params_data.items():
|
||||
if key not in field_mappings:
|
||||
converted_params[key] = value
|
||||
|
||||
# Add warning if present
|
||||
if "warning" in response_data:
|
||||
converted_params["warning"] = response_data["warning"]
|
||||
|
||||
return CrawlParamsData(**converted_params)
|
||||
else:
|
||||
raise Exception(response_data.get("error", "Unknown error occurred"))
|
||||
|
||||
|
||||
def get_crawl_errors(http_client: HttpClient, crawl_id: str) -> CrawlErrorsResponse:
|
||||
"""
|
||||
Get errors from a crawl job.
|
||||
|
||||
Args:
|
||||
http_client: HTTP client for making requests
|
||||
crawl_id: The ID of the crawl job
|
||||
|
||||
Returns:
|
||||
CrawlErrorsResponse containing errors and robots blocked URLs
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails
|
||||
"""
|
||||
response = http_client.get(f"/v2/crawl/{crawl_id}/errors")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "check crawl errors")
|
||||
|
||||
try:
|
||||
body = response.json()
|
||||
payload = body.get("data", body)
|
||||
# Manual key normalization since we avoid Pydantic aliases
|
||||
normalized = {
|
||||
"errors": payload.get("errors", []),
|
||||
"robots_blocked": payload.get("robotsBlocked", payload.get("robots_blocked", [])),
|
||||
}
|
||||
return CrawlErrorsResponse(**normalized)
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to parse crawl errors response: {e}")
|
||||
|
||||
|
||||
def get_active_crawls(client: HttpClient) -> ActiveCrawlsResponse:
|
||||
"""
|
||||
Get a list of currently active crawl jobs.
|
||||
|
||||
Args:
|
||||
client: HTTP client instance
|
||||
|
||||
Returns:
|
||||
ActiveCrawlsResponse containing a list of active crawl jobs
|
||||
|
||||
Raises:
|
||||
Exception: If the request fails
|
||||
"""
|
||||
response = client.get("/v2/crawl/active")
|
||||
|
||||
if not response.ok:
|
||||
handle_response_error(response, "get active crawls")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
crawls_in = body.get("crawls", [])
|
||||
normalized_crawls = []
|
||||
for c in crawls_in:
|
||||
if isinstance(c, dict):
|
||||
normalized_crawls.append({
|
||||
"id": c.get("id"),
|
||||
"team_id": c.get("teamId", c.get("team_id")),
|
||||
"url": c.get("url"),
|
||||
"options": c.get("options"),
|
||||
})
|
||||
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized_crawls])
|
||||
@@ -0,0 +1,201 @@
|
||||
from typing import Any, Dict, List, Optional
|
||||
import time
|
||||
import warnings
|
||||
|
||||
from ..types import ExtractResponse, ScrapeOptions, ThreatProtectionOptions
|
||||
from ..types import AgentOptions
|
||||
from ..utils.http_client import HttpClient
|
||||
from ..utils.validation import prepare_scrape_options
|
||||
from ..utils.error_handler import handle_response_error
|
||||
|
||||
_EXTRACT_DEPRECATION_MSG = (
|
||||
"The extract endpoint is in maintenance mode and its use is discouraged. "
|
||||
"Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor "
|
||||
"to find a replacement."
|
||||
)
|
||||
|
||||
|
||||
def _prepare_extract_request(
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> Dict[str, Any]:
|
||||
body: Dict[str, Any] = {}
|
||||
if urls is not None:
|
||||
body["urls"] = urls
|
||||
if prompt is not None:
|
||||
body["prompt"] = prompt
|
||||
if schema is not None:
|
||||
body["schema"] = schema
|
||||
if system_prompt is not None:
|
||||
body["systemPrompt"] = system_prompt
|
||||
if allow_external_links is not None:
|
||||
body["allowExternalLinks"] = allow_external_links
|
||||
if enable_web_search is not None:
|
||||
body["enableWebSearch"] = enable_web_search
|
||||
if show_sources is not None:
|
||||
body["showSources"] = show_sources
|
||||
if ignore_invalid_urls is not None:
|
||||
body["ignoreInvalidURLs"] = ignore_invalid_urls
|
||||
if scrape_options is not None:
|
||||
prepared = prepare_scrape_options(scrape_options)
|
||||
if prepared:
|
||||
body["scrapeOptions"] = prepared
|
||||
if integration is not None and str(integration).strip():
|
||||
body["integration"] = str(integration).strip()
|
||||
if agent is not None:
|
||||
try:
|
||||
body["agent"] = agent.model_dump(exclude_none=True) # type: ignore[attr-defined]
|
||||
except AttributeError:
|
||||
body["agent"] = agent # fallback
|
||||
if threat_protection is not None:
|
||||
body["threatProtection"] = threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
return body
|
||||
|
||||
|
||||
def _normalize_extract_response_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
out = dict(payload)
|
||||
if "expiresAt" in out and "expires_at" not in out:
|
||||
out["expires_at"] = out["expiresAt"]
|
||||
if "creditsUsed" in out and "credits_used" not in out:
|
||||
out["credits_used"] = out["creditsUsed"]
|
||||
if "tokensUsed" in out and "tokens_used" not in out:
|
||||
out["tokens_used"] = out["tokensUsed"]
|
||||
return out
|
||||
|
||||
|
||||
def start_extract(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Start an extract job (non-blocking).
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
body = _prepare_extract_request(
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
agent=agent,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
resp = client.post("/v2/extract", body)
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "extract")
|
||||
payload = _normalize_extract_response_payload(resp.json())
|
||||
return ExtractResponse(**payload)
|
||||
|
||||
|
||||
def get_extract_status(client: HttpClient, job_id: str) -> ExtractResponse:
|
||||
"""Get the current status of an extract job.
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
resp = client.get(f"/v2/extract/{job_id}")
|
||||
if not resp.ok:
|
||||
handle_response_error(resp, "extract-status")
|
||||
payload = _normalize_extract_response_payload(resp.json())
|
||||
return ExtractResponse(**payload)
|
||||
|
||||
|
||||
def wait_extract(
|
||||
client: HttpClient,
|
||||
job_id: str,
|
||||
*,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
) -> ExtractResponse:
|
||||
start_ts = time.time()
|
||||
while True:
|
||||
status = get_extract_status(client, job_id)
|
||||
if status.status in ("completed", "failed", "cancelled"):
|
||||
return status
|
||||
if timeout is not None and (time.time() - start_ts) > timeout:
|
||||
return status
|
||||
time.sleep(max(1, poll_interval))
|
||||
|
||||
|
||||
def extract(
|
||||
client: HttpClient,
|
||||
urls: Optional[List[str]],
|
||||
*,
|
||||
prompt: Optional[str] = None,
|
||||
schema: Optional[Dict[str, Any]] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
allow_external_links: Optional[bool] = None,
|
||||
enable_web_search: Optional[bool] = None,
|
||||
show_sources: Optional[bool] = None,
|
||||
scrape_options: Optional[ScrapeOptions] = None,
|
||||
ignore_invalid_urls: Optional[bool] = None,
|
||||
poll_interval: int = 2,
|
||||
timeout: Optional[int] = None,
|
||||
integration: Optional[str] = None,
|
||||
agent: Optional[AgentOptions] = None,
|
||||
threat_protection: Optional[ThreatProtectionOptions] = None,
|
||||
) -> ExtractResponse:
|
||||
"""Extract structured data and wait until completion.
|
||||
|
||||
.. deprecated::
|
||||
The extract endpoint is in maintenance mode and its use is discouraged.
|
||||
Review https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor
|
||||
to find a replacement.
|
||||
"""
|
||||
warnings.warn(_EXTRACT_DEPRECATION_MSG, DeprecationWarning, stacklevel=2)
|
||||
started = start_extract(
|
||||
client,
|
||||
urls,
|
||||
prompt=prompt,
|
||||
schema=schema,
|
||||
system_prompt=system_prompt,
|
||||
allow_external_links=allow_external_links,
|
||||
enable_web_search=enable_web_search,
|
||||
show_sources=show_sources,
|
||||
scrape_options=scrape_options,
|
||||
ignore_invalid_urls=ignore_invalid_urls,
|
||||
integration=integration,
|
||||
agent=agent,
|
||||
threat_protection=threat_protection,
|
||||
)
|
||||
job_id = getattr(started, "id", None)
|
||||
if not job_id:
|
||||
return started
|
||||
return wait_extract(client, job_id, poll_interval=poll_interval, timeout=timeout)
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Mapping functionality for Firecrawl v2 API.
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
from ..types import MapOptions, MapData, LinkResult
|
||||
from ..utils import HttpClient, handle_response_error
|
||||
|
||||
|
||||
def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict[str, Any]:
|
||||
if not url or not url.strip():
|
||||
raise ValueError("URL cannot be empty")
|
||||
|
||||
payload: Dict[str, Any] = {"url": url.strip()}
|
||||
|
||||
if options is not None:
|
||||
# Unified sitemap parameter already provided in options
|
||||
data: Dict[str, Any] = {}
|
||||
if getattr(options, "sitemap", None) is not None:
|
||||
data["sitemap"] = options.sitemap
|
||||
|
||||
if options.search is not None:
|
||||
data["search"] = options.search
|
||||
if options.include_subdomains is not None:
|
||||
data["includeSubdomains"] = options.include_subdomains
|
||||
if options.ignore_query_parameters is not None:
|
||||
data["ignoreQueryParameters"] = options.ignore_query_parameters
|
||||
if options.limit is not None:
|
||||
if options.limit <= 0:
|
||||
raise ValueError("Limit must be positive")
|
||||
data["limit"] = options.limit
|
||||
if options.timeout is not None:
|
||||
data["timeout"] = options.timeout
|
||||
if options.integration is not None and options.integration.strip():
|
||||
data["integration"] = options.integration.strip()
|
||||
if options.location is not None:
|
||||
data["location"] = options.location.model_dump(exclude_none=True)
|
||||
if options.threat_protection is not None:
|
||||
data["threatProtection"] = options.threat_protection.model_dump(
|
||||
by_alias=True, exclude_none=True
|
||||
)
|
||||
if options.audit_metadata is not None:
|
||||
data["auditMetadata"] = options.audit_metadata.model_dump()
|
||||
payload.update(data)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def map(client: HttpClient, url: str, options: Optional[MapOptions] = None) -> MapData:
|
||||
"""
|
||||
Map a URL and return MapData (links list with optional titles/descriptions).
|
||||
"""
|
||||
request_data = _prepare_map_request(url, options)
|
||||
response = client.post("/v2/map", request_data)
|
||||
if not response.ok:
|
||||
handle_response_error(response, "map")
|
||||
|
||||
body = response.json()
|
||||
if not body.get("success"):
|
||||
raise Exception(body.get("error", "Unknown error occurred"))
|
||||
|
||||
# shouldnt return inside data?
|
||||
# data = body.get("data", {})
|
||||
# result_links: list[LinkResult] = []
|
||||
# for item in data.get("links", []):
|
||||
# if isinstance(item, dict):
|
||||
# result_links.append(
|
||||
# LinkResult(
|
||||
# url=item.get("url", ""),
|
||||
# title=item.get("title"),
|
||||
# description=item.get("description"),
|
||||
# )
|
||||
# )
|
||||
# elif isinstance(item, str):
|
||||
# result_links.append(LinkResult(url=item))
|
||||
|
||||
result_links: list[LinkResult] = []
|
||||
for item in body.get("links", []):
|
||||
if isinstance(item, dict):
|
||||
result_links.append(
|
||||
LinkResult(
|
||||
url=item.get("url", ""),
|
||||
title=item.get("title"),
|
||||
description=item.get("description"),
|
||||
)
|
||||
)
|
||||
elif isinstance(item, str):
|
||||
result_links.append(LinkResult(url=item))
|
||||
|
||||
return MapData(links=result_links)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user