firecrawl main branch snapshot (tar.gz, no history)

This commit is contained in:
xiaoxue_admin
2026-08-11 23:50:06 +08:00
commit d938f0df44
1712 changed files with 398375 additions and 0 deletions
@@ -0,0 +1,277 @@
"""
Integration tests for agent method with mocked requests.
"""
import unittest
from unittest.mock import patch, MagicMock
from pydantic import BaseModel, Field
from typing import List, Optional
from firecrawl import FirecrawlApp
class Founder(BaseModel):
name: str = Field(description="Full name of the founder")
role: Optional[str] = Field(None, description="Role or position")
background: Optional[str] = Field(None, description="Professional background")
class FoundersSchema(BaseModel):
founders: List[Founder] = Field(description="List of founders")
class TestAgent(unittest.TestCase):
"""Integration tests for agent method."""
@patch('firecrawl.v2.utils.http_client.requests.post')
@patch('firecrawl.v2.utils.http_client.requests.get')
def test_agent_basic(self, mock_get, mock_post):
"""Test basic agent call."""
# Mock start agent response
mock_start_response = MagicMock()
mock_start_response.ok = True
mock_start_response.status_code = 200
mock_start_response.json.return_value = {
"success": True,
"id": "test-agent-123",
"status": "processing"
}
mock_post.return_value = mock_start_response
# Mock get status response (completed)
mock_status_response = MagicMock()
mock_status_response.ok = True
mock_status_response.status_code = 200
mock_status_response.json.return_value = {
"success": True,
"id": "test-agent-123",
"status": "completed",
"data": {
"founders": [
{"name": "John Doe", "role": "CEO", "background": "Tech entrepreneur"},
{"name": "Jane Smith", "role": "CTO", "background": "Software engineer"}
]
},
"creditsUsed": 10,
"expiresAt": "2024-01-01T00:00:00Z"
}
mock_get.return_value = mock_status_response
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(
prompt="Find the founders of Firecrawl",
schema=FoundersSchema
)
# Verify post was called with correct URL and data
mock_post.assert_called_once()
post_call_args = mock_post.call_args
post_url = post_call_args[1]["url"] if "url" in post_call_args[1] else post_call_args[0][0]
assert "/v2/agent" in str(post_url)
# Check request body
request_body = post_call_args[1]["json"]
assert request_body["prompt"] == "Find the founders of Firecrawl"
assert "schema" in request_body
assert request_body["schema"]["type"] == "object"
assert "founders" in request_body["schema"]["properties"]
# Verify get was called to check status
mock_get.assert_called()
# Check result
assert result.status == "completed"
assert result.data is not None
@patch('firecrawl.v2.utils.http_client.requests.post')
def test_agent_with_urls(self, mock_post):
"""Test agent call with URLs."""
mock_response = MagicMock()
mock_response.ok = True
mock_response.status_code = 200
mock_response.json.return_value = {
"success": True,
"status": "completed",
"data": {"result": "done"}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(
urls=["https://example.com", "https://test.com"],
prompt="Extract information",
schema={"type": "object", "properties": {"info": {"type": "string"}}}
)
# Check request body includes URLs
post_call_args = mock_post.call_args
request_body = post_call_args[1]["json"]
assert request_body["urls"] == ["https://example.com", "https://test.com"]
assert request_body["prompt"] == "Extract information"
@patch('firecrawl.v2.utils.http_client.requests.post')
def test_agent_with_dict_schema(self, mock_post):
"""Test agent call with dict schema."""
mock_response = MagicMock()
mock_response.ok = True
mock_response.status_code = 200
mock_response.json.return_value = {
"success": True,
"status": "completed",
"data": {"result": "done"}
}
mock_post.return_value = mock_response
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"}
}
}
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(
prompt="Extract person data",
schema=schema
)
# Check request body includes schema
post_call_args = mock_post.call_args
request_body = post_call_args[1]["json"]
assert request_body["schema"] == schema
@patch('firecrawl.v2.utils.http_client.requests.post')
def test_agent_with_all_params(self, mock_post):
"""Test agent call with all parameters."""
mock_response = MagicMock()
mock_response.ok = True
mock_response.status_code = 200
mock_response.json.return_value = {
"success": True,
"status": "completed",
"data": {"result": "done"}
}
mock_post.return_value = mock_response
schema = {"type": "object"}
urls = ["https://example.com"]
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(
urls=urls,
prompt="Complete test",
schema=schema,
integration="test-integration",
max_credits=50,
strict_constrain_to_urls=True,
poll_interval=1,
timeout=30
)
# Check all parameters are in request body
post_call_args = mock_post.call_args
request_body = post_call_args[1]["json"]
assert request_body["prompt"] == "Complete test"
assert request_body["urls"] == urls
assert request_body["schema"] == schema
assert request_body["integration"] == "test-integration"
assert request_body["maxCredits"] == 50
assert request_body["strictConstrainToURLs"] is True
@patch('firecrawl.v2.utils.http_client.requests.post')
def test_agent_pydantic_schema_normalization(self, mock_post):
"""Test that Pydantic schemas are properly normalized."""
mock_response = MagicMock()
mock_response.ok = True
mock_response.status_code = 200
mock_response.json.return_value = {
"success": True,
"status": "completed",
"data": {"result": "done"}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(
prompt="Find founders",
schema=FoundersSchema
)
# Check that schema was normalized to JSON schema format
post_call_args = mock_post.call_args
request_body = post_call_args[1]["json"]
assert "schema" in request_body
schema = request_body["schema"]
assert schema["type"] == "object"
assert "properties" in schema
assert "founders" in schema["properties"]
assert schema["properties"]["founders"]["type"] == "array"
@patch('firecrawl.v2.utils.http_client.requests.post')
@patch('firecrawl.v2.utils.http_client.requests.get')
def test_agent_url_construction(self, mock_get, mock_post):
"""Test that agent requests are sent to correct URL."""
# Mock start agent response
mock_start_response = MagicMock()
mock_start_response.ok = True
mock_start_response.status_code = 200
mock_start_response.json.return_value = {
"success": True,
"id": "test-agent-123",
"status": "processing"
}
mock_post.return_value = mock_start_response
# Mock get status response
mock_status_response = MagicMock()
mock_status_response.ok = True
mock_status_response.status_code = 200
mock_status_response.json.return_value = {
"success": True,
"id": "test-agent-123",
"status": "completed",
"data": {"result": "done"}
}
mock_get.return_value = mock_status_response
app = FirecrawlApp(api_key="test-api-key", api_url="https://api.firecrawl.dev")
result = app.agent(prompt="Test prompt")
# Check POST URL - requests.post is called with url as keyword arg
post_call_args = mock_post.call_args
post_url = post_call_args[1].get("url") if "url" in post_call_args[1] else post_call_args[0][0]
assert "/v2/agent" in str(post_url)
# Check GET URL
get_call_args = mock_get.call_args
get_url = get_call_args[1].get("url") if "url" in get_call_args[1] else get_call_args[0][0]
assert "/v2/agent/test-agent-123" in str(get_url)
@patch('firecrawl.v2.utils.http_client.requests.post')
def test_agent_headers(self, mock_post):
"""Test that agent requests include correct headers."""
mock_response = MagicMock()
mock_response.ok = True
mock_response.status_code = 200
mock_response.json.return_value = {
"success": True,
"status": "completed",
"data": {"result": "done"}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key="test-api-key")
result = app.agent(prompt="Test prompt")
# Check headers
post_call_args = mock_post.call_args
headers = post_call_args[1]["headers"]
assert "Authorization" in headers
assert headers["Authorization"] == "Bearer test-api-key"
assert headers["Content-Type"] == "application/json"
if __name__ == '__main__':
unittest.main()
@@ -0,0 +1,44 @@
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from firecrawl.v2.client import FirecrawlClient
from firecrawl.v2.client_async import AsyncFirecrawlClient
@pytest.fixture(autouse=True)
def clear_firecrawl_api_key_env(monkeypatch):
monkeypatch.delenv("FIRECRAWL_API_KEY", raising=False)
yield
def test_cloud_requires_api_key():
with pytest.raises(ValueError):
FirecrawlClient(api_url="https://api.firecrawl.dev")
def test_self_host_allows_missing_api_key():
client = FirecrawlClient(api_url="http://localhost:3000")
assert client.http_client.api_key is None
def test_async_cloud_requires_api_key():
with pytest.raises(ValueError):
AsyncFirecrawlClient(api_url="https://api.firecrawl.dev")
@pytest.mark.asyncio
async def test_async_self_host_allows_missing_api_key():
client = AsyncFirecrawlClient(api_url="http://localhost:3000")
try:
assert client.http_client.api_key is None
await client.async_http_client.close()
finally:
# Ensure the underlying HTTPX client is closed even if assertions fail
if not client.async_http_client._client.is_closed:
await client.async_http_client.close()
@@ -0,0 +1,98 @@
import unittest
from unittest.mock import patch, MagicMock
import json
import os
from firecrawl import FirecrawlApp
class TestChangeTracking(unittest.TestCase):
@patch('requests.post')
def test_change_tracking_format(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'success': True,
'data': {
'markdown': 'Test markdown content',
'changeTracking': {
'previousScrapeAt': '2023-01-01T00:00:00Z',
'changeStatus': 'changed',
'visibility': 'visible'
}
}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
result = app.scrape_url('https://example.com', {
'formats': ['markdown', 'changeTracking']
})
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
self.assertEqual(result['changeTracking']['visibility'], 'visible')
@patch('requests.post')
def test_change_tracking_options(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'success': True,
'data': {
'markdown': 'Test markdown content',
'changeTracking': {
'previousScrapeAt': '2023-01-01T00:00:00Z',
'changeStatus': 'changed',
'visibility': 'visible',
'diff': {
'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
'json': {
'files': [{
'from': None,
'to': None,
'chunks': [{
'content': '@@ -1,1 +1,1 @@',
'changes': [{
'type': 'del',
'content': '-old content',
'del': True,
'ln': 1
}, {
'type': 'add',
'content': '+new content',
'add': True,
'ln': 1
}]
}]
}]
}
},
'json': {
'title': {
'previous': 'Old Title',
'current': 'New Title'
}
}
}
}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
result = app.scrape_url('https://example.com', {
'formats': ['markdown', 'changeTracking'],
'changeTrackingOptions': {
'modes': ['git-diff', 'json'],
'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
}
})
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')
@@ -0,0 +1,261 @@
"""Tests for direct scrape kwargs on crawl() and start_crawl()."""
import asyncio
import inspect
import pytest
from firecrawl.v2.client import FirecrawlClient, _SCRAPE_OPTION_KEYS
from firecrawl.v2.client_async import AsyncFirecrawlClient
from firecrawl.v2.types import ScrapeOptions
import firecrawl.v2.methods.crawl as crawl_mod
from firecrawl.v2.methods.aio import crawl as async_crawl_mod
@pytest.fixture
def client():
return FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
@pytest.fixture
def async_client():
return AsyncFirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
class TestSyncCrawlDirectKwargs:
def test_crawl_signature_accepts_formats(self):
sig = inspect.signature(FirecrawlClient.crawl)
assert "formats" in sig.parameters
def test_start_crawl_signature_accepts_formats(self):
sig = inspect.signature(FirecrawlClient.start_crawl)
assert "formats" in sig.parameters
def test_direct_kwargs_build_scrape_options(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", formats=["markdown"])
assert captured["scrape_options"] is not None
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "markdown" in fmt_strs
def test_explicit_scrape_options_wins(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl(
"https://example.com",
scrape_options=ScrapeOptions(formats=["html"]),
formats=["markdown"],
)
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "html" in fmt_strs
assert "markdown" not in fmt_strs
def test_no_scrape_kwargs_means_no_scrape_options(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", limit=10)
assert captured["scrape_options"] is None
def test_crawl_also_builds_scrape_options(self, client, monkeypatch):
captured = {}
def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
with pytest.raises(ConnectionError):
client.crawl("https://example.com", formats=["markdown"], only_main_content=True)
assert captured["scrape_options"] is not None
assert captured["scrape_options"].only_main_content is True
def test_start_crawl_accepts_scrape_timeout(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl("https://example.com", timeout=5000)
assert captured["scrape_options"] is not None
assert captured["scrape_options"].timeout == 5000
def test_crawl_timeout_is_poll_timeout_not_scrape(self, client, monkeypatch):
captured = {}
def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
captured["scrape_options"] = request.scrape_options
captured["poll_timeout"] = timeout
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
with pytest.raises(ConnectionError):
client.crawl("https://example.com", timeout=60, formats=["markdown"])
assert captured["poll_timeout"] == 60
assert captured["scrape_options"] is not None
assert captured["scrape_options"].timeout is None
class TestAsyncCrawlDirectKwargs:
def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl("https://example.com", formats=["markdown"])
)
assert captured["scrape_options"] is not None
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "markdown" in fmt_strs
def test_async_explicit_scrape_options_wins(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl(
"https://example.com",
scrape_options=ScrapeOptions(formats=["html"]),
formats=["markdown"],
)
)
fmt_strs = [str(f) for f in captured["scrape_options"].formats]
assert "html" in fmt_strs
def test_async_integration_preserved_as_crawl_param(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["scrape_options"] = request.scrape_options
captured["integration"] = request.integration
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl(
"https://example.com",
formats=["markdown"],
integration="test-int",
)
)
assert captured["integration"] == "test-int"
assert captured["scrape_options"] is not None
class TestThreatProtectionDirectKwargs:
EXPECTED_WIRE_PAYLOAD = {
"mode": "normal",
"riskScoreThreshold": 80,
"blockedTlds": ["zip"],
}
@staticmethod
def _threat_protection():
from firecrawl.v2.types import ThreatProtectionOptions
return ThreatProtectionOptions(
mode="normal", risk_score_threshold=80, blocked_tlds=["zip"]
)
def test_sync_threat_protection_kwarg_lands_in_wire_body(self, client, monkeypatch):
captured = {}
def fake_start_crawl(http, request):
captured["request"] = request
raise ConnectionError("captured")
monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
client.start_crawl(
"https://example.com",
threat_protection=self._threat_protection(),
)
request = captured["request"]
assert request.scrape_options is not None
assert request.scrape_options.threat_protection is not None
wire_body = crawl_mod._prepare_crawl_request(request)
assert (
wire_body["scrapeOptions"]["threatProtection"]
== self.EXPECTED_WIRE_PAYLOAD
)
def test_async_threat_protection_kwarg_lands_in_wire_body(self, async_client, monkeypatch):
captured = {}
async def fake_start_crawl(http, request):
captured["request"] = request
raise ConnectionError("captured")
monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
with pytest.raises(ConnectionError):
asyncio.get_event_loop().run_until_complete(
async_client.start_crawl(
"https://example.com",
threat_protection=self._threat_protection(),
)
)
request = captured["request"]
assert request.scrape_options is not None
assert request.scrape_options.threat_protection is not None
wire_body = async_crawl_mod._prepare_crawl_request(request)
assert (
wire_body["scrapeOptions"]["threatProtection"]
== self.EXPECTED_WIRE_PAYLOAD
)
def test_threat_protection_in_scrape_option_keys(self):
assert "threat_protection" in _SCRAPE_OPTION_KEYS
@@ -0,0 +1,59 @@
"""Tests for JsonFormat and ChangeTrackingFormat default type fields and serialization."""
import pytest
from firecrawl.v2.types import JsonFormat, ChangeTrackingFormat, ScrapeOptions
from firecrawl.v2.utils.validation import prepare_scrape_options
class TestJsonFormat:
def test_defaults_type_to_json(self):
fmt = JsonFormat(prompt="Extract titles.")
assert fmt.type == "json"
def test_explicit_type_json_still_works(self):
fmt = JsonFormat(type="json", prompt="Extract titles.")
assert fmt.type == "json"
def test_rejects_wrong_type(self):
with pytest.raises(Exception):
JsonFormat(type="markdown", prompt="x")
def test_serializes_through_prepare(self):
opts = ScrapeOptions(formats=[JsonFormat(prompt="Extract titles.")])
prepared = prepare_scrape_options(opts)
fmts = prepared["formats"]
assert any(
isinstance(f, dict) and f.get("type") == "json" and f.get("prompt") == "Extract titles."
for f in fmts
)
class TestChangeTrackingFormat:
def test_defaults_type(self):
fmt = ChangeTrackingFormat(modes=["git-diff"])
assert fmt.type in ("change_tracking", "changeTracking")
def test_serializes_with_modes(self):
opts = ScrapeOptions(formats=[ChangeTrackingFormat(modes=["git-diff"])])
prepared = prepare_scrape_options(opts)
fmts = prepared["formats"]
assert fmts == [{"type": "changeTracking", "modes": ["git-diff"]}]
def test_serializes_with_all_options(self):
opts = ScrapeOptions(formats=[
ChangeTrackingFormat(modes=["git-diff"], tag="daily", prompt="track changes")
])
prepared = prepare_scrape_options(opts)
fmts = prepared["formats"]
assert fmts == [
{"type": "changeTracking", "modes": ["git-diff"], "prompt": "track changes", "tag": "daily"}
]
def test_does_not_drop_schema(self):
opts = ScrapeOptions(formats=[
ChangeTrackingFormat(modes=["json"], schema={"type": "object"})
])
prepared = prepare_scrape_options(opts)
fmts = prepared["formats"]
assert fmts[0]["schema"] == {"type": "object"}
@@ -0,0 +1,27 @@
import pytest
from firecrawl.v2.types import SearchData, SearchResultWeb, SearchResultNews
class TestSearchDataDotData:
def test_raises_with_web_results(self):
sd = SearchData(web=[SearchResultWeb(title="T", url="http://a.com")])
with pytest.raises(AttributeError, match=r"\.web \(1 results\)"):
sd.data
def test_raises_with_multiple_sources(self):
sd = SearchData(
web=[SearchResultWeb(title="T", url="http://a.com")],
news=[SearchResultNews(title="N", url="http://b.com")],
)
with pytest.raises(AttributeError, match=r"\.web.*\.news"):
sd.data
def test_raises_with_empty_response(self):
sd = SearchData()
with pytest.raises(AttributeError, match="grouped by source"):
sd.data
def test_web_still_works(self):
sd = SearchData(web=[SearchResultWeb(title="T", url="http://a.com")])
assert len(sd.web) == 1
assert sd.web[0].title == "T"
@@ -0,0 +1,117 @@
import unittest
from unittest.mock import patch, MagicMock
import os
from firecrawl import FirecrawlApp
class TestTimeoutConversion(unittest.TestCase):
@patch('requests.post')
def test_scrape_url_timeout_conversion(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'success': True,
'data': {
'markdown': 'Test content'
}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
app.scrape_url('https://example.com', timeout=60000)
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 65.0)
@patch('requests.post')
def test_scrape_url_default_timeout(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'success': True,
'data': {
'markdown': 'Test content'
}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
app.scrape_url('https://example.com')
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 35.0)
@patch('requests.post')
def test_post_request_timeout_conversion(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
data = {'timeout': 30000}
headers = {'Content-Type': 'application/json'}
app._post_request('https://example.com/api', data, headers)
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 35.0)
@patch('requests.post')
def test_post_request_default_timeout(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
data = {'timeout': 30000, 'url': 'https://example.com'}
headers = {'Content-Type': 'application/json'}
app._post_request('https://example.com/api', data, headers)
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 35.0)
@patch('requests.post')
def test_timeout_edge_cases(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'success': True,
'data': {
'markdown': 'Test content'
}
}
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
app.scrape_url('https://example.com', timeout=1000)
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 6.0)
app.scrape_url('https://example.com', timeout=0)
args, kwargs = mock_post.call_args
self.assertEqual(kwargs['timeout'], 5.0)
@patch('requests.post')
def test_post_request_no_timeout_key(self, mock_post):
mock_response = MagicMock()
mock_response.status_code = 200
mock_post.return_value = mock_response
app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
data = {'url': 'https://example.com'}
headers = {'Content-Type': 'application/json'}
app._post_request('https://example.com/api', data, headers)
args, kwargs = mock_post.call_args
self.assertIsNone(kwargs['timeout'])
if __name__ == '__main__':
unittest.main()
+60
View File
@@ -0,0 +1,60 @@
import pytest
from unittest.mock import patch
ALIAS_MAP = {
"scrape_url": "scrape",
"crawl_url": "crawl",
"map_url": "map",
"async_crawl_url": "start_crawl",
"check_crawl_status": "get_crawl_status",
"check_crawl_errors": "get_crawl_errors",
"batch_scrape_urls": "batch_scrape",
"async_batch_scrape_urls": "start_batch_scrape",
"check_batch_scrape_status": "get_batch_scrape_status",
"check_batch_scrape_errors": "get_batch_scrape_errors",
}
class TestV2ClientAliases:
def test_aliases_exist(self):
from firecrawl.v2.client import FirecrawlClient
client = FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
for alias in ALIAS_MAP:
assert hasattr(client, alias), f"Missing alias: {alias}"
def test_aliases_delegate(self):
from firecrawl.v2.client import FirecrawlClient
client = FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
sentinel = object()
for alias, target in ALIAS_MAP.items():
with patch.object(client, target, return_value=sentinel) as mock:
result = getattr(client, alias)("arg1")
mock.assert_called_once()
assert result is sentinel
class TestTopLevelFirecrawlAliases:
def test_aliases_exposed(self):
from firecrawl import Firecrawl
client = Firecrawl(api_key="fc-test", api_url="http://localhost:9")
for alias in ALIAS_MAP:
assert hasattr(client, alias), f"Missing on Firecrawl: {alias}"
def test_aliases_delegate(self):
from firecrawl import Firecrawl
client = Firecrawl(api_key="fc-test", api_url="http://localhost:9")
sentinel = object()
for alias, target in ALIAS_MAP.items():
with patch.object(client._v2_client, target, return_value=sentinel) as mock:
result = getattr(client, alias)("arg1")
mock.assert_called_once()
assert result is sentinel
class TestAsyncFirecrawlAliases:
def test_aliases_exposed(self):
from firecrawl import AsyncFirecrawl
client = AsyncFirecrawl(api_key="fc-test", api_url="http://localhost:9")
for alias in ALIAS_MAP:
assert hasattr(client, alias), f"Missing on AsyncFirecrawl: {alias}"
@@ -0,0 +1,29 @@
import asyncio
from firecrawl.v1.client import AsyncV1FirecrawlApp
def test_async_batch_scrape_urls_accepts_parsed_json_response(monkeypatch):
app = AsyncV1FirecrawlApp(api_key="fc-test", api_url="http://localhost:9")
calls = []
async def fake_post_request(url, data, headers):
calls.append((url, data, headers))
return {
"success": True,
"id": "batch-123",
"url": "http://localhost:9/v1/batch/scrape/batch-123",
}
async def fail_handle_error(response, action):
raise AssertionError(f"unexpected error path: {action} {response!r}")
monkeypatch.setattr(app, "_async_post_request", fake_post_request)
monkeypatch.setattr(app, "_handle_error", fail_handle_error)
result = asyncio.run(app.async_batch_scrape_urls(["https://example.com"]))
assert result.success is True
assert result.id == "batch-123"
assert result.url == "http://localhost:9/v1/batch/scrape/batch-123"
assert calls[0][1]["urls"] == ["https://example.com"]