from __future__ import annotations from typing import TYPE_CHECKING import pytest from bs4 import BeautifulSoup from parsel import Selector from crawlee.crawlers._beautifulsoup._utils import html_to_text as html_to_text_beautifulsoup from crawlee.crawlers._parsel._utils import html_to_text as html_to_text_parsel if TYPE_CHECKING: from collections.abc import Callable _EXPECTED_TEXT = ( "Let's start with a simple text. \n" "The ships hung in the sky, much the way that bricks don't. \n" "These aren't the Droids you're looking for\n" "I'm sorry, Dave. I'm afraid I can't do that.\n" "I'm sorry, Dave. I'm afraid I can't do that.\n" 'A1\tA2\tA3\t\n' 'B1\tB2\tB3\tB 4\t\n' 'This is some text with inline elements and HTML entities (>bla<) \n' 'Test\n' 'a\n' 'few\n' 'line\n' 'breaks\n' 'Spaces in an inline text should be completely ignored. \n' 'But,\n' ' a pre-formatted\n' ' block should be kept\n' ' pre-formatted.\n' 'The Greatest Science Fiction Quotes Of All Time \n' "Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You " 'Nexus, huh? I design your eyes.' ) _EXAMPLE_HTML = """ Title SHOULD NOT be converted Let's start with a simple text.

The ships hung in the sky, much the way that bricks don't.

This should be ignored
A1 A2 A3
B1 B2 B3 B 4

This is some text with inline elements and HTML entities (>bla<)

Test
a
few
line
breaks
Spaces in an inline text should be completely ignored. 
But,
    a pre-formatted
                block  should  be  kept
                                       pre-formatted.
These special elements SHOULD NOT BE CONVERTED. This should be skipped too. The Greatest Science Fiction Quotes Of All Time

Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.

""" @pytest.mark.parametrize('html_to_text', [html_to_text_parsel, html_to_text_beautifulsoup]) @pytest.mark.parametrize( ('source', 'expected_text'), [ pytest.param(_EXAMPLE_HTML, _EXPECTED_TEXT, id='Complex html'), (' Plain text node ', 'Plain text node'), (' \nPlain text node \n ', 'Plain text node'), ('

Header 1

Header 2

', 'Header 1\nHeader 2'), ('

Header 1

Header 2


', 'Header 1\nHeader 2'), ('

Header 1

Header 2



', 'Header 1\nHeader 2'), ('

Header 1

Header 2




', 'Header 1\nHeader 2'), ('

Header 1


Header 2




', 'Header 1\n\nHeader 2'), ('

Header 1


Header 2




', 'Header 1\n\nHeader 2'), ('

Header 1

\n
\n

Header 2




', 'Header 1\n\nHeader 2'), ('

Header 1

\n
\n

Header 2




', 'Header 1\n\n\nHeader 2'), ('

Header 1

\n
\n

Header 2




', 'Header 1\n\n\n\nHeader 2'), ('
Div

Paragraph

', 'Div\nParagraph'), ('
Div1
Div2
', 'Div1\nDiv2'), ('
Div1
', 'Div1'), ('
Div1
', 'Div1'), ('
Div1
', 'Div1'), ('Skip svg
Div1
', 'Div1'), ('Skip canvas
Div1
', 'Div1'), ('A B C D E\n\nF G', 'A B C D E F G'), ('
A  B  C  D  E\n\nF  G
', 'A B C D E\n\nF G'), ( '

Heading 1

Deep Div

Heading 2

', 'Heading 1\nDeep Div\nHeading 2', ), ('this_word_should_be_one', 'this_word_should_be_one'), ('some text', 'some text'), pytest.param( ( """
Cell A1Cell A2 Cell A3
Cell B1Cell B2
""" ), 'Cell A1\tCell A2\tCell A3 \t\nCell B1\tCell B2', id='Table', ), ('á é', 'á é'), ], ) def test_html_to_text(source: str, expected_text: str, html_to_text: Callable[[str], str]) -> None: assert html_to_text(source) == expected_text @pytest.mark.parametrize('html_to_text', [html_to_text_parsel, html_to_text_beautifulsoup]) def test_html_to_text_raises_on_wrong_input_type(html_to_text: Callable[[str], str]) -> None: with pytest.raises(TypeError): # Intentional wrong type test. html_to_text(1) # ty: ignore[invalid-argument-type] def test_html_to_text_parsel() -> None: assert html_to_text_parsel(Selector(_EXAMPLE_HTML)) == _EXPECTED_TEXT def test_html_to_text_beautifulsoup() -> None: assert html_to_text_beautifulsoup(BeautifulSoup(_EXAMPLE_HTML, features='lxml')) == _EXPECTED_TEXT