Avoiding CDATA nonsense

Author: Yomguithereal

requirements.txt

@@ -17,7 +17,7 @@ charset-normalizer==3.3.2
 dateparser==1.1.6
 ebbe==1.13.2
 json5==0.9.11
-lxml>=4.9.2,<5
+lxml>=4.9.2
 nanoid==2.0.0
 playwright==1.35.0
 playwright_stealth==1.0.5

test/scraper_test.py

@@ -97,6 +97,8 @@
     </table>
 """
 
+
+# NOTE: CDATA is handled very differently depending on lxml & bs4 versions
 THE_WORST_HTML = """
     <div>Some text isn't
     it?
@@ -136,8 +138,6 @@
                 <li>Other</li>
                 <li>Again</li>
             </ol>
-            <p>
-            <![CDATA[some very interesting stuff]]></p>
             <p>
                 This is      <span>a large span  </span>
                     with something else over <strong>here</strong>.
@@ -997,8 +997,6 @@ def clean(t):
             Other
             Again
 
-            some very interesting stuff
-
             This is a large span with something else over here.
 
             Hello gorgeous!