Merge pull request #29 from Gallaecio/scrapy-iterable-single-values

Restore support for nested Scrapy items

Author: wRAR

.travis.yml

@@ -7,20 +7,21 @@ branches:
 
 matrix:
   include:
-  - python: 3.5
-    env: TOXENV=py35
+  - python: 3.7
+    env: TOXENV=docs
+
   - python: 3.6
-    env: TOXENV=py36
+    env: TOXENV=py
   - python: 3.7
-    env: TOXENV=py37
+    env: TOXENV=py
   - python: 3.8
-    env: TOXENV=py38
+    env: TOXENV=py
 
   - python: pypy3
     env: TOXENV=pypy3
 
-  - python: 3.7
-    env: TOXENV=docs
+  - python: 3.8
+    env: TOXENV=extra-deps
 
 install:
 - pip install -U tox codecov

itemloaders/utils.py

@@ -2,11 +2,13 @@
 Copy/paste from scrapy source at the moment, to ensure tests are working.
 Refactoring to come later
 """
-from functools import partial
 import inspect
+from functools import partial
+
+from itemadapter import is_item
 
 
-_ITERABLE_SINGLE_VALUES = dict, str, bytes
+_ITERABLE_SINGLE_VALUES = str, bytes
 
 
 def arg_to_iter(arg):
@@ -17,7 +19,11 @@ def arg_to_iter(arg):
     """
     if arg is None:
         return []
-    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
+    elif (
+        hasattr(arg, '__iter__')
+        and not isinstance(arg, _ITERABLE_SINGLE_VALUES)
+        and not is_item(arg)
+    ):
         return arg
     else:
         return [arg]

setup.py

@@ -27,14 +27,13 @@
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
     ],
-    python_requires='>=3.5',
+    python_requires='>=3.6',
     install_requires=[
         # before updating these versions, be sure they are not higher than
         # scrapy's requirements

tests/test_nested_items.py

@@ -0,0 +1,50 @@
+import unittest
+
+from itemloaders import ItemLoader
+
+
+class NestedItemTest(unittest.TestCase):
+    """Test that adding items as values works as expected."""
+
+    def _test_item(self, item):
+        il = ItemLoader()
+        il.add_value('item_list', item)
+        self.assertEqual(il.load_item(), {'item_list': [item]})
+
+    def test_attrs(self):
+        try:
+            import attr
+        except ImportError:
+            self.skipTest("Cannot import attr")
+
+        @attr.s
+        class TestItem:
+            foo = attr.ib()
+
+        self._test_item(TestItem(foo='bar'))
+
+    def test_dataclass(self):
+        try:
+            from dataclasses import dataclass
+        except ImportError:
+            self.skipTest("Cannot import dataclasses.dataclass")
+
+        @dataclass
+        class TestItem:
+            foo: str
+
+        self._test_item(TestItem(foo='bar'))
+
+    def test_dict(self):
+        self._test_item({'foo': 'bar'})
+
+    def test_scrapy_item(self):
+        try:
+            from scrapy import Field, Item
+        except ImportError:
+            self.skipTest("Cannot import Field or Item from scrapy")
+
+        class TestItem(Item):
+            foo = Field()
+
+        self._test_item(TestItem(foo='bar'))

tox.ini

@@ -12,6 +12,12 @@ commands =
         --doctest-modules \
         {posargs:itemloaders tests}
 
+[testenv:extra-deps]
+deps =
+    {[testenv]deps}
+    attrs
+    scrapy
+
 [testenv:pypy3]
 basepython = pypy3