Merge pull request #398 from scrapy/fix-ci-2021

Fix tests and CI

.travis.yml

@@ -1,29 +1,23 @@
 language: python
 dist: xenial
-matrix:
-    include:
-        - python: 2.7
-          env: TOXENV=py27
-        - python: pypy
-          env: TOXENV=pypy
-        - python: 3.4
-          env: TOXENV=py34
-        - python: 3.5
-          env: TOXENV=py35
-        - python: 3.6
-          env: TOXENV=py36
-        - python: 3.7
-          env: TOXENV=py37
-          sudo: true
+python:
+  - "2.7"
+  - "3.5"
+  - "3.6"
+  - "3.7"
+  - "3.8"
+  - "3.9"
 
 before_install:
-    - pip install -U six
+    - pip install -U pip wheel setuptools codecov
 
 install:
-    - pip install -U tox codecov
+    - pip install -rreqs/requirements-latest.txt -rreqs/requirements-tests.txt
 
 script:
-    - tox
+    - coverage erase
+    - coverage run --branch bin/trial scrapyd
+    - coverage report
 
 after_success:
     - codecov
@@ -34,15 +28,3 @@ notifications:
             - "irc.freenode.org#scrapy"
         use_notice: true
         skip_join: true
-
-deploy:
-    provider: pypi
-    distributions: sdist bdist_wheel
-    user: redapple
-    password:
-        secure: IuHONsIRm8w54LkmUlZe0X0dFqW2iAjTyqeRyOyVMvHWIPaiydNpX+xipvRArJnswlnzf7/X3CtZw9ZF9ePznCrvG+3Y1SLHutncRbr6IxSQrcRJm+FTF2I2KNEaOkWqoQjO7xbEehpqo6fttHImCNrCzgZw32hCZqeGbDGPGJs=
-    on:
-        all_branches: true
-        tags: true
-        repo: scrapy/scrapyd
-        condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+([ab][0-9]+|rc[0-9]+|dev[0-9]+)?$"

docs/news.rst

@@ -13,12 +13,15 @@ Added
 - Jobs website shortcut to cancel a job using the cancel.json webservice.
 - Make project argument to listjobs.json optional,
   so that we can easily query for all jobs.
+- Python 3.7, 3.8 and 3.9 support
 
 Removed
 ~~~~~~~
 
 - Doc for ubunut installs removed. Scrapinghub no longer maintains ubuntu repo.
 - Python 3.3 support (although never officially supported)
+- Python 3.4 support
+- Pypy 2 support
 
 1.2.1
 -----

reqs/requirements-tests.txt

@@ -1 +1,2 @@
 coverage
+mock; python_version=='2.7'

scrapyd/tests/test_environ.py

@@ -33,7 +33,7 @@ def test_get_environment_with_eggfile(self):
         if env.get('SCRAPY_FEED_URI'):  # Not compulsory
             self.assert_(env['SCRAPY_FEED_URI'].startswith('file://{}'.format(os.getcwd())))
             self.assert_(env['SCRAPY_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl')))
-        self.failIf('SCRAPY_SETTINGS_MODULE' in env)
+        self.assertNotIn('SCRAPY_SETTINGS_MODULE', env)
 
     def test_get_environment_with_no_items_dir(self):
         config = Config(values={'items_dir': '', 'logs_dir': ''})
@@ -43,5 +43,5 @@ def test_get_environment_with_no_items_dir(self):
         slot = 3
         environ = Environment(config, initenv={})
         env = environ.get_environment(msg, slot)
-        self.failUnless('SCRAPY_FEED_URI' not in env)
-        self.failUnless('SCRAPY_LOG_FILE' not in env)
+        self.assertNotIn('SCRAPY_FEED_URI', env)
+        self.assertNotIn('SCRAPY_LOG_FILE', env)

scrapyd/tests/test_poller.py

@@ -36,22 +36,23 @@ def test_poll_next(self):
 
         d1 = self.poller.next()
         d2 = self.poller.next()
-        self.failUnless(isinstance(d1, Deferred))
-        self.failIf(hasattr(d1, 'result'))
+        self.assertIsInstance(d1, Deferred)
+        self.assertFalse(hasattr(d1, 'result'))
 
         # poll once
         self.poller.poll()
-        self.failUnless(hasattr(d1, 'result') and getattr(d1, 'called', False))
+        self.assertTrue(hasattr(d1, 'result'))
+        self.assertTrue(getattr(d1, 'called', False))
 
         # which project got run: project1 or project2?
-        self.failUnless(d1.result.get('_project'))
+        self.assertTrue(d1.result.get('_project'))
         prj = d1.result['_project']
-        self.failUnlessEqual(d1.result['_spider'], cfg.pop(prj))
+        self.assertEqual(d1.result['_spider'], cfg.pop(prj))
 
         self.queues[prj].pop()
 
         # poll twice
         # check that the other project's spider got to run
         self.poller.poll()
         prj, spd = cfg.popitem()
-        self.failUnlessEqual(d2.result, {'_project': prj, '_spider': spd})
+        self.assertEqual(d2.result, {'_project': prj, '_spider': spd})

scrapyd/tests/test_scheduler.py

@@ -35,7 +35,7 @@ def test_list_update_projects(self):
 
     def test_schedule(self):
         q1, q2 = self.queues['mybot1'], self.queues['mybot2']
-        self.failIf(q1.count())
+        self.assertFalse(q1.count())
         self.sched.schedule('mybot1', 'myspider1', 2, a='b')
         self.sched.schedule('mybot2', 'myspider2', 1, c='d')
         self.sched.schedule('mybot2', 'myspider3', 10, e='f')

scrapyd/tests/test_sqlite.py

@@ -16,9 +16,9 @@ def test_basic_types(self):
         test = self.test_dict
         d = self.dict_class()
         d.update(test)
-        self.failUnlessEqual(list(d.items()), list(test.items()))
+        self.assertEqual(list(d.items()), list(test.items()))
         d.clear()
-        self.failIf(d.items())
+        self.assertFalse(d.items())
 
     def test_in(self):
         d = self.dict_class()
@@ -57,14 +57,14 @@ def setUp(self):
         self.q = self.queue_class()
 
     def test_empty(self):
-        self.failUnless(self.q.pop() is None)
+        self.assertIs(self.q.pop(), None)
 
     def test_one(self):
         msg = "a message"
         self.q.put(msg)
-        self.failIf("_id" in msg)
-        self.failUnlessEqual(self.q.pop(), msg)
-        self.failUnless(self.q.pop() is None)
+        self.assertNotIn("_id", msg)
+        self.assertEqual(self.q.pop(), msg)
+        self.assertIs(self.q.pop(), None)
 
     def test_multiple(self):
         msg1 = "first message"
@@ -74,9 +74,9 @@ def test_multiple(self):
         out = []
         out.append(self.q.pop())
         out.append(self.q.pop())
-        self.failUnless(msg1 in out)
-        self.failUnless(msg2 in out)
-        self.failUnless(self.q.pop() is None)
+        self.assertIn(msg1, out)
+        self.assertIn(msg2, out)
+        self.assertIs(self.q.pop(), None)
 
     def test_priority(self):
         msg1 = "message 1"
@@ -87,14 +87,14 @@ def test_priority(self):
         self.q.put(msg2, priority=5.0)
         self.q.put(msg3, priority=3.0)
         self.q.put(msg4, priority=2.0)
-        self.failUnlessEqual(self.q.pop(), msg2)
-        self.failUnlessEqual(self.q.pop(), msg3)
-        self.failUnlessEqual(self.q.pop(), msg4)
-        self.failUnlessEqual(self.q.pop(), msg1)
+        self.assertEqual(self.q.pop(), msg2)
+        self.assertEqual(self.q.pop(), msg3)
+        self.assertEqual(self.q.pop(), msg4)
+        self.assertEqual(self.q.pop(), msg1)
 
     def test_iter_len_clear(self):
-        self.failUnlessEqual(len(self.q), 0)
-        self.failUnlessEqual(list(self.q), [])
+        self.assertEqual(len(self.q), 0)
+        self.assertEqual(list(self.q), [])
         msg1 = "message 1"
         msg2 = "message 2"
         msg3 = "message 3"
@@ -103,16 +103,16 @@ def test_iter_len_clear(self):
         self.q.put(msg2, priority=5.0)
         self.q.put(msg3, priority=3.0)
         self.q.put(msg4, priority=2.0)
-        self.failUnlessEqual(len(self.q), 4)
-        self.failUnlessEqual(list(self.q), \
+        self.assertEqual(len(self.q), 4)
+        self.assertEqual(list(self.q), \
             [(msg2, 5.0), (msg3, 3.0), (msg4, 2.0), (msg1, 1.0)])
         self.q.clear()
-        self.failUnlessEqual(len(self.q), 0)
-        self.failUnlessEqual(list(self.q), [])
+        self.assertEqual(len(self.q), 0)
+        self.assertEqual(list(self.q), [])
 
     def test_remove(self):
-        self.failUnlessEqual(len(self.q), 0)
-        self.failUnlessEqual(list(self.q), [])
+        self.assertEqual(len(self.q), 0)
+        self.assertEqual(list(self.q), [])
         msg1 = "good message 1"
         msg2 = "bad message 2"
         msg3 = "good message 3"
@@ -122,9 +122,9 @@ def test_remove(self):
         self.q.put(msg3)
         self.q.put(msg4)
         self.q.remove(lambda x: x.startswith("bad"))
-        self.failUnlessEqual(list(self.q), [(msg1, 0.0), (msg3, 0.0)])
+        self.assertEqual(list(self.q), [(msg1, 0.0), (msg3, 0.0)])
 
     def test_types(self):
         for x in self.supported_values:
             self.q.put(x)
-            self.failUnlessEqual(self.q.pop(), x)
+            self.assertEqual(self.q.pop(), x)

scrapyd/tests/test_utils.py

@@ -9,6 +9,11 @@
 import six
 
 from twisted.trial import unittest
+if six.PY2:
+    import mock
+else:
+    from unittest import mock
+from subprocess import Popen
 
 from scrapy.utils.test import get_pythonpath
 from scrapyd.interfaces import IEggStorage
@@ -102,8 +107,16 @@ def test_failed_spider_list(self):
         self.add_test_version('mybot3.egg', 'mybot3', 'r1')
         pypath = get_pythonpath_scrapyd()
         # Workaround missing support for context manager in twisted < 15
-        exc = self.assertRaises(RuntimeError,
-                                get_spider_list, 'mybot3', pythonpath=pypath)
+
+        # Add -W ignore to sub-python to prevent warnings & tb mixup in stderr
+        def popen_wrapper(*args, **kwargs):
+            cmd, args = args[0], args[1:]
+            cmd = [cmd[0], '-W', 'ignore'] + cmd[1:]
+            return Popen(cmd, *args, **kwargs)
+
+        with mock.patch('scrapyd.utils.Popen', wraps=popen_wrapper):
+            exc = self.assertRaises(RuntimeError,
+                                    get_spider_list, 'mybot3', pythonpath=pypath)
         tb = str(exc).rstrip()
         tb = tb.decode('unicode_escape') if six.PY2 else tb
         tb_regex = (

setup.py

@@ -29,9 +29,11 @@
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
         'Development Status :: 5 - Production/Stable',

tox.ini

@@ -4,8 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27
-
+envlist = py27,py35,py36,py37,py38,py39
 
 [testenv]
 deps =
@@ -16,9 +15,3 @@ commands =
     coverage erase
     coverage run --branch {toxinidir}/bin/trial {posargs:scrapyd}
     coverage report
-
-[testenv:precise]
-basepython = python2.7
-deps =
-    -rreqs/requirements-precise.txt
-    -rreqs/requirements-tests.txt