Add some improvements

.gitignore

@@ -13,7 +13,4 @@ dist/
 *venv*/
 
 # test
-.tox/
-
-# ide
-.vscode/
+.tox/

tests/test_on_fetched_robotstxt.py

@@ -6,7 +6,7 @@
 from protego import Protego
 
 test_data_directory = join(dirname(abspath(__file__)), 'test_data')
-robotstxts = [(f) for f in listdir(test_data_directory) if isfile(join(test_data_directory, f))]
+robotstxts = [f for f in listdir(test_data_directory) if isfile(join(test_data_directory, f))]
 
 
 @pytest.mark.parametrize('path_to_robotstxt', robotstxts)

tests/test_protego.py

@@ -822,12 +822,12 @@ def test_implicit_allow(self):
         self.assertFalse(rp.can_fetch("/blahblahblah", "SomeOtherBot"))
 
     def test_grouping_unknown_keys(self):
-        '''
+        """
         When we encounter unknown keys, we should disregard any grouping that may have
         happened between user agent rules.
         This is an example from the wild. `Invalid_directive` not being a valid directive,
         we'll not consider the '*' and 'ia_archiver' rules together.
-        '''
+        """
         content = """
         User-agent: abc
         Disallow: /content/2/
@@ -999,7 +999,7 @@ def test_url_case_sensitivity(self):
         self.assertTrue(rp.can_fetch("http://foo.bar/x/y", "FooBot"))
 
     def test_nonterminal_dollar(self):
-        '''Non terminal dollar sign should be treated as an odinary character.'''
+        """Non terminal dollar sign should be treated as an odinary character."""
         content = ("user-agent: FooBot\n"
                    "disallow: /x$/abc\n"
                    "disallow: /y/abc$\n"
@@ -1016,7 +1016,7 @@ def test_nonterminal_dollar(self):
         self.assertTrue(rp.can_fetch("http://foo.bar/yabcy/abcdef", "FooBot"))
 
     def test_escaped_special_symbols(self):
-        '''Percent encoded special symbols should be treated as ordinary characters.'''
+        """Percent encoded special symbols should be treated as ordinary characters."""
         content = ("user-agent: FooBot\n"
                    "disallow: /x/abc%24\n"
                    "disallow: /x%2Ax/abc\n")
@@ -1029,7 +1029,7 @@ def test_escaped_special_symbols(self):
         self.assertTrue(rp.can_fetch("http://foo.bar/xabcx/abc", "FooBot"))
 
     def test_special_symbols_dual_behaviour(self):
-        '''Special symbols such as * and $, should also be treated as an ordinary character'''
+        """Special symbols such as * and $, should also be treated as an ordinary character"""
         content = ("user-agent: FooBot\n"
                    "disallow: /x/abc$\n"
                    "disallow: /x*x/abc\n")

tests/test_unquote.py

@@ -69,7 +69,6 @@ def test_unquoting_plus(self):
         result = _unquote(given)
         self.assertEqual(expect, result,
                          "using unquote(): %r != %r" % (expect, result))
-        expect = given.replace('+', ' ')
 
     def test_unquote_with_unicode(self):
         # Characters in the Latin-1 range, encoded with UTF-8