Merge pull request #19 from Laerte/fix/disallow-when-target-path-is-m…

…issing

Fix disallow not working when no path is provided

src/protego.py

@@ -170,7 +170,7 @@ def _quote_path(self, path):
 
         parts = ParseResult('', '', path, parts.params, parts.query, parts.fragment)
         path = urlunparse(parts)
-        return path
+        return path or '/'
 
     def _quote_pattern(self, pattern):
         # Corner case for query only (e.g. '/abc?') and param only (e.g. '/abc;') URLs.

tests/test_protego.py

@@ -1053,3 +1053,9 @@ def test_sitemaps_come_first(self):
                    "Disallow: /something")
         rp = Protego.parse(content=content)
         self.assertEquals(list(rp.sitemaps), ["https://www.foo.bar/sitmap.xml"])
+
+    def test_disallow_target_url_path_is_missing(self):
+        content = "User-Agent: *\nDisallow: /\n"
+        rp = Protego.parse(content)
+        self.assertFalse(rp.can_fetch("http://example.com/", "FooBot"))
+        self.assertFalse(rp.can_fetch("http://example.com", "FooBot"))