Adds xpath_alias, sample re commands

selectorlib/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-"""Top-level package for selectorlib."""
+"""Top-level package for Selectorlib."""
 
 __author__ = """scrapehero"""
 __email__ = '[email protected]'

selectorlib/cli.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 
-"""Console script for selectorlib."""
+"""Console script for Selectorlib."""
 import sys
 import click
 
 
 @click.command()
 def main(args=None):
-    """Console script for selectorlib."""
+    """Console script for Selectorlib."""
     click.echo("Replace this message by putting your code into "
-               "selectorlib.cli.main")
+               "Selectorlib.cli.main")
     click.echo("See click documentation at http://click.pocoo.org/")
     return 0
 

selectorlib/selectorlib.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
+import re
 import parsel
 import yaml
 import inspect
 
-
 def extract_field(element, item_type, attribute=None, formatter=None):
     if item_type == 'Text':
         texts = [i.strip() for i in element.xpath('.//text()').getall() if i.strip()]
@@ -35,7 +35,7 @@ def __init__(self, config, formatters=None):
     def from_yaml_string(cls, yaml_string: str, formatters=None):
         """create `Extractor` object from yaml string
 
-        >>> yaml_string = '''
+        >>> yaml_string = ''
             title:
                 css: "h1"
                 type: Text
@@ -77,13 +77,19 @@ def extract(self, html: str, base_url: str = None):
     def _extract_selector(self, field_config, parent_parser):
         if field_config.get("xpath") is not None:
             elements = parent_parser.xpath(field_config['xpath'])
+            if len(elements) == 0:
+                if field_config.get("xpath_alias") is not None:
+                    elements = parent_parser.xpath(field_config['alias'])
+
         else:
             css = field_config['css']
             if css == '':
                 elements = [parent_parser]
             else:
                 elements = parent_parser.css(field_config['css'])
+
         item_type = field_config.get('type', 'Text')
+        # print(field_config) # Returns all fields
         if not elements:
             return None
         values = []
@@ -100,6 +106,11 @@ def _extract_selector(self, field_config, parent_parser):
                 value = extract_field(element, item_type, **kwargs)
 
             if field_config.get('multiple') is not True:
+                if 're' in field_config:
+                    pattern = re.compile(f'{field_config.get("re")}')
+                    regex = re.sub(pattern, '', value)
+                    return regex
+
                 return value
             else:
                 values.append(value)