Zgrab2: support JARM (zmap/zgrab2#282)

ivre/db/__init__.py

@@ -1891,7 +1891,7 @@ def store_scan_json_zgrab(self, fname, filehash=None,
                               needports=False, needopenports=False,
                               categories=None, source=None,
                               add_addr_infos=True, force_info=False,
-                              callback=None, **_):
+                              callback=None, zgrab_port=None, **_):
         """This method parses a JSON scan result produced by zgrab, displays
         the parsing result, and return True if everything went fine,
         False otherwise.
@@ -1909,6 +1909,8 @@ def store_scan_json_zgrab(self, fname, filehash=None,
             categories = []
         scan_doc_saved = False
         self.start_store_hosts()
+        if zgrab_port is not None:
+            zgrab_port = int(zgrab_port)
         with utils.open_file(fname) as fdesc:
             for line in fdesc:
                 rec = json.loads(line.decode())
@@ -1980,7 +1982,7 @@ def store_scan_json_zgrab(self, fname, filehash=None,
                             key,
                         )
                     else:
-                        port = parser(value, host)
+                        port = parser(value, host, port=zgrab_port)
                         if port:
                             host.setdefault('ports', []).append(port)
                 if not host.get('ports'):

ivre/tools/scan2db.py

@@ -73,6 +73,10 @@ def main():
                         help='Additional Nmap probes to use when trying to '
                         'match Masscan results against Nmap service '
                         'fingerprints.')
+    parser.add_argument('--zgrab-port', metavar='PORT',
+                        help='Port used for the zgrab scan. This might be '
+                        'needed since the port number does not appear in the'
+                        'result.')
     parser.add_argument('--force-info', action='store_true',
                         help='Force information (AS, country, city, etc.)'
                         ' renewal (only useful with JSON format)')
@@ -120,6 +124,7 @@ def callback(x):
                     needports=args.ports, needopenports=args.open_ports,
                     force_info=args.force_info,
                     masscan_probes=args.masscan_probes, callback=callback,
+                    zgrab_port=args.zgrab_port,
             ):
                 count += 1
         except Exception:

ivre/zgrabout.py

@@ -49,7 +49,7 @@
                'Product_Version', 'NTLM_Version']
 
 
-def zgrap_parser_http(data, hostrec):
+def zgrap_parser_http(data, hostrec, port=None):
     """This function handles data from `{"data": {"http": [...]}}`
 records. `data` should be the content, i.e. the `[...]`. It should
 consist of simple dictionary, that may contain a `"response"` key
@@ -109,17 +109,27 @@ def zgrap_parser_http(data, hostrec):
                     add_cert_hostnames(cert,
                                        hostrec.setdefault('hostnames', []))
     if url:
-        port = None
+        guessed_port = None
         if ':' in url.get('host', ''):
             try:
-                port = int(url['host'].split(':', 1)[1])
+                guessed_port = int(url['host'].split(':', 1)[1])
             except ValueError:
                 pass
         if port is None:
-            if url.get('scheme') == 'https':
-                port = 443
+            if guessed_port is None:
+                if url.get('scheme') == 'https':
+                    port = 443
+                else:
+                    port = 80
             else:
-                port = 80
+                port = guessed_port
+        elif port != guessed_port:
+            utils.LOGGER.warning(
+                'Port %d found from the URL %s differs from the provided port '
+                'value %d',
+                guessed_port, url.get('path'), port
+            )
+            port = guessed_port
         # Specific paths
         if url.get('path').endswith('/.git/index'):
             if resp.get('status_code') != 200:
@@ -203,11 +213,11 @@ def zgrap_parser_http(data, hostrec):
             utils.LOGGER.warning('URL path not supported yet: %s',
                                  url.get('path'))
             return {}
-    elif req.get('tls_handshake') or req.get('tls_log'):
-        # zgrab / zgrab2
-        port = 443
-    else:
-        port = 80
+    elif port is None:
+        if req.get('tls_handshake') or req.get('tls_log'):
+            port = 443
+        else:
+            port = 80
     res['port'] = port
     # Since Zgrab does not preserve the order of the headers, we need
     # to reconstruct a banner to use Nmap fingerprints
@@ -300,4 +310,45 @@ def zgrap_parser_http(data, hostrec):
     return res
 
 
-ZGRAB_PARSERS = {'http': zgrap_parser_http}
+def zgrap_parser_jarm(data, hostrec, port=None):
+    """This function handles data from `{"data": {"jarm": [...]}}`
+records. `data` should be the content, i.e. the `[...]`. It should
+consist of simple dictionary, that must contain a `"status"` key and a
+`"fingerprint"` key (that may be in a `"result"` sub-document).
+
+The output is a port dict (i.e., the content of the "ports" key of an
+`nmap` of `view` record in IVRE), that may be empty.
+
+    """
+    if not data:
+        return {}
+    # for zgrab2 results
+    if 'result' in data:
+        data.update(data.pop('result'))
+    if data.get('status') != 'success':
+        return {}
+    if 'fingerprint' not in data:
+        utils.LOGGER.warning('Missing "fingerprint" field in zgrab JARM result')
+        return {}
+    if (
+            data['fingerprint'] ==
+            '00000000000000000000000000000000000000000000000000000000000000'
+    ):
+        utils.LOGGER.warning('Null "fingerprint" in zgrab JARM result')
+        return {}
+    if port is None:
+        port = 443  # default
+        utils.LOGGER.warning(
+            'No port provided; using default %d. '
+            'Use --zgrab-port to change it.',
+            port,
+        )
+    return {"state_state": "open", "state_reason": "response",
+            "port": port, "protocol": "tcp", "service_tunnel": "ssl",
+            "scripts": [{'id': 'ssl-jarm', 'output': data['fingerprint']}]}
+
+
+ZGRAB_PARSERS = {
+    'http': zgrap_parser_http,
+    'jarm': zgrap_parser_jarm,
+}