HTTP: rework state machine (#1966)

The goal if to correlate the right request-response pair, exporting
metadata from only one transaction (for example, the right url & return
state pair)

As a nice side effect, the code should be much cleaner, but that is a
matter of taste.

Two differences respect to the previous code:
* as it happens in the CI, if in the flow there are only one response
(before) and one request (after), only the metadata of the response are
saved/exported
* for performance reasons, we don't call `ndpi_parse_packet_line_info()`
anymore for ALL packets triggering the HTTP dissector, but only for the
packets that we already know belong to an HTTP flow. This is the reason
for the changes in RTSP/SOAP/... code

src/include/ndpi_typedefs.h

@@ -741,7 +741,7 @@ struct ndpi_flow_tcp_struct {
   u_int32_t usenet_stage:2;
 
   /* NDPI_PROTOCOL_HTTP */
-  u_int32_t http_stage:2;
+  u_int32_t http_stage:3;
 
   /* NDPI_PROTOCOL_GNUTELLA */
   u_int32_t gnutella_stage:2;		       // 0 - 2
@@ -1581,9 +1581,6 @@ struct ndpi_flow_struct {
   u_int8_t bittorrent_stage;		      // can be 0 - 255
   u_int8_t bt_check_performed : 1;
 
-  /* NDPI_PROTOCOL_HTTP */
-  u_int8_t http_detected:1;
-
   /* NDPI_PROTOCOL_RTSP */
   u_int8_t rtsprdt_stage:2;
 

src/lib/ndpi_main.c

@@ -7232,20 +7232,6 @@ void ndpi_parse_single_packet_line(struct ndpi_detection_module_struct *ndpi_str
     packet->http_response.ptr = &packet->line[0].ptr[NDPI_STATICSTRING_LEN("HTTP/1.1 ")];
     packet->http_response.len = packet->line[0].len - NDPI_STATICSTRING_LEN("HTTP/1.1 ");
     packet->http_num_headers++;
-
-    /* Set server HTTP response code */
-    if(packet->payload_packet_len >= 12) {
-      char buf[4];
-
-      /* Set server HTTP response code */
-      strncpy(buf, (char *) &packet->payload[9], 3);
-      buf[3] = '\0';
-
-      flow->http.response_status_code = atoi(buf);
-      /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
-      if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
-	flow->http.response_status_code = 0; /* Out of range */
-    }
   }
 
   if((packet->parsed_lines == 0) && (packet->line[0].len > 0)) {

src/lib/protocols/gnutella.c

@@ -59,6 +59,10 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
     /* this case works asymmetrically */
     if (packet->payload_packet_len > 17 && memcmp(packet->payload, "GNUTELLA CONNECT/", 17) == 0) {
       ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+      /* Extract some metadata HTTP-like */
+      ndpi_parse_packet_line_info(ndpi_struct, flow);
+      if(packet->user_agent_line.ptr != NULL)
+        ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
       return;
     }
 
@@ -73,6 +77,9 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
 	    || (packet->line[c].len > 36 && memcmp(packet->line[c].ptr,
 						   "Content-Type: application/x-gnutella-", 37) == 0)) {
 	  ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+	  /* Extract some metadata HTTP-like */
+	  if(packet->user_agent_line.ptr != NULL)
+	    ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
 	  return;
 	}
       }
@@ -84,6 +91,9 @@ static void ndpi_search_gnutella(struct ndpi_detection_module_struct *ndpi_struc
 	  || (packet->accept_line.ptr != NULL && packet->accept_line.len > 24
 	      && memcmp(packet->accept_line.ptr, "application n/x-gnutella", 24) == 0)) {
 	ndpi_int_gnutella_add_connection(ndpi_struct, flow, NDPI_CONFIDENCE_DPI);
+	/* Extract some metadata HTTP-like */
+	if(packet->user_agent_line.ptr != NULL)
+	  ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
       }
 
     }