HTTP/2 2020 queries

sql/2020/22_HTTP_2/adoption_of_http_2_by_site_and_requests.sql

@@ -0,0 +1,20 @@
+#standardSQL
+# 22_HTTP_2 - Adoption of HTTP/2 by site and requests
+SELECT 
+  client,
+  firstHtml, 
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS http_version, 
+  COUNT(*) AS num_requests,
+  ROUND(COUNT(0) * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct
+FROM 
+  `httparchive.almanac.requests` 
+WHERE
+  date='2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  http_version
+ORDER BY 
+  client,
+  firstHtml,
+  http_version

sql/2020/22_HTTP_2/avg_number_of_h2_pushed_resources_and_avg_bytes.sql

@@ -0,0 +1,26 @@
+#standardSQL
+# 22_HTTP_2 - Average number of HTTP/2 Pushed Resources and Average Bytes
+SELECT 
+  client,
+  COUNT(DISTINCT page) AS num_pages,
+  ROUND(AVG(num_requests),2) AS avg_pushed_requests,
+  ROUND(AVG(kb_transfered),2) AS avg_kb_transfered
+FROM (
+
+SELECT 
+    client,
+    page,
+    SUM(CAST(JSON_EXTRACT_SCALAR(payload, "$._bytesIn") AS INT64)/1024) AS kb_transfered,
+    COUNT(*) AS num_requests
+  FROM 
+    `httparchive.almanac.requests` 
+  WHERE 
+    JSON_EXTRACT_SCALAR(payload, "$._protocol") = "HTTP/2"
+    AND JSON_EXTRACT_SCALAR(payload, "$._was_pushed") = "1"
+    AND date='2020-08-01'
+  GROUP BY 
+    client,
+    page
+)
+GROUP BY
+  client

sql/2020/22_HTTP_2/avg_number_of_h2_pushed_resources_and_avg_bytes_by_content_type.sql

@@ -0,0 +1,30 @@
+#standardSQL
+# 22_HTTP_2 - Average number of HTTP/2 Pushed Resources and Average Bytes by Content type
+SELECT 
+  client,
+  content_type,
+  COUNT(DISTINCT page) AS num_pages,
+  ROUND(AVG(num_requests),2) AS avg_pushed_requests,
+  ROUND(AVG(kb_transfered),2) AS avg_kb_transfered
+FROM (
+
+SELECT 
+    client,
+    page,
+    JSON_EXTRACT_SCALAR(payload, "$._contentType") as content_type,
+    SUM(CAST(JSON_EXTRACT_SCALAR(payload, "$._bytesIn") AS INT64)/1024) AS kb_transfered,
+    COUNT(*) AS num_requests
+  FROM 
+    `httparchive.almanac.requests` 
+  WHERE 
+    JSON_EXTRACT_SCALAR(payload, "$._protocol") = "HTTP/2"
+    AND JSON_EXTRACT_SCALAR(payload, "$._was_pushed") = "1"
+    AND date='2020-08-01'
+  GROUP BY 
+    client,
+    page,
+    content_type
+)
+GROUP BY
+  client,
+  content_type

sql/2020/22_HTTP_2/avg_percent_of_resources_loaded_over_h2_or_h1_1_per_site.sql

@@ -0,0 +1,25 @@
+#standardSQL
+# 22_HTTP_2 - Average percentage of resources loaded over HTTP/2 or HTTP/1.1 per site
+SELECT
+  client,
+  ROUND(AVG(http_1_1/num_requests) * 100,2) avg_pct_http_1_1,
+  ROUND(AVG(http_2/num_requests) * 100,2) avg_pct_http_2
+FROM (
+  SELECT 
+    client,
+    page,
+    COUNT(*) AS num_requests, 
+    SUM(IF(JSON_EXTRACT_SCALAR(payload, "$._protocol") ="http/0.9",1,0)) AS http_0_9, 
+    SUM(IF(JSON_EXTRACT_SCALAR(payload, "$._protocol") ="http/1.0",1,0)) AS http_1_0, 
+    SUM(IF(JSON_EXTRACT_SCALAR(payload, "$._protocol") ="http/1.1",1,0)) AS http_1_1,
+    SUM(IF(JSON_EXTRACT_SCALAR(payload, "$._protocol") ="HTTP/2",1,0)) AS http_2
+  FROM 
+    `httparchive.almanac.requests`
+  WHERE
+    date='2020-08-01'
+  GROUP BY
+    client,
+    page
+)
+GROUP BY 
+  client

sql/2020/22_HTTP_2/count_of_h2_sites_grouped_by_server.sql

@@ -0,0 +1,35 @@
+#standardSQL
+# 22_HTTP_2 - Count of HTTP/2 Sites Grouped By Server
+CREATE TEMPORARY FUNCTION getServerHeader(payload STRING)
+RETURNS STRING
+LANGUAGE js AS """
+  try {
+    var $ = JSON.parse(payload);
+    var headers = $.response.headers;
+    // Find server header
+    var st = headers.find(function(e) { 
+      return e['name'].toLowerCase() === 'server'
+    });
+    // Remove everything after / in the server header value and return
+    return st['value'].split("/")[0]; 
+  } catch (e) {
+    return '';
+  }
+""";
+
+SELECT 
+  client,
+  getServerHeader(payload) AS server_header,
+  COUNT(*) AS num_pages,
+  ROUND(COUNT(0) * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct
+FROM 
+  `httparchive.almanac.requests` 
+WHERE
+  firstHtml
+  AND JSON_EXTRACT_SCALAR(payload, "$._protocol") = "HTTP/2"
+  AND date='2020-08-01'
+GROUP BY
+  client,
+  server_header
+ORDER BY 
+  num_pages DESC

sql/2020/22_HTTP_2/count_of_h2_sites_using_h2_push.sql

@@ -0,0 +1,19 @@
+#standardSQL
+# 22_HTTP_2 - Count of HTTP/2 Sites using HTTP/2 Push
+SELECT 
+  client,
+  COUNT(DISTINCT page) AS num_pages
+FROM (
+
+  SELECT 
+    client,
+    page
+  FROM 
+    `httparchive.almanac.requests` 
+  WHERE 
+    JSON_EXTRACT_SCALAR(payload, "$._protocol") = "HTTP/2"
+    AND JSON_EXTRACT_SCALAR(payload, "$._was_pushed") = "1"
+    AND date='2020-08-01'
+)
+GROUP BY
+  client

sql/2020/22_HTTP_2/count_of_non_h2_sites_grouped_by_server.sql

@@ -0,0 +1,35 @@
+#standardSQL
+# 22_HTTP_2 - Count of non-HTTP/2 Sites Grouped By Server
+CREATE TEMPORARY FUNCTION getServerHeader(payload STRING)
+RETURNS STRING
+LANGUAGE js AS """
+  try {
+    var $ = JSON.parse(payload);
+    var headers = $.response.headers;
+    // Find server header
+    var st = headers.find(function(e) { 
+      return e['name'].toLowerCase() === 'server'
+    });
+    // Remove everything after / in the server header value and return
+    return st['value'].split("/")[0]; 
+  } catch (e) {
+    return '';
+  }
+""";
+
+SELECT 
+  client,
+  getServerHeader(payload) AS server_header,
+  COUNT(*) AS num_pages,
+  ROUND(COUNT(0) * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct
+FROM 
+  `httparchive.almanac.requests` 
+WHERE
+  firstHtml
+  AND JSON_EXTRACT_SCALAR(payload, "$._protocol") != "HTTP/2"
+  AND date='2020-08-01'
+GROUP BY
+  client,
+  server_header
+ORDER BY
+  num_pages DESC

sql/2020/22_HTTP_2/count_of_preload_http_headers_with_nopush_attribute_set.sql

@@ -0,0 +1,40 @@
+#standardSQL
+# 22_HTTP_2 - Count of preload HTTP Headers with nopush attribute set. Once off stat for last crawl
+CREATE TEMPORARY FUNCTION getLinkHeaders(payload STRING)
+RETURNS ARRAY<STRING>
+LANGUAGE js AS """
+  var $ = JSON.parse(payload);
+  var headers = $.response.headers;
+  var preload=[];
+
+  for (i in headers) {
+      if (headers[i].name.toLowerCase() === 'link')
+        preload.push(headers[i].value);
+      }
+     return preload;  
+
+""";
+
+SELECT 
+  client, 
+  firstHtml, 
+  COUNT(*) as num_requests,
+  ROUND(COUNT(0) * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct
+FROM (
+  SELECT 
+    client,
+    firstHtml,  
+    getLinkHeaders(payload) AS link_headers
+  FROM 
+   `httparchive.almanac.requests`
+  WHERE
+   date='2020-08-01'
+)
+CROSS JOIN
+  UNNEST(link_headers) AS link_header
+WHERE 
+  link_header LIKE '%preload%' 
+  AND link_header LIKE '%nopush%'
+GROUP BY
+	client, 
+	firstHtml

sql/2020/22_HTTP_2/detailed_alt_svc_headers.sql

@@ -0,0 +1,34 @@
+#standardSQL
+# 22_HTTP_2 - Detailed alt-svc headers
+CREATE TEMPORARY FUNCTION getUpgradeHeader(payload STRING)
+RETURNS STRING
+LANGUAGE js AS """
+  try {
+    var $ = JSON.parse(payload);
+    var headers = $.response.headers;
+    var st = headers.find(function(e) { 
+      return e['name'].toLowerCase() === 'alt-svc'
+    });
+    return st['value'];
+  } catch (e) {
+    return '';
+  }
+""";
+
+SELECT 
+  client,
+  firstHtml, 
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS protocol,
+  IF(url LIKE "https://%","https","http") AS http_or_https,
+  getUpgradeHeader(payload) AS upgrade,
+  COUNT(*) AS num_requests
+FROM 
+  `httparchive.almanac.requests`
+WHERE
+  date='2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  protocol,
+  http_or_https,
+  upgrade 

sql/2020/22_HTTP_2/detailed_upgrade_headers.sql

@@ -0,0 +1,34 @@
+#standardSQL
+# 22_HTTP_2 - Detailed upgrade headers for 20.04, 20.05 and 20.06
+CREATE TEMPORARY FUNCTION getUpgradeHeader(payload STRING)
+RETURNS STRING
+LANGUAGE js AS """
+  try {
+    var $ = JSON.parse(payload);
+    var headers = $.response.headers;
+    var st = headers.find(function(e) { 
+      return e['name'].toLowerCase() === 'upgrade'
+    });
+    return st['value'];
+  } catch (e) {
+    return '';
+  }
+""";
+
+SELECT 
+  client,
+  firstHtml,  
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS protocol,
+  IF(url LIKE "https://%","https","http") AS http_or_https,
+  getUpgradeHeader(payload) AS upgrade,
+  COUNT(*) AS num_requests
+FROM 
+  `httparchive.almanac.requests`
+WHERE
+  date='2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  protocol,
+  http_or_https,
+  upgrade

sql/2020/22_HTTP_2/measure_number_of_tcp_connections_per_site.sql

@@ -0,0 +1,41 @@
+#standardSQL
+# 22_HTTP_2 - Measure number of TCP Connections per site.
+SELECT 
+  "mobile" AS client,
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS protocol,
+  COUNT(*) AS num_pages,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(50)] AS median,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(75)] AS p75,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(95)] AS p95
+FROM 
+  `httparchive.requests.2020_08_01_mobile` AS requests
+INNER JOIN
+  `httparchive.summary_pages.2020_08_01_mobile` AS summary
+ON
+  requests.url = summary.url
+WHERE
+  JSON_EXTRACT_SCALAR(payload, "$._is_base_page") = "true"
+GROUP BY 
+  client, 
+  protocol
+
+UNION ALL
+
+SELECT 
+  "desktop" AS client,
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS protocol,
+  COUNT(*) AS num_pages,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(50)] AS median,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(75)] AS p75,
+  APPROX_QUANTILES(_connections, 100)[SAFE_ORDINAL(95)] AS p95
+FROM 
+  `httparchive.requests.2020_08_01_desktop` AS requests
+INNER JOIN
+  `httparchive.summary_pages.2020_08_01_desktop` AS summary
+ON
+  requests.url = summary.url
+WHERE
+  JSON_EXTRACT_SCALAR(payload, "$._is_base_page") = "true"
+GROUP BY 
+  client, 
+  protocol

sql/2020/22_HTTP_2/measure_of_all_http_versions_for_main_page_of_all_sites.sql

@@ -0,0 +1,20 @@
+#standardSQL
+# 22_HTTP_2 - Measure of all HTTP versions (0.9, 1.0, 1.1, 2, QUIC) for main page of all sites, and for HTTPS sites. Table for last crawl.
+SELECT 
+  client,
+  JSON_EXTRACT_SCALAR(payload, "$._protocol") AS protocol,
+  COUNT(0) AS num_pages,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total, 
+  COUNTIF(url LIKE "https://%") AS num_https_pages,
+  ROUND(COUNT(0) * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct_pages,
+  ROUND(COUNTIF(url LIKE "https://%") * 100 / SUM(COUNT(0)) OVER (PARTITION BY client), 2) AS pct_https
+FROM 
+   `httparchive.almanac.requests`
+WHERE
+   firstHtml
+   AND date='2020-08-01'
+GROUP BY
+  client,
+  protocol
+ORDER BY 
+  num_pages / total DESC