HTTP/2 2020 queries

sql/2020/22_HTTP_2/adoption_of_http_2_by_site_and_requests.sql

@@ -0,0 +1,18 @@
+# standardSQL
+# Adoption of HTTP/2 by site and requests
+SELECT
+  client,
+  firstHtml,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') AS http_version,
+  COUNT(0) AS num_requests,
+  ROUND(COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client), 4) AS pct
+FROM
+  `httparchive.almanac.requests`
+WHERE
+  date = '2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  http_version
+ORDER BY
+  pct DESC

sql/2020/22_HTTP_2/count_of_h2_and_h3_sites_grouped_by_server.sql

@@ -0,0 +1,28 @@
+# standardSQL
+# Count of H2 and H3 Sites Grouped By Server
+SELECT
+  client,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+  # Omit server version
+  NORMALIZE_AND_CASEFOLD(REGEXP_EXTRACT(resp_server, r'\s*([^/]*)\s*')) AS server_header,
+  COUNT(0) AS num_pages,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  ROUND(COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client), 4) AS pct
+FROM
+  `httparchive.almanac.requests`
+WHERE
+  date = '2020-08-01' AND
+  firstHtml AND
+  (LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/2" OR
+   LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "%quic%" OR
+   LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "h3%" OR
+   LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/3%"
+  )
+GROUP BY
+  client,
+  http_version,
+  server_header
+HAVING
+  num_pages >= 100
+ORDER BY
+  num_pages DESC

sql/2020/22_HTTP_2/count_of_h2_and_h3_sites_using_push.sql

@@ -0,0 +1,26 @@
+# standardSQL
+# Count of H2 and H3 Sites using Push
+SELECT
+  client,
+  http_version,
+  COUNT(DISTINCT IF(was_pushed, page, NULL)) AS num_pages_with_push,
+  COUNT(DISTINCT page) AS total,
+  COUNT(DISTINCT IF(was_pushed, page, NULL)) / COUNT(DISTINCT page) AS pct
+FROM (
+  SELECT
+    client,
+    page,
+    JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+    JSON_EXTRACT_SCALAR(payload, '$._was_pushed') = '1' AS was_pushed
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2020-08-01' AND
+    (LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/2" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "%quic%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "h3%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/3%")   
+    )
+GROUP BY
+  client,
+  http_version

sql/2020/22_HTTP_2/count_of_non_h2_and_h3_sites_grouped_by_server.sql

@@ -0,0 +1,27 @@
+# standardSQL
+# Count of non H2 and H3 Sites Grouped By Server
+SELECT
+  client,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+  # Omit server version
+  NORMALIZE_AND_CASEFOLD(REGEXP_EXTRACT(resp_server, r'\s*([^/]*)\s*')) AS server_header,
+  COUNT(0) AS num_pages,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  ROUND(COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client), 4) AS pct
+FROM
+  `httparchive.almanac.requests`
+WHERE
+  date = '2020-08-01' AND
+  firstHtml AND
+  LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) NOT LIKE "http/2" AND
+  LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) NOT LIKE "%quic%" AND
+  LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) NOT LIKE "h3%" AND
+  LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) NOT LIKE "http/3%"
+GROUP BY
+  client,
+  http_version,
+  server_header
+HAVING
+  num_pages >= 100
+ORDER BY
+  num_pages DESC

sql/2020/22_HTTP_2/count_of_preload_http_headers_with_nopush_attribute_set.sql

@@ -0,0 +1,32 @@
+# standardSQL
+# Count of preload HTTP Headers with nopush attribute set. Once off stat for last crawl
+CREATE TEMPORARY FUNCTION getLinkHeaders(payload STRING)
+RETURNS ARRAY<STRING> LANGUAGE js AS """
+try {
+  var $ = JSON.parse(payload);
+  var headers = $.response.headers;
+  return headers.filter(h => h.name.toLowerCase() == 'link').map(h => h.value);
+} catch (e) {
+  return [];
+}
+""";
+
+SELECT
+  client,
+  COUNTIF(link_header LIKE '%nopush%') as num_nopush,
+  COUNT(0) AS total_preload,
+  ROUND(COUNTIF(link_header LIKE '%nopush%') / COUNT(0), 4) AS pct_nopush
+FROM (
+  SELECT
+    client,
+    getLinkHeaders(payload) AS link_headers
+  FROM
+   `httparchive.almanac.requests`
+  WHERE
+   date = '2020-08-01' AND
+   firstHtml),
+  UNNEST(link_headers) AS link_header
+WHERE
+  link_header LIKE '%preload%'
+GROUP BY
+  client

sql/2020/22_HTTP_2/detailed_alt_svc_headers.sql

@@ -0,0 +1,36 @@
+# standardSQL
+# Detailed alt-svc headers
+CREATE TEMPORARY FUNCTION getUpgradeHeader(payload STRING)
+RETURNS STRING LANGUAGE js AS """
+try {
+  var $ = JSON.parse(payload);
+  var headers = $.response.headers;
+  return headers.find(h => h.name.toLowerCase() === 'alt-svc').value.trim();
+} catch (e) {
+  return '';
+}
+""";
+
+SELECT
+  client,
+  firstHtml,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') AS protocol,
+  IF(url LIKE 'https://%', 'https', 'http') AS http_or_https,
+  NORMALIZE_AND_CASEFOLD(getUpgradeHeader(payload)) AS upgrade,
+  COUNT(0) AS num_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
+FROM
+  `httparchive.almanac.requests`
+WHERE
+  date = '2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  protocol,
+  http_or_https,
+  upgrade
+HAVING
+  num_requests >= 100
+ORDER BY
+  num_requests DESC

sql/2020/22_HTTP_2/detailed_upgrade_headers.sql

@@ -0,0 +1,25 @@
+# standardSQL
+# Detailed upgrade headers for 20.04, 20.05 and 20.06
+SELECT
+  client,
+  firstHtml,  
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') AS protocol,
+  IF(url LIKE 'https://%', 'https', 'http') AS http_or_https,
+  regexp_extract(regexp_extract(respOtherHeaders, r'(?is)Upgrade = (.*)'), r'(?im)^([^=]*?)(?:, [a-z-]+ = .*)') IS NOT NULL AS upgrade,
+  COUNT(0) AS num_requests,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct
+FROM
+  `httparchive.almanac.requests`
+WHERE
+  date = '2020-08-01'
+GROUP BY
+  client,
+  firstHtml,
+  protocol,
+  http_or_https,
+  upgrade
+HAVING
+  num_requests >= 100
+ORDER BY
+  num_requests DESC

sql/2020/22_HTTP_2/measure_number_of_tcp_connections_per_site.sql

@@ -0,0 +1,36 @@
+# standardSQL
+# Measure number of TCP Connections per site.
+SELECT
+  percentile,
+  client,
+  protocol,
+  COUNT(0) AS num_pages,
+  APPROX_QUANTILES(_connections, 1000)[OFFSET(percentile * 10)] AS connections
+FROM (
+  SELECT
+    client,
+    page,
+    JSON_EXTRACT_SCALAR(payload, '$._protocol') AS protocol,
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2020-08-01' AND
+    firstHtml)
+JOIN (
+  SELECT
+    _TABLE_SUFFIX AS client,
+    url AS page,
+    _connections,
+  FROM
+    `httparchive.summary_pages.2020_08_01_*`)
+USING
+  (client, page),
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  protocol
+ORDER BY
+  percentile,
+  client,
+  protocol

sql/2020/22_HTTP_2/measure_of_all_http_versions_for_main_page_of_all_sites.sql

@@ -0,0 +1,20 @@
+# standardSQL
+# Measure of all HTTP versions (0.9, 1.0, 1.1, 2, QUIC) for main page of all sites, and for HTTPS sites. Table for last crawl.
+SELECT
+  client,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') AS protocol,
+  COUNT(0) AS num_pages,
+  SUM(COUNT(0)) OVER (PARTITION BY client) AS total,
+  COUNTIF(url LIKE 'https://%') AS num_https_pages,
+  COUNT(0) / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_pages,
+  COUNTIF(url LIKE 'https://%') / SUM(COUNT(0)) OVER (PARTITION BY client) AS pct_https
+FROM
+   `httparchive.almanac.requests`
+WHERE
+   date = '2020-08-01' AND
+   firstHtml
+GROUP BY
+  client,
+  protocol
+ORDER BY
+  num_pages / total DESC

sql/2020/22_HTTP_2/number_of_h2_and h3_pushed_resources_and_bytes_transferred.sql

@@ -0,0 +1,37 @@
+# standardSQL
+# Number of H2 and H3 Pushed Resources and bytes transferred
+SELECT
+  percentile,
+  client,
+  http_version,
+  COUNT(DISTINCT page) AS num_pages,
+  APPROX_QUANTILES(num_requests, 1000)[OFFSET(percentile * 10)] AS pushed_requests,
+  APPROX_QUANTILES(kb_transfered, 1000)[OFFSET(percentile * 10)] AS kb_transfered,
+FROM (
+ SELECT
+    client,
+    page,
+    JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+    SUM(CAST(JSON_EXTRACT_SCALAR(payload, '$._bytesIn') AS INT64) / 1024) AS kb_transfered,
+    COUNT(0) AS num_requests
+  FROM
+    `httparchive.almanac.requests`
+  WHERE
+    date = '2020-08-01' AND
+    JSON_EXTRACT_SCALAR(payload, '$._was_pushed') = '1' AND
+    (LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/2" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "%quic%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "h3%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/3%")   
+  GROUP BY
+    client,
+    http_version,
+    page),
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  http_version
+ORDER BY
+  percentile,
+  client

sql/2020/22_HTTP_2/number_of_h2_and_h3_pushed_resources_and_bytes_by_content_type.sql

@@ -0,0 +1,41 @@
+# standardSQL
+# Number of H2 and H3 Pushed Resources and Bytes by Content type
+SELECT 
+  percentile,
+  client,
+  http_version,
+  content_type,
+  COUNT(DISTINCT page) AS num_pages,
+  APPROX_QUANTILES(num_requests, 1000)[OFFSET(500)] AS pushed_requests,
+  APPROX_QUANTILES(kb_transfered, 1000)[OFFSET(500)] AS kb_transfered,
+FROM (
+  SELECT 
+    client,
+    page,
+    JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+    JSON_EXTRACT_SCALAR(payload, "$._contentType") as content_type,
+    SUM(CAST(JSON_EXTRACT_SCALAR(payload, "$._bytesIn") AS INT64)/1024) AS kb_transfered,
+    COUNT(0) AS num_requests
+  FROM 
+    `httparchive.almanac.requests`  
+  WHERE 
+    date='2020-08-01' AND
+    JSON_EXTRACT_SCALAR(payload, "$._was_pushed") = "1" AND
+    (LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/2" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "%quic%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "h3%" OR
+     LOWER(JSON_EXTRACT_SCALAR(payload, "$._protocol")) LIKE "http/3%")  
+  GROUP BY 
+    client,
+    http_version,
+    page,
+    content_type),
+  UNNEST([10, 25, 50, 75, 90]) AS percentile
+GROUP BY
+  percentile,
+  client,
+  http_version,
+  content_type
+ORDER BY
+  percentile,
+  client

sql/2020/22_HTTP_2/number_of_http_requests_returning_upgrade_http_header_containing_h2.sql

@@ -0,0 +1,32 @@
+# standardSQL
+# Number of HTTP (not HTTPS) requests which return upgrade HTTP header containing h2.
+CREATE TEMPORARY FUNCTION getUpgradeHeader(payload STRING)
+RETURNS STRING
+LANGUAGE js AS """
+  try {
+    var $ = JSON.parse(payload);
+    var headers = $.response.headers;
+    var st = headers.find(function(e) { 
+      return e['name'].toLowerCase() === 'upgrade'
+    });
+    return st['value'];
+  } catch (e) {
+    return '';
+  }
+""";
+SELECT 
+  client,
+  firstHtml,
+  JSON_EXTRACT_SCALAR(payload, '$._protocol') as http_version,
+  COUNTIF(getUpgradeHeader(payload) LIKE "%h2%") AS num_requests,
+  COUNT(0) AS total
+FROM 
+  `httparchive.almanac.requests`
+WHERE
+  date='2020-08-01' AND 
+  url LIKE "http://%" AND
+  getUpgradeHeader(payload) LIKE "%h2%"
+GROUP BY
+  client,
+  firstHtml,
+  http_version