feat: introduce test group

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
 - Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145)
 - Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152)
+- Test Group Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/156)
 
 ## [0.3.0] - 2023-07-31
 ### Added

aggregate-into-table.js

@@ -1,6 +1,7 @@
 const fs = require("fs");
 
 const TestMetadata = "TestMetadata";
+const METADATA_TEST_GROUP = "group";
 
 // retrieve the list of input files from the command line
 const files = process.argv.slice(2);
@@ -10,24 +11,77 @@ const inputs = files.map((file) => {
     return JSON.parse(fs.readFileSync(file, 'utf8'));
 });
 
-// merge all the unique keys from all the inputs
-let keys = new Set();
+// merge all the unique keys & metadata from all the inputs
+const metadata = {}
 inputs.forEach((input) => {
     Object.keys(input).forEach((key) => {
-        keys.add(key);
+        metadata[key] = { ...metadata[key], ...input[key]["meta"] || {} };
     });
 });
-keys.delete(TestMetadata); // Extract TestMetadata which is a special case
-keys = Array.from(keys).sort();
+delete metadata[TestMetadata]; // Extract TestMetadata which is a special case
+
+// generate groups: an array of {group, key} objects
+// where group is the group name (or undefined), and key is the test key name (or undefined)
+// It represents the table leftmost column.
+// 
+// Group1
+//  Group1 - Test1
+//  Group1 - Test2
+// Group2
+// ...
+const groups = []
+const groupsAdded = new Set();
+Object.entries(metadata).forEach(([key, value]) => {
+    const group = value[METADATA_TEST_GROUP] || undefined;
+
+    if (!groupsAdded.has(group)) {
+        groups.push({ group, key: undefined });
+        groupsAdded.add(group);
+    }
+
+    groups.push({ group, key });
+});
+
+// sort the groups so that the tests are ordered by group, then by key.
+// undefined groups are always at the end.
+groups.sort((a, b) => {
+    if (a.group === b.group) {
+        if (a.key === undefined) {
+            return -1;
+        }
+        if (b.key === undefined) {
+            return 1;
+        }
+        return a.key.localeCompare(b.key);
+    }
+
+    if (a.group === undefined) {
+        return 1;
+    }
+
+    if (b.group === undefined) {
+        return -1;
+    }
+
+    return a.group.localeCompare(b.group);
+});
 
 // generate a table
 const columns = [];
 
 // add the leading column ("gateway", "version", "key1", "key2", ... "keyN")
 const leading = ["gateway", "version"];
-keys.forEach((key) => {
+groups.forEach(({ group, key }) => {
+    if (key === undefined) {
+        leading.push(`**${group || 'Other'}**`);
+        return;
+    }
+
+    const m = metadata[key];
+
     // Skip the "Test" prefix
-    const niceKey = key.replace(/^Test/, '');
+    let niceKey = key.replace(/^Test/, '');
+
     leading.push(niceKey);
 });
 columns.push(leading);
@@ -73,7 +127,11 @@ inputs.forEach((input, index) => {
     const col = [name, versionCell];
 
     // extract results
-    keys.forEach((key) => {
+    groups.forEach(({ group, key }) => {
+        if (key === undefined) {
+            col.push(null);
+            return;
+        }
         col.push(cellRender(input[key] || null));
     });
     columns.push(col);

aggregate.js

@@ -16,7 +16,7 @@ lines = lines.filter((line) => {
 // # extract test metadata
 //   action is output, and starts with ".* --- META: (.*)"
 //   see details in https://github.com/ipfs/gateway-conformance/pull/125
-const getMetadata = (line) => {
+const extractMetadata = (line) => {
   const { Action, Output } = line;
 
   if (Action !== "output") {
@@ -34,7 +34,7 @@ const getMetadata = (line) => {
 }
 
 lines = lines.map((line) => {
-  const metadata = getMetadata(line);
+  const metadata = extractMetadata(line);
 
   if (!metadata) {
     return line;
@@ -87,6 +87,44 @@ lines.forEach((line) => {
   });
 })
 
+// prepare metadata up the tree
+const metadataTree = {};
+
+// sort lines so that the one with the longest path is processed first
+const sortedLines = lines.sort((a, b) => {
+  return b.Path.length - a.Path.length;
+});
+
+sortedLines.forEach((line) => {
+  const { Path, Action, Metadata } = line;
+  let current = metadataTree;
+
+  if (Action !== "meta") {
+    return;
+  }
+
+  Path.forEach((path) => {
+    if (!current[path]) {
+      current[path] = {};
+    }
+    current = current[path];
+    current["meta"] = { ...current["meta"], ...Metadata };
+  });
+});
+
+const getMetadata = (path) => {
+  let current = metadataTree;
+
+  path.forEach((path) => {
+    if (!current[path]) {
+      return null;
+    }
+    current = current[path];
+  });
+
+  return current["meta"];
+}
+
 // # Drop all lines where the Test "Path" does not point to a leaf
 //   if the test has children then we don't really care about it's pass / fail / skip status,
 //   we'll aggregate its children results'
@@ -118,7 +156,7 @@ lines.forEach((line) => {
   const key = path.join(" > ");
 
   if (!current[key]) {
-    current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} };
+    current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": getMetadata(path) || {} };
   }
   current = current[key];
 

tests/metadata_test.go

@@ -11,3 +11,11 @@ func TestMetadata(t *testing.T) {
 	tooling.LogJobURL(t)
 	tooling.LogGatewayURL(t)
 }
+
+const (
+	GroupTrustlessGateway = "Trustless Gateway"
+	GroupPathGateway      = "Path Gateway"
+	GroupSubdomainGateway = "Subdomain Gateway"
+	GroupCORS             = "CORS"
+	GroupIPNS             = "IPNS"
+)

tests/path_gateway_cors_test.go

@@ -3,11 +3,14 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestCors(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	cidHello := "bafkqabtimvwgy3yk" // hello
 
 	tests := SugarTests{

tests/path_gateway_dag_test.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/ipns"
@@ -12,6 +13,8 @@ import (
 )
 
 func TestGatewayJsonCbor(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")
 
 	fileJSON := fixture.MustGetNode("ą", "ę", "t.json")
@@ -66,6 +69,8 @@ func TestGatewayJsonCbor(t *testing.T) {
 // ## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON
 // ## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format)
 func TestDagPbConversion(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")
 
 	dir := fixture.MustGetRoot()
@@ -205,6 +210,8 @@ func TestDagPbConversion(t *testing.T) {
 // # Requesting CID with plain json (0x0200) and cbor (0x51) codecs
 // # (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec)
 func TestPlainCodec(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	table := []struct {
 		Name        string
 		Format      string
@@ -308,6 +315,8 @@ func TestPlainCodec(t *testing.T) {
 
 // ## Pathing, traversal over DAG-JSON and DAG-CBOR
 func TestPathing(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	dagJSONTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-json-traversal.car").MustGetRoot()
 	dagCBORTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-cbor-traversal.car").MustGetRoot()
 
@@ -381,6 +390,8 @@ func TestPathing(t *testing.T) {
 // ## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71):
 // ## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested
 func TestNativeDag(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	missingCID := car.RandomCID()
 
 	table := []struct {
@@ -570,6 +581,8 @@ func TestNativeDag(t *testing.T) {
 }
 
 func TestGatewayJSONCborAndIPNS(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	ipnsIdDagJSON := "k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2"
 	ipnsIdDagCBOR := "k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l"
 

tests/path_gateway_ipns_test.go

@@ -3,11 +3,14 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestRedirectCanonicalIPNS(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	tests := SugarTests{
 		{
 			Name: "GET for /ipns/{b58-multihash-of-ed25519-key} redirects to /ipns/{cidv1-libp2p-key-base36}",

tests/path_gateway_raw_test.go

@@ -5,12 +5,15 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestGatewayBlock(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("gateway-raw-block.car")
 
 	tests := SugarTests{

tests/path_gateway_tar_test.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
@@ -11,6 +12,8 @@ import (
 )
 
 func TestTar(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixtureOutside := car.MustOpenUnixfsCar("path_gateway_tar/outside-root.car")
 	fixtureInside := car.MustOpenUnixfsCar("path_gateway_tar/inside-root.car")
 

tests/path_gateway_unixfs_test.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/ipns"
@@ -13,6 +14,8 @@ import (
 )
 
 func TestUnixFSDirectoryListing(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car")
 	root := fixture.MustGetNode()
 	file := fixture.MustGetNode("ą", "ę", "file-źł.txt")
@@ -76,6 +79,8 @@ func TestUnixFSDirectoryListing(t *testing.T) {
 }
 
 func TestGatewayCache(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car")
 
 	tests := SugarTests{
@@ -308,6 +313,8 @@ func TestGatewayCache(t *testing.T) {
 }
 
 func TestGatewayCacheWithIPNS(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("gateway-cache/fixtures.car")
 	ipns := ipns.MustOpenIPNSRecordWithKey("gateway-cache/k51qzi5uqu5dlxdsdu5fpuu7h69wu4ohp32iwm9pdt9nq3y5rpn3ln9j12zfhe.ipns-record")
 	ipnsKey := ipns.Key()
@@ -404,6 +411,8 @@ func TestGatewayCacheWithIPNS(t *testing.T) {
 }
 
 func TestGatewaySymlink(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	fixture := car.MustOpenUnixfsCar("path_gateway_unixfs/symlink.car")
 	rootDirCID := fixture.MustGetCid()
 
@@ -443,6 +452,8 @@ func TestGatewaySymlink(t *testing.T) {
 }
 
 func TestGatewayUnixFSFileRanges(t *testing.T) {
+	tooling.LogTestGroup(t, GroupPathGateway)
+
 	// Multi-range requests MUST conform to the HTTP semantics. The server does not
 	// need to be able to support returning multiple ranges. However, it must respond
 	// correctly.