Add support for Notation3 test suite

* feat: support Notation3 spec

* fix: skip rejected tests and add option to only run approved tests

* chore: add tests for skipping rejected / unapproved cases

* chore: add tests for loading rdft:approval

* fix: make url normalization opt-in

* chore: add tests for optional url normalization

* chore: document N3 spec and approval/rejection CLI args

README.md

@@ -23,6 +23,7 @@ Currently, the following test suites are supported:
 * [JSON-LD (1.0 and 1.1)](https://w3c.github.io/json-ld-api/tests/)
 * [RDFa](http://rdfa.info/test-suite/)
 * [Microdata to RDF](https://w3c.github.io/microdata-rdf/tests/)
+* [Notation3](https://w3c.github.io/N3/spec/)
 * [RDF-star and SPARQL-star](https://www.w3.org/2021/12/rdf-star.html)
 * [JSON-LD-Star](https://json-ld.github.io/json-ld-star/)
 
@@ -248,6 +249,31 @@ $ rdf-test-suite myengine.js http://w3c.github.io/rdf-tests/sparql11/data-sparql
   -m http://w3c.github.io/rdf-tests/sparql11/data-sparql11/~/path/to/my/files/
 ```
 
+### Using rejected tests
+
+In cases where you wish to execute a test suite that contains
+`rdft:Rejected` test entries rdf-test-suite will skip those tests
+by default. If you wish to execute those tests as well, you can
+use the `-r` option.
+
+```bash
+$ rdf-test-suite myengine.js http://w3c.github.io/rdf-tests/sparql11/data-sparql11/manifest-all.ttl \
+  -r
+```
+
+### Only using explicitly approved tests
+
+In cases where you wish to execute a test suite that contains
+`rdft:Approved` test entries rdf-test-suite will, by default
+execute those tests as well as all other tests that are not
+explicitly rejected. If you wish to only execute those tests
+that are explicitly approved, you can use the `-a` option.
+
+```bash
+$ rdf-test-suite myengine.js http://w3c.github.io/rdf-tests/sparql11/data-sparql11/manifest-all.ttl \
+  -a
+```
+
 ## Supported test suites
 
 | Manifest | Specification | Interface | Entry manifest |
@@ -274,6 +300,8 @@ $ rdf-test-suite myengine.js http://w3c.github.io/rdf-tests/sparql11/data-sparql
 | [JSON-LD-Star Test Suite](https://json-ld.github.io/json-ld-star/tests/) | [JSON-LD-Star](https://json-ld.github.io/json-ld-star/) | [`ISerializer`](https://github.com/rubensworks/rdf-test-suite.js/blob/master/lib/testcase/rdfsyntax/ISerializer.ts) | https://json-ld.github.io/json-ld-star/tests/fromRdf-manifest.jsonld |
 | [RDFa Test Suite](http://rdfa.info/test-suite/) | [RDFa 1.1](https://www.w3.org/TR/html-rdfa/) | [`IParser`](https://github.com/rubensworks/rdf-test-suite.js/blob/master/lib/testcase/rdfsyntax/IParser.ts) | http://rdfa.info/test-suite/test-cases/rdfa1.1/html5/manifest.ttl |
 | [Microdata to RDF Test Suite](https://w3c.github.io/microdata-rdf/tests/) | [Microdata to RDF](https://w3c.github.io/microdata-rdf/) | [`IParser`](https://github.com/rubensworks/rdf-test-suite.js/blob/master/lib/testcase/rdfsyntax/IParser.ts) | https://w3c.github.io/microdata-rdf/tests/manifest.ttl |
+| [Notation3 Test Suite](https://w3c.github.io/N3/tests/) | [N3 Grammar](https://w3c.github.io/N3/tests/N3Tests/manifest-parser.ttl) | [`IParser`](https://github.com/rubensworks/rdf-test-suite.js/blob/master/lib/testcase/rdfsyntax/IParser.ts) | https://w3c.github.io/N3/tests/N3Tests/manifest-parser.ttl |
+| [Notation3 Test Suite](https://w3c.github.io/N3/tests/) | [Extended N3 Grammar](https://w3c.github.io/N3/tests/N3Tests/manifest-extended.ttl) | [`IParser`](https://github.com/rubensworks/rdf-test-suite.js/blob/master/lib/testcase/rdfsyntax/IParser.ts) | https://w3c.github.io/N3/tests/N3Tests/manifest-extended.ttl |
 | [Turtle-star Evaluation Tests](https://w3c.github.io/rdf-star/tests/turtle/eval/manifest.html) | [RDF-star and SPARQL-star](https://www.w3.org/2021/12/rdf-star.html) | ✖ | https://w3c.github.io/rdf-star/tests/turtle/eval/manifest.jsonld |
 | [Turtle-star Syntax Tests](https://w3c.github.io/rdf-star/tests/turtle/syntax/manifest.html) | [RDF-star and SPARQL-star](https://www.w3.org/2021/12/rdf-star.html) | ✖ | https://w3c.github.io/rdf-star/tests/turtle/syntax/manifest.jsonld |
 | [TriG-star Evaluation Tests](https://w3c.github.io/rdf-star/tests/trig/eval/manifest.html) | [RDF-star and SPARQL-star](https://www.w3.org/2021/12/rdf-star.html) | ✖ | https://w3c.github.io/rdf-star/tests/trig/eval/manifest.jsonld |

bin/Runner.ts

@@ -30,6 +30,8 @@ Options:
   -i    JSON string with custom options that need to be passed to the engine
   -d    time out duration for test cases (in milliseconds, default 3000)
   -m    URL to local path mapping (e.g. 'https://w3c.github.io/json-ld-api/|/path/to/folder/')
+  -a    Only run tests that have an explicit rdft:Accepted status [default: false]
+  -r    Run tests that have an explicit rdft:Rejected status [default: false]
 `);
   process.exit(1);
 }
@@ -61,6 +63,8 @@ const config: ITestSuiteConfig = {
   testRegex: new RegExp(args.t),
   timeOutDuration: args.d || defaultConfig.timeOutDuration,
   urlToFileMapping: args.m,
+  runRejected: !!args.r,
+  explicitApproval: !!args.a,
 };
 
 // Fetch the manifest, run the tests, and print them

lib/TestSuiteRunner.ts

@@ -15,6 +15,8 @@ const DF = new DataFactory();
 
 export interface ITestSuiteConfig {
   exitWithStatusCode0: boolean;
+  explicitApproval?: boolean;
+  runRejected?: boolean;
   outputFormat: string;
   timeOutDuration: number;
   customEngingeOptions: object;
@@ -91,33 +93,41 @@ export class TestSuiteRunner {
     if (manifest.testEntries) {
       for (const test of manifest.testEntries) {
         if (!config.testRegex || config.testRegex.test(test.uri)) {
-          let timeout: Timeout = null;
-          const timeStart = process.hrtime();
-          let testResultOverride: ITestResultOverride | undefined;
-          try {
-            await Promise.race([
-              test.test(handler, config.customEngingeOptions)
-                .then((result) => {
-                  if (result) {
-                    testResultOverride = result;
-                  }
-                }),
-              new Promise((res, rej) => {
-                // global. is needed because TSC may otherwise pick the browser version of setTimeout, which returns int
-                timeout = global.setTimeout(
-                  () => rej(new Error(`Test case '${test.uri}' timed out`)),
-                  config.timeOutDuration);
-              },
-              ),
-            ]);
-          } catch (error) {
+          if (!config.runRejected && test.approval === 'http://www.w3.org/ns/rdftest#Rejected') {
+            // Skip tests that are explicitly rejected
+            results.push({ test, ok: false, skipped: true, error: new Error('Rejected Test') });
+          } else if (config.explicitApproval && test.approval !== 'http://www.w3.org/ns/rdftest#Approved') {
+            // Skip tests that are explicitly rejected
+            results.push({ test, ok: false, skipped: true, error: new Error('Test not explicitly approved') });
+          } else {
+            let timeout: Timeout = null;
+            const timeStart = process.hrtime();
+            let testResultOverride: ITestResultOverride | undefined;
+            try {
+              await Promise.race([
+                test.test(handler, config.customEngingeOptions)
+                  .then((result) => {
+                    if (result) {
+                      testResultOverride = result;
+                    }
+                  }),
+                new Promise((res, rej) => {
+                  // global. is needed because TSC may otherwise pick the browser version of setTimeout, which returns int
+                  timeout = global.setTimeout(
+                    () => rej(new Error(`Test case '${test.uri}' timed out`)),
+                    config.timeOutDuration);
+                },
+                ),
+              ]);
+            } catch (error) {
+              clearTimeout(timeout);
+              results.push({ test, ok: false, error, skipped: error.skipped });
+              continue;
+            }
+            const timeEnd = process.hrtime(timeStart);
             clearTimeout(timeout);
-            results.push({ test, ok: false, error, skipped: error.skipped });
-            continue;
+            results.push({ test, ok: true, duration: (timeEnd[0] * 1000) + (timeEnd[1] / 1000000), ...(testResultOverride || {}) });
           }
-          const timeEnd = process.hrtime(timeStart);
-          clearTimeout(timeout);
-          results.push({ test, ok: true, duration: (timeEnd[0] * 1000) + (timeEnd[1] / 1000000), ...(testResultOverride || {}) });
         }
       }
     }

lib/context-manifest.json

@@ -11,6 +11,9 @@
   "approval": "dawg:approval",
   "approvedBy": "dawg:approvedBy",
 
+  "rdft": "http://www.w3.org/ns/rdftest#",
+  "rdftApproval": "rdft:approval",
+
   "mf": "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#",
   "action": "mf:action",
   "conformanceRequirements": "mf:conformanceRequirement",

lib/testcase/ITestCase.ts

@@ -31,7 +31,7 @@ export interface ITestCase<H> extends ITestCaseData {
 export async function testCaseFromResource(testCaseHandlers: {[uri: string]: ITestCaseHandler<ITestCase<any>>},
                                            options: IFetchOptions, resource: Resource): Promise<ITestCase<any>> {
   const baseTestCase: ITestCaseData = {
-    approval: resource.property.approval ? resource.property.approval.value : null,
+    approval: resource.property.approval ? resource.property.approval.value : (resource.property.rdftApproval ? resource.property.rdftApproval.value : null),
     approvedBy: resource.property.approvedBy ? resource.property.approvedBy.value : null,
     comment: resource.property.comment ? resource.property.comment.value : null,
     name: resource.property.name ? resource.property.name.value : null,

lib/testcase/TestCaseHandlers.ts

@@ -44,7 +44,7 @@ module.exports = {
 
   // RDF/XML test suite
   'http://www.w3.org/ns/rdftest#TestXMLEval':
-    new TestCaseEvalHandler(),
+    new TestCaseEvalHandler({ normalizeUrl: true }),
   'http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax':
     new TestCaseSyntaxHandler(false),
 
@@ -103,6 +103,15 @@ module.exports = {
     new TestCaseEvalHandler(),
   'http://www.w3.org/ns/rdftest#TestMicrodataNegativeSyntax':
     new TestCaseSyntaxHandler(true), // Microdata-RDF test suite never expect errors, just empty documents
+
+  // N3 test suite
+  'https://w3c.github.io/N3/tests/test.n3#TestN3PositiveSyntax':
+    new TestCaseSyntaxHandler(true),
+  'https://w3c.github.io/N3/tests/test.n3#TestN3NegativeSyntax':
+    new TestCaseSyntaxHandler(false),
+  'https://w3c.github.io/N3/tests/test.n3#TestN3Eval':
+    new TestCaseEvalHandler(),
+
 };
 // tslint:enable:object-literal-sort-keys
 // tslint:enable:max-line-length

lib/testcase/rdfsyntax/TestCaseEval.ts

@@ -17,6 +17,12 @@ const stringifyStream = require('stream-to-string');
  * Test case handler for testing if two RDF serialization are isomorphic.
  */
 export class TestCaseEvalHandler implements ITestCaseHandler<TestCaseEval> {
+  private shouldNormalizeUrl: boolean;
+
+  constructor(options?: { normalizeUrl?: boolean }) {
+    this.shouldNormalizeUrl = options?.normalizeUrl === true;
+  }
+
   public async resourceToTestCase(resource: Resource, testCaseData: ITestCaseData,
                                   options?: IFetchOptions): Promise<TestCaseEval> {
     if (!resource.property.action) {
@@ -28,14 +34,13 @@ export class TestCaseEvalHandler implements ITestCaseHandler<TestCaseEval> {
     return new TestCaseEval(testCaseData,
       await stringifyStream((await Util.fetchCached(resource.property.action.value, options)).body),
       await arrayifyStream(<any> (await Util.fetchRdf(resource.property.result.value,
-        {...options, normalizeUrl: true}))[1]),
+        {...options, normalizeUrl: this.shouldNormalizeUrl }))[1]),
       this.normalizeUrl(resource.property.action.value));
   }
 
   protected normalizeUrl(url: string) {
-    return Util.normalizeBaseUrl(url);
+    return this.shouldNormalizeUrl ? Util.normalizeBaseUrl(url) : url;
   }
-
 }
 
 export class TestCaseEval implements ITestCaseRdfSyntax {

test/TestSuiteRunner-test.ts

@@ -22,6 +22,20 @@ const mockTest4 = {
   test: () => Promise.resolve({ duration: 1337 }),
   uri: 'http://ex.org/test4',
 };
+const mockTest5 = {
+  comment: 'Test5 comment',
+  approval: 'http://www.w3.org/ns/rdftest#Rejected',
+  name: 'Test5',
+  test: () => Promise.resolve(),
+  uri: 'http://ex.org/test5',
+};
+const mockTest6 = {
+  comment: 'Test6 comment',
+  approval: 'http://www.w3.org/ns/rdftest#Approved',
+  name: 'Test6',
+  test: () => Promise.resolve(),
+  uri: 'http://ex.org/test6',
+};
 
 const timeOutMockTest1 = {
   name: 'Timeout1',
@@ -49,6 +63,8 @@ jest.mock('../lib/ManifestLoader', () => ({
               mockTest1,
               mockTest2,
               mockTest3,
+              mockTest5,
+              mockTest6,
             ],
             uri: manifestUrl,
           });
@@ -168,6 +184,17 @@ describe('TestSuiteRunner', () => {
           ok: false,
           test: mockTest3,
         },
+        {
+          error: new Error('Rejected Test'),
+          ok: false,
+          skipped: true,
+          test: mockTest5,
+        },
+        {
+          ok: true,
+          test: mockTest6,
+          duration: 1000.000001,
+        },
       ]);
     });
 
@@ -269,6 +296,52 @@ describe('TestSuiteRunner', () => {
           ok: false,
           test: mockTest3,
         },
+        {
+          error: new Error('Rejected Test'),
+          ok: false,
+          skipped: true,
+          test: mockTest5,
+        },
+        {
+          ok: true,
+          test: mockTest6,
+          duration: 1000.000001,
+        },
+      ]);
+    });
+
+    it('should produce results for a valid manifest requiring explicit approval to run tests', () => {
+      const config: ITestSuiteConfig = { ...defaultConfig, explicitApproval: true };
+      return expect(runner.runManifest('valid', handler, config)).resolves.toEqual([
+        {
+          error: new Error('Test not explicitly approved'),
+          ok: false,
+          skipped: true,
+          test: mockTest1,
+        },
+        {
+          error: new Error('Test not explicitly approved'),
+          ok: false,
+          skipped: true,
+          test: mockTest2,
+        },
+        {
+          error: new Error('Test not explicitly approved'),
+          ok: false,
+          skipped: true,
+          test: mockTest3,
+        },
+        {
+          error: new Error('Rejected Test'),
+          ok: false,
+          skipped: true,
+          test: mockTest5,
+        },
+        {
+          ok: true,
+          test: mockTest6,
+          duration: 1000.000001,
+        },
       ]);
     });
 

test/testcase/ITestCase-test.ts

@@ -19,6 +19,7 @@ describe('ITestCase', () => {
   let context;
   let pType;
   let pApproval;
+  let rApproval;
   let pApprovedBy;
   let pComment;
   let pName;
@@ -32,6 +33,8 @@ describe('ITestCase', () => {
           { term: DF.namedNode('http://www.w3.org/1999/02/22-rdf-syntax-ns#type'), context });
         pApproval = new Resource(
           { term: DF.namedNode('http://www.w3.org/2001/sw/DataAccess/tests/test-dawg#approval'), context });
+        rApproval = new Resource(
+          { term: DF.namedNode('http://www.w3.org/ns/rdftest#approval'), context });
         pApprovedBy = new Resource(
           { term: DF.namedNode('http://www.w3.org/2001/sw/DataAccess/tests/test-dawg#approvedBy'), context });
         pComment = new Resource(
@@ -92,5 +95,43 @@ describe('ITestCase', () => {
         uri: 'http://example.org/test1',
       });
     });
+
+    it('should return a test case for a valid type with optional properties with dawg:approval and rdft:approval',
+    async () => {
+      const resource = new Resource({ term: DF.namedNode('http://example.org/test1'), context });
+      resource.addProperty(pType, new Resource({ term: DF.namedNode('abc'), context}));
+      resource.addProperty(pApproval, new Resource({ term: DF.literal('APPROVAL'), context}));
+      resource.addProperty(rApproval, new Resource({ term: DF.literal('R_APPROVAL'), context}));
+      resource.addProperty(pApprovedBy, new Resource({ term: DF.literal('APPROVED_BY'), context}));
+      resource.addProperty(pComment, new Resource({ term: DF.literal('COMMENT'), context}));
+      resource.addProperty(pName, new Resource({ term: DF.literal('NAME'), context}));
+      return expect(await testCaseFromResource(handlers, null, resource)).toEqual({
+        approval: 'APPROVAL',
+        approvedBy: 'APPROVED_BY',
+        comment: 'COMMENT',
+        name: 'NAME',
+        test: true,
+        types: [ 'abc' ],
+        uri: 'http://example.org/test1',
+      });
+    });
+
+    it('should return a test case for a valid type with optional properties with rdft:apporval', async () => {
+      const resource = new Resource({ term: DF.namedNode('http://example.org/test1'), context });
+      resource.addProperty(pType, new Resource({ term: DF.namedNode('abc'), context}));
+      resource.addProperty(rApproval, new Resource({ term: DF.literal('R_APPROVAL'), context}));
+      resource.addProperty(pApprovedBy, new Resource({ term: DF.literal('APPROVED_BY'), context}));
+      resource.addProperty(pComment, new Resource({ term: DF.literal('COMMENT'), context}));
+      resource.addProperty(pName, new Resource({ term: DF.literal('NAME'), context}));
+      return expect(await testCaseFromResource(handlers, null, resource)).toEqual({
+        approval: 'R_APPROVAL',
+        approvedBy: 'APPROVED_BY',
+        comment: 'COMMENT',
+        name: 'NAME',
+        test: true,
+        types: [ 'abc' ],
+        uri: 'http://example.org/test1',
+      });
+    });
   });
 });