From 5543a898f90325d38bf883d8f26a2ea9b17362e9 Mon Sep 17 00:00:00 2001 From: Kunihiko Toumura Date: Wed, 3 Mar 2021 14:32:11 +0900 Subject: [PATCH 1/4] improve CoAP and IPv6 support --- lib/webofthings/index.js | 4 +- lib/webofthings/wotutils.js | 9 +- templates/webofthings/node.js.mustache | 155 +++++++++++++------- templates/webofthings/package.json.mustache | 12 +- 4 files changed, 116 insertions(+), 64 deletions(-) diff --git a/lib/webofthings/index.js b/lib/webofthings/index.js index c6a1c92..66e79a2 100644 --- a/lib/webofthings/index.js +++ b/lib/webofthings/index.js @@ -4,6 +4,7 @@ const fs = require('fs'); const path = require('path'); const mustache = require('mustache'); const obfuscator = require('javascript-obfuscator'); +const axios = require('axios').default; const wotutils = require('./wotutils'); @@ -13,7 +14,8 @@ async function getSpec(src) { let spec; if (typeof src === "string") { if (/^https?:/.test(src)) { - spec = JSON.parse(util.skipBom(await axios.get(src))); + const response = await axios.get(src); + spec = response.data; } else { spec = JSON.parse(util.skipBom(await fs.promises.readFile(src))); } diff --git a/lib/webofthings/wotutils.js b/lib/webofthings/wotutils.js index 8b19bd7..0e95a4c 100644 --- a/lib/webofthings/wotutils.js +++ b/lib/webofthings/wotutils.js @@ -73,9 +73,12 @@ function normalizeTd(td) { function formconv(intr, f, affordance) { if (f.hasOwnProperty("href")) { - f.href = url.resolve(baseUrl, f.href) - .replace(/%7B/gi,'{') - .replace(/%7D/gi,'}'); + if (td.base) { + f.href = new URL(f.href, td.base).toString() + } else { + f.href = new URL(f.href).toString() + } + f.href = f.href.replace(/%7B/gi,'{').replace(/%7D/gi,'}'); } if (f.hasOwnProperty("security") && typeof f.security === 'string') { f.security = [f.security]; diff --git a/templates/webofthings/node.js.mustache b/templates/webofthings/node.js.mustache index fd639d6..88f89a2 100644 --- a/templates/webofthings/node.js.mustache +++ b/templates/webofthings/node.js.mustache @@ -6,6 +6,7 @@ module.exports = function (RED) { const WebSocket = require('ws'); const urltemplate = require('url-template'); const Ajv = require('ajv'); + const nodecoap = require('coap'); function extractTemplate(href, context={}) { return urltemplate.parse(href).expand(context); @@ -68,14 +69,33 @@ module.exports = function (RED) { } } + function createCoapReqOpts(resource, method, observe=false) { + let urlobj = new URL(resource); + let hostname = urlobj.hostname; + if (hostname.startsWith('[') && hostname.endsWith(']')) { + // remove square brackets from IPv6 address + hostname = hostname.slice(1,-1); + } + let query = urlobj.search; + if (query.startsWith('?')) { + query = query.slice(1); + } + return { + hostname: hostname, + port: urlobj.port, + method: method, + pathname: urlobj.pathname, + query: query, + observe: observe + }; + } + function bindingCoap(node, send, done, form, options={}) { // options.psk - const coap = require("node-coap-client").CoapClient; node.trace("bindingCoap called"); const msg = options.msg || {}; const resource = extractTemplate(form.href, options.urivars); let payload = null; let method = null; - if (options.interaction === "property-read") { method = form.hasOwnProperty("cov:methodName") ? coapMethodCodeToName(form['cov:methodName']) : 'get'; @@ -89,78 +109,105 @@ module.exports = function (RED) { payload = options.reqbody; } - node.trace(`CoAP request: resource=${resource}, method=${method}, payload=${payload}`); - coap.request(resource, method, payload) - .then(response => { // code, format, payload - node.trace(`CoAP response: code=${response.code.toString()}, format=${response.format}, payload=${response.payload}`); - if (response.format === 50) { // application/json; rfc7252 section 12.3, table 9 + const coapreqopt = createCoapReqOpts(resource, method, false); + + node.trace(`CoAP request: reqopt=${JSON.stringify(coapreqopt)},payload=${payload}`); + let outmsg = nodecoap.request(coapreqopt); + if (payload) { + outmsg.write(payload); + } + outmsg.on('response', response => { + const cf = response.options.find(e=>e.name === 'Content-Format'); + if (cf && cf.value === 'application/json') { try { - msg.payload = JSON.parse(response.payload); + msg.payload = JSON.parse(response.payload.toString()); + if (options.outschema) { + const ajv = new Ajv({allErrors: true}); + if (!ajv.validate(options.outschema, msg.payload)) { + node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); + } + } } catch (e) { msg.payload = response.payload; } + } else if (cf && cf.value === 'text/plain') { + msg.payload = response.payload.toString(); } else { msg.payload = response.payload; } - if (options.outschema) { - const ajv = new Ajv({allErrors: true}); - if (!ajv.validate(options.outschema, msg.payload)) { - node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); - } - } send(msg); - if (done) { - done(); - } - }) - .catch(err => { - node.log(`Error: ${err.toString()}`); - msg.payload = `${err.toString()}: ${resource}`; + response.on('end', () => { if (done) { done(); }}); + }); + const errorHandler = (err) => { + node.warn('CoAP request error: ${err.message}'); + delete msg.payload; + msg.error = err; send(msg); - if (done) { - done(); - } - }); + if (done) { done() }; + }; + outmsg.on('timeout', errorHandler); + outmsg.on('error', errorHandler); + outmsg.end(); } function bindingCoapObserve(node, form, options={}) { - const coap = require("node-coap-client").CoapClient; node.status({fill:"yellow",shape:"dot",text:"CoAP try to observe ..."}); const resource = extractTemplate(form.href,options.urivars); const method = form.hasOwnProperty("cov:methodName") ? coapMethodCodeToName(form['cov:methodName']) : 'get'; const payload = options.reqbody; - const callback = response => { // code, format, payload - const msg = {}; - node.trace(`CoAP observe: code=${response.code.toString()}, format=${response.format}, payload=${response.payload}`); - if (response.format === 50) { // application/json; rfc7252 section 12.3, table 9 - try { - msg.payload = JSON.parse(response.payload); - } catch (e) { - msg.payload = response.payload; - } - } else { - msg.payload = response.payload; - } - if (options.outschema) { - const ajv = new Ajv({allErrors: true}); - if (!ajv.validate(options.outschema, msg.payload)) { - node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); + let observingStream; + + const coapreqopt = createCoapReqOpts(resource, method, true); + node.trace(`CoAP observe request: reqopt=${JSON.stringify(coapreqopt)},payload=${payload}`); + + let outmsg = nodecoap.request(coapreqopt); + if (payload) { + outmsg.write(payload); + } + outmsg.on('response', response => { + observingStream = response; + const cf = response.options.find(e=>e.name === 'Content-Format'); + response.on('end', () => {}); + response.on('data', chunk => { + const msg = {}; + if (cf && cf.value === 'application/json') { + try { + msg.payload = JSON.parse(chunk.toString()); + if (options.outschema) { + const ajv = new Ajv({allErrors: true}); + if (!ajv.validate(options.outschema, msg.payload)) { + node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); + } + } + } catch (e) { + msg.payload = chunk; + } + } else if (cf && cf.value === 'text/plain') { + msg.payload = chunk.toString(); + } else { + msg.payload = chunk; } - } - node.send(msg); - }; - coap.observe(resource,method,callback,payload,options) - .then(() => { - node.status({fill:"green",shape:"dot",text:"CoAP Observing"}); - }) - .catch(err => { - node.status({fill:"red",shape:"dot",text:`CoAP Error: ${err}`}); - node.log(`Error: ${err.toString()}`); + node.send(msg); + }); }); + const errorHandler = (err) => { + node.warn('CoAP request error: ${err.message}'); + node.status({fill:'red',shape:'dot',text:`CoAP Error: ${err}`}); + delete msg.payload; + msg.error = err; + node.send(msg); + }; + outmsg.on('timeout', errorHandler); + outmsg.on('error', errorHandler); + outmsg.end(); + node.status({fill:'green',shape:'dot',text:'CoAP Observing'}); node.on('close', () => { node.trace('Close node'); - coap.stopObserving(resource); + if (observingStream) { + observingStream.close(); + observingStream = null; + }; node.status({}); }); } @@ -438,7 +485,7 @@ module.exports = function (RED) { if (form.href.match(/^https?:/)) { bindingHttp(node, send, done, form, {interaction:"property-read", auth, msg, urivars, outschema: prop}); } else if (form.href.match(/^coaps?:/)) { - bindingCoap(node, send, done, form, send, done, {interaction:"property-read", auth, msg, urivars, outschema: prop}); + bindingCoap(node, send, done, form, {interaction:"property-read", auth, msg, urivars, outschema: prop}); } }); } else if (node.proptype === "write") { diff --git a/templates/webofthings/package.json.mustache b/templates/webofthings/package.json.mustache index e2149f8..9fad4ad 100644 --- a/templates/webofthings/package.json.mustache +++ b/templates/webofthings/package.json.mustache @@ -17,16 +17,16 @@ {{/keywords}} ], "dependencies": { - "https-proxy-agent": "^3.0.1", + "https-proxy-agent": "^5.0.0", "request": "^2.88.2", - "ws": "^7.2.0", + "ws": "^7.4.3", "url-template": "^2.0.8", - "ajv": "^6.10.2", - "node-coap-client": "^1.0.2" + "ajv": "^7.1.1", + "coap": "^0.24.0" }, "devDependencies": { - "node-red": "^1.0.4", - "node-red-node-test-helper": "^0.2.3" + "node-red": "^1.2.9", + "node-red-node-test-helper": "^0.2.7" }, "license": "{{&licenseName}}", "wot": { From 6b492a0ba082c6423f434a194e39af7bde07c2e4 Mon Sep 17 00:00:00 2001 From: Kunihiko Toumura Date: Wed, 3 Mar 2021 20:07:18 +0900 Subject: [PATCH 2/4] use non-strict mode in Ajv --- templates/webofthings/node.js.mustache | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/templates/webofthings/node.js.mustache b/templates/webofthings/node.js.mustache index 88f89a2..f444a27 100644 --- a/templates/webofthings/node.js.mustache +++ b/templates/webofthings/node.js.mustache @@ -5,7 +5,7 @@ module.exports = function (RED) { const HttpsProxyAgent = require('https-proxy-agent'); const WebSocket = require('ws'); const urltemplate = require('url-template'); - const Ajv = require('ajv'); + const Ajv = require('ajv').default; const nodecoap = require('coap'); function extractTemplate(href, context={}) { @@ -122,7 +122,7 @@ module.exports = function (RED) { try { msg.payload = JSON.parse(response.payload.toString()); if (options.outschema) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -175,7 +175,7 @@ module.exports = function (RED) { try { msg.payload = JSON.parse(chunk.toString()); if (options.outschema) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -262,7 +262,7 @@ module.exports = function (RED) { msg.payload = data; } if (options.outschema) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -329,7 +329,7 @@ module.exports = function (RED) { } // TODO: validation of return value if (options.outschema) { - const ajv = new Ajv(); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -417,7 +417,7 @@ module.exports = function (RED) { } // TODO: validation of return value if (options.outschema) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -477,7 +477,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = prop.hasOwnProperty("uriVariables") ? msg.payload : {}; if (prop.uriVariables) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(prop.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -494,7 +494,7 @@ module.exports = function (RED) { const prop = normTd.properties[node.propname]; const form = prop.forms[node.formindex];// formSelection("property-write", prop.forms); const auth = makeauth(normTd, form, username, password, token); - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(prop, msg.payload)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -511,7 +511,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = prop.hasOwnProperty("uriVariables") ? msg.payload : {}; if (prop.uriVariables) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(prop.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -532,13 +532,13 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = act.hasOwnProperty("uriVariables") ? msg.payload : {}; if (act.uriVariables) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(act.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } } if (act.input) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(act.input, msg.payload)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -555,7 +555,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = ev.hasOwnProperty("uriVariables") ? msg.payload : {}; if (ev.uriVariables) { - const ajv = new Ajv({allErrors: true}); + const ajv = new Ajv({allErrors: true, strict: false}); if (!ajv.validate(ev.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } From 040926b5e7724edcf9e4ef64209677921226321e Mon Sep 17 00:00:00 2001 From: Kunihiko Toumura Date: Wed, 3 Mar 2021 20:24:54 +0900 Subject: [PATCH 3/4] suppress format validation error --- templates/webofthings/node.js.mustache | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/templates/webofthings/node.js.mustache b/templates/webofthings/node.js.mustache index f444a27..4ecf110 100644 --- a/templates/webofthings/node.js.mustache +++ b/templates/webofthings/node.js.mustache @@ -122,7 +122,7 @@ module.exports = function (RED) { try { msg.payload = JSON.parse(response.payload.toString()); if (options.outschema) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -175,7 +175,7 @@ module.exports = function (RED) { try { msg.payload = JSON.parse(chunk.toString()); if (options.outschema) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -262,7 +262,7 @@ module.exports = function (RED) { msg.payload = data; } if (options.outschema) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -329,7 +329,7 @@ module.exports = function (RED) { } // TODO: validation of return value if (options.outschema) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -417,7 +417,7 @@ module.exports = function (RED) { } // TODO: validation of return value if (options.outschema) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(options.outschema, msg.payload)) { node.warn(`output schema validation error: ${ajv.errorsText()}`, msg); } @@ -477,7 +477,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = prop.hasOwnProperty("uriVariables") ? msg.payload : {}; if (prop.uriVariables) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(prop.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -494,7 +494,7 @@ module.exports = function (RED) { const prop = normTd.properties[node.propname]; const form = prop.forms[node.formindex];// formSelection("property-write", prop.forms); const auth = makeauth(normTd, form, username, password, token); - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(prop, msg.payload)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -511,7 +511,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = prop.hasOwnProperty("uriVariables") ? msg.payload : {}; if (prop.uriVariables) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(prop.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -532,13 +532,13 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = act.hasOwnProperty("uriVariables") ? msg.payload : {}; if (act.uriVariables) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(act.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } } if (act.input) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(act.input, msg.payload)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } @@ -555,7 +555,7 @@ module.exports = function (RED) { const auth = makeauth(normTd, form, username, password, token); const urivars = ev.hasOwnProperty("uriVariables") ? msg.payload : {}; if (ev.uriVariables) { - const ajv = new Ajv({allErrors: true, strict: false}); + const ajv = new Ajv({allErrors: true, strict: false, validateFormats: false}); if (!ajv.validate(ev.uriVariables, urivars)) { node.warn(`input schema validation error: ${ajv.errorsText()}`, msg); } From 045cf7197c5867f1fef18a546840c5cd9517ebba Mon Sep 17 00:00:00 2001 From: Kunihiko Toumura Date: Wed, 3 Mar 2021 20:48:52 +0900 Subject: [PATCH 4/4] fix typo --- templates/webofthings/node.js.mustache | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/webofthings/node.js.mustache b/templates/webofthings/node.js.mustache index 4ecf110..bab03d2 100644 --- a/templates/webofthings/node.js.mustache +++ b/templates/webofthings/node.js.mustache @@ -139,7 +139,7 @@ module.exports = function (RED) { response.on('end', () => { if (done) { done(); }}); }); const errorHandler = (err) => { - node.warn('CoAP request error: ${err.message}'); + node.warn(`CoAP request error: ${err.message}`); delete msg.payload; msg.error = err; send(msg); @@ -192,7 +192,7 @@ module.exports = function (RED) { }); }); const errorHandler = (err) => { - node.warn('CoAP request error: ${err.message}'); + node.warn(`CoAP request error: ${err.message}`); node.status({fill:'red',shape:'dot',text:`CoAP Error: ${err}`}); delete msg.payload; msg.error = err;