From 826d2a11a260b78f4b81099ad66d11758ad93fe6 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Mon, 14 Aug 2023 15:07:27 +0200 Subject: [PATCH 01/42] add host to configuration --- src/main.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.js b/src/main.js index 66ac779..32edd05 100644 --- a/src/main.js +++ b/src/main.js @@ -53,6 +53,12 @@ function ymlContentToConfig(ymlContent) { throw new Error("Error parsing YAML: Google sheet id should be specified"); } + if (configJson.host) { + config.host = configJson.host + } else { + throw new Error("Error parsing YAML: host value should be specified") + } + config.interval = configJson.sheet.interval ? configJson.sheet.interval : 5000; } From 54d5dd935858177c4c724f7f5cfc6a8285841958 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Mon, 14 Aug 2023 15:47:16 +0200 Subject: [PATCH 02/42] util function object comparison. --- src/util.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/util.js diff --git a/src/util.js b/src/util.js new file mode 100644 index 0000000..dabb617 --- /dev/null +++ b/src/util.js @@ -0,0 +1,16 @@ +export function shallowEqual(obj1, obj2) { + const keys1 = Object.keys(obj1); + const keys2 = Object.keys(obj2); + + if (keys1.length !== keys2.length) { + return false; + } + + for (let key of keys1) { + if (!obj2.hasOwnProperty(key) || obj1[key] !== obj2[key]) { + return false; + } + } + + return true; +} \ No newline at end of file From 7c89b8ca706449c68623d92bc5aa4037e28c74ed Mon Sep 17 00:00:00 2001 From: sevrijss Date: Mon, 14 Aug 2023 15:47:50 +0200 Subject: [PATCH 03/42] function to get websocket registration endpoint --- src/solid.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/solid.js b/src/solid.js index 46591e0..6b5fae7 100644 --- a/src/solid.js +++ b/src/solid.js @@ -37,6 +37,21 @@ export async function queryResource(config) { }); } +export async function getNotificationChannelTypes(url){ + const myEngine = new QueryEngine(); + const result = await (await myEngine.queryBindings(` + SELECT DISTINCT ?channel WHERE { + ?s a . + ?s ?channel . + ?channel + }`, + { + sources: [url], + } + )).toArray(); + return result.map(binding => binding.get("channel").value) +} + /** * Convert the "fields" configuration data into a SPARQL query * @param {Object} config - Configuration object containing the necessary information build the SPARQL query (required and optional fields). From b958eef70a44cdcb7ccde7dafc47fd341c6879e1 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Mon, 14 Aug 2023 15:48:08 +0200 Subject: [PATCH 04/42] add websocket to package --- package-lock.json | 4 ++-- package.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index ac1b3a9..853b82e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,8 @@ "googleapis": "^122.0.0", "http-server": "^14.1.1", "js-yaml": "^4.1.0", - "n3": "^1.17.0" + "n3": "^1.17.0", + "ws": "^8.13.0" }, "devDependencies": { "@solid/community-server": "^6.0.1", @@ -8362,7 +8363,6 @@ "version": "8.13.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.13.0.tgz", "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", - "dev": true, "engines": { "node": ">=10.0.0" }, diff --git a/package.json b/package.json index fa309a1..b7cdb50 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,8 @@ "googleapis": "^122.0.0", "http-server": "^14.1.1", "js-yaml": "^4.1.0", - "n3": "^1.17.0" + "n3": "^1.17.0", + "ws": "^8.13.0" }, "type": "module" } From 8feefb15817e0ab66c2defb4055dcb7c7b09ceb7 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:06:34 +0200 Subject: [PATCH 05/42] array compare for objects --- src/util.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/util.js b/src/util.js index dabb617..c59e113 100644 --- a/src/util.js +++ b/src/util.js @@ -13,4 +13,19 @@ export function shallowEqual(obj1, obj2) { } return true; +} + +export function compareArrays(first, second) { + let equal = true + first.forEach((element) => { + if (second.filter((entry) => shallowEqual(entry, element)).length === 0){ + equal = false; + } + }) + second.forEach((element) => { + if(first.filter((entry) => shallowEqual(entry, element)).length === 0) { + equal = false; + } + }) + return equal; } \ No newline at end of file From f8d917578ac9cac0c0f9a0a37da2d8a7e08ddae9 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:07:02 +0200 Subject: [PATCH 06/42] noCache option when querying resources --- src/solid.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/solid.js b/src/solid.js index 6b5fae7..59bb144 100644 --- a/src/solid.js +++ b/src/solid.js @@ -4,11 +4,15 @@ import {Writer} from "n3"; /** * Query the data from the Solid pod/resource(s) using the configuration * @param {Object} config - Configuration object containing the necessary information to query and process the retrieved data. + * @param {boolean} noCache - clear http cache to get most recent document. * @return {Promise<{array, array}>} Map objects containing the retrieved data * and all possible keys representing the properties contained in the maps. */ -export async function queryResource(config) { +export async function queryResource(config, noCache = false) { const myEngine = new QueryEngine(); + if (noCache){ + await myEngine.invalidateHttpCache(); + } const results = []; const keys = new Set(); const query = config.query !== undefined ? config.query : configToSPARQLQuery(config); From 688b95622d91461fe31637d911064dcd6bfec95c Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:23:19 +0200 Subject: [PATCH 07/42] add sheet name to config to monitor entire sheet --- src/main.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main.js b/src/main.js index 32edd05..d1bfbbc 100644 --- a/src/main.js +++ b/src/main.js @@ -53,6 +53,12 @@ function ymlContentToConfig(ymlContent) { throw new Error("Error parsing YAML: Google sheet id should be specified"); } + if (configJson.sheet.name) { + config.sheetName = configJson.sheet.name + } else { + throw new Error("Error parsing YAML: Google sheet name should be specified") + } + if (configJson.host) { config.host = configJson.host } else { From 43602a9b779fde009c06f60517f61d51e1922086 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:25:20 +0200 Subject: [PATCH 08/42] add pagename in google code --- src/google.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/google.js b/src/google.js index 708df57..39ba5f5 100644 --- a/src/google.js +++ b/src/google.js @@ -57,11 +57,12 @@ export async function writeToSheet(array, sheetId) { /** * Pull data from the sheet and check if there are any changes with the previously pulled data. * @param {String} sheetId - ID of the Google sheet from which the data should be pulled and checked. + * @param {String} pageName - Name of the excel page to check * @return {Promise<{Boolean, Array}>} - 2D-array containing the latest data from the sheet * and a boolean indicating a possible change. */ -export async function checkSheetForChanges(sheetId) { - const rows = await getFromSheet(sheetId); +export async function checkSheetForChanges(sheetId, pageName) { + const rows = await getFromSheet(sheetId, pageName); const hasChanged = previousRows !== undefined && !areArraysEqual(rows, previousRows); previousRows = rows; return { @@ -73,12 +74,13 @@ export async function checkSheetForChanges(sheetId) { /** * Get the data from the sheet in the initial range. * @param {String} sheetId - ID from the sheet from which the data should be pulled. + * @param {String} pageName - Name of the excel page to check * @return {Array} 2D-array containing the data from the sheet. */ -async function getFromSheet(sheetId){ +async function getFromSheet(sheetId, pageName) { const response = await sheets.spreadsheets.values.get({ spreadsheetId: sheetId, - range: 'A:ZZZ' + range: pageName }); return response.data.values; From 58675deba5a8ea04b687dab4bd4e489e50e23508 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:26:15 +0200 Subject: [PATCH 09/42] give name from config + save some data for later checks --- src/main.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index d1bfbbc..66ddd66 100644 --- a/src/main.js +++ b/src/main.js @@ -9,6 +9,7 @@ let config = {}; // Array containing all quads on the sheet when the last change was detected. let previousQuads; +let previousMap; /** * Parse YAML data and store it in the configuration object. @@ -174,6 +175,7 @@ async function startFromFile(configPath, rulesPath) { await makeClient(); const rows = await writeToSheet(arrays, config.sheetid); const maps = rowsToObjects(rows); + previousMap = maps; previousQuads = await objectsToRdf({data: maps}, rml); @@ -181,10 +183,11 @@ async function startFromFile(configPath, rulesPath) { // Sheet -> Pod sync setInterval(async () => { - const {rows, hasChanged} = await checkSheetForChanges(config.sheetid); + const {rows, hasChanged} = await checkSheetForChanges(config.sheetid, config.sheetName); if (hasChanged) { console.log("Changes detected. Synchronizing..."); const maps = rowsToObjects(rows); + previousMap = maps; const quads = await objectsToRdf({data: maps}, rml); From 9638b333a41cebb4dea8972e9111b5955a97b065 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:32:03 +0200 Subject: [PATCH 10/42] websocket requestOptions util method --- src/util.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/util.js b/src/util.js index c59e113..9d61092 100644 --- a/src/util.js +++ b/src/util.js @@ -28,4 +28,24 @@ export function compareArrays(first, second) { } }) return equal; +} + +export function getWebsocketRequestOptions(source){ + let myHeaders = new Headers(); + myHeaders.append("Content-Type", "application/ld+json"); + + let raw = JSON.stringify({ + "@context": [ + "https://www.w3.org/ns/solid/notification/v1" + ], + "type": "http://www.w3.org/ns/solid/notifications#WebSocketChannel2023", + "topic": source + }); + + return { + method: 'POST', + headers: myHeaders, + body: raw, + redirect: 'follow' + }; } \ No newline at end of file From 8b3335eef9b5ce333db93209d16beb5222bec036 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:37:05 +0200 Subject: [PATCH 11/42] websocket setup working - I think --- src/main.js | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/main.js b/src/main.js index 66ddd66..796fd0f 100644 --- a/src/main.js +++ b/src/main.js @@ -1,8 +1,10 @@ import {checkSheetForChanges, makeClient, writeToSheet} from "./google.js"; import {load} from "js-yaml"; import {objectsToRdf, yarrrmlToRml} from "./rdf-generation.js"; -import {queryResource, updateResource} from "./solid.js"; +import {getNotificationChannelTypes, queryResource, updateResource} from "./solid.js"; import {readFile} from 'fs/promises' +import {WebSocket} from 'ws'; +import {compareArrays, getWebsocketRequestOptions} from "./util.js"; // Object containing information relating to the configuration of the synchronisation app. let config = {}; @@ -180,6 +182,38 @@ async function startFromFile(configPath, rulesPath) { console.log("Synchronisation cold start completed"); + // Pod -> Sheet sync + + let websocket_endpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); + + if (websocket_endpoints.length > 0 && websocket_endpoints[0].length > 0) { + // listen using websockets + let url = websocket_endpoints[0] + let requestOptions = getWebsocketRequestOptions(config.source) + + let response = await (await fetch(url, requestOptions)).json() + let endpoint = response["receiveFrom"]; + const ws = new WebSocket(endpoint); + ws.on("message", async (notification) => { + let content = JSON.parse(notification); + if (content.type === "Update") { + const {results} = await queryResource(config, true); + const arrays = mapsTo2DArray(results); + const maps = rowsToObjects(arrays); + if (!compareArrays(maps, previousMap)) { + const rows = await writeToSheet(arrays, config.sheetid); + const maps2 = rowsToObjects(rows); + previousMap = maps2; + previousQuads = await objectsToRdf({data: maps2}, rml); + } else { + console.log("got notified but the latest changes are already present"); + } + } + }) + } else { + // polling using timers + // TODO + } // Sheet -> Pod sync setInterval(async () => { From c218b7c560bd63d64ca7c0119776c4ed2d83ae92 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:44:00 +0200 Subject: [PATCH 12/42] add debug config options + timer to check changes on pod --- src/main.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main.js b/src/main.js index 796fd0f..996175b 100644 --- a/src/main.js +++ b/src/main.js @@ -68,6 +68,12 @@ function ymlContentToConfig(ymlContent) { throw new Error("Error parsing YAML: host value should be specified") } + if (configJson.debug){ + if (configJson.debug.websockets) { + config.debug_noWebSockets = configJson.debug.websockets === "false" + } + } + config.interval = configJson.sheet.interval ? configJson.sheet.interval : 5000; } @@ -186,7 +192,7 @@ async function startFromFile(configPath, rulesPath) { let websocket_endpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); - if (websocket_endpoints.length > 0 && websocket_endpoints[0].length > 0) { + if (websocket_endpoints.length > 0 && websocket_endpoints[0].length > 0 && (!config.debug_noWebSockets)) { // listen using websockets let url = websocket_endpoints[0] let requestOptions = getWebsocketRequestOptions(config.source) @@ -212,7 +218,17 @@ async function startFromFile(configPath, rulesPath) { }) } else { // polling using timers - // TODO + setInterval(async () => { + const {results} = await queryResource(config, true); + const arrays = mapsTo2DArray(results); + const maps = rowsToObjects(arrays); + if (!compareArrays(maps, previousMap)) { + const rows = await writeToSheet(arrays, config.sheetid); + const maps2 = rowsToObjects(rows); + previousMap = maps2; + previousQuads = await objectsToRdf({data: maps2}, rml); + } + }, config.interval); } // Sheet -> Pod sync From ae67c5ff43f93a0e918431e0b476bf0aaf8a809f Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 09:50:32 +0200 Subject: [PATCH 13/42] gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e8ec0b3..a2c10e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.idea/ +src/testing.js .env config.yml credentials.json From d401d0a7a75fbc51688754893da6119bf2de0fdc Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 10:33:30 +0200 Subject: [PATCH 14/42] recode comparison. Second loop removed and added length check. --- src/util.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/util.js b/src/util.js index 9d61092..66b2231 100644 --- a/src/util.js +++ b/src/util.js @@ -16,18 +16,15 @@ export function shallowEqual(obj1, obj2) { } export function compareArrays(first, second) { - let equal = true + if(first.length !== second.length) { + return false; + } first.forEach((element) => { if (second.filter((entry) => shallowEqual(entry, element)).length === 0){ - equal = false; - } - }) - second.forEach((element) => { - if(first.filter((entry) => shallowEqual(entry, element)).length === 0) { - equal = false; + return false; } }) - return equal; + return true; } export function getWebsocketRequestOptions(source){ From 5d57db25d8264901ae081e72afc56870d671fe1b Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 10:33:51 +0200 Subject: [PATCH 15/42] comments + auto format file --- src/util.js | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/util.js b/src/util.js index 66b2231..079ed77 100644 --- a/src/util.js +++ b/src/util.js @@ -1,3 +1,9 @@ +/** + * Compares objects - checks keys and values + * @param {Object} obj1 + * @param {Object} obj2 + * @returns {boolean} Boolean that indicates if the objects are equal + */ export function shallowEqual(obj1, obj2) { const keys1 = Object.keys(obj1); const keys2 = Object.keys(obj2); @@ -15,19 +21,31 @@ export function shallowEqual(obj1, obj2) { return true; } + +/** + * Compares 2 arrays and checks if they contain the same objects (order doesn't matter) + * @param {Object[]} first + * @param {Object[]} second + * @returns {boolean} Boolean that indicates if the arrays are equal (not counting order) + */ export function compareArrays(first, second) { - if(first.length !== second.length) { + if (first.length !== second.length) { return false; } first.forEach((element) => { - if (second.filter((entry) => shallowEqual(entry, element)).length === 0){ + if (second.filter((entry) => shallowEqual(entry, element)).length === 0) { return false; } }) return true; } -export function getWebsocketRequestOptions(source){ +/** + * Generates RequestOptions to establish a websocket connection for the source parameter resource + * @param {string} source - resource to which a websocket should be provided. + * @return {{redirect: string, headers: Headers, method: string, body: string}} RequestOptions to request a websocket connection to source parameter + */ +export function getWebsocketRequestOptions(source) { let myHeaders = new Headers(); myHeaders.append("Content-Type", "application/ld+json"); From 1c7996c891f8eb49412cb04d8003c2dd8e360455 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Wed, 16 Aug 2023 14:01:32 +0200 Subject: [PATCH 16/42] added entry in TESTS.md --- TESTS.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/TESTS.md b/TESTS.md index 8ee6a79..fb08403 100644 --- a/TESTS.md +++ b/TESTS.md @@ -27,4 +27,26 @@ Set the `id` section to the id of an existing Google Sheet. 2. Wait at least the configured amount of milliseconds as configured under `interval` in the configuration file (default 5000). ### Postconditions -- The changes are correctly converted and written back to the resource `http://localhost:3000/example/software`. \ No newline at end of file +- The changes are correctly converted and written back to the resource `http://localhost:3000/example/software`. + +## Test if changes on the Pod are synced back to the Google Sheet + +### Preconditions +- Follow and execute all steps in the "cold start" test above. + +### Steps +Using postman or another software to make requests, send +``` +@prefix solid: . +@prefix software: . +@prefix schema: . +_:rename a solid:InsertDeletePatch; +solid:inserts { software:test schema:name "test"; schema:description "abracadabra". }. +``` +The resource endpoint ([http://localhost:3000/example/softwate](http://localhost:3000/example/softwate)). + +When using websockets, the change should be almost immediately be shown, +otherwise wait at least the configured amount of milliseconds as configured under interval in the configuration file (default 5000). + +### Postconditions +- The changes are correctly converted and visible in the google sheet \ No newline at end of file From 83a5d0d4315d25afeb74a192cf678957343ca99e Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 08:43:01 +0200 Subject: [PATCH 17/42] remove .idea from gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a2c10e5..6e0ddc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.idea/ src/testing.js .env config.yml From c345d437934e449a454288408fc74d894a0e48da Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 08:45:15 +0200 Subject: [PATCH 18/42] rename page to sheet --- src/google.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/google.js b/src/google.js index 39ba5f5..6950d6b 100644 --- a/src/google.js +++ b/src/google.js @@ -57,12 +57,12 @@ export async function writeToSheet(array, sheetId) { /** * Pull data from the sheet and check if there are any changes with the previously pulled data. * @param {String} sheetId - ID of the Google sheet from which the data should be pulled and checked. - * @param {String} pageName - Name of the excel page to check + * @param {String} sheetName - Name of the excel page to check * @return {Promise<{Boolean, Array}>} - 2D-array containing the latest data from the sheet * and a boolean indicating a possible change. */ -export async function checkSheetForChanges(sheetId, pageName) { - const rows = await getFromSheet(sheetId, pageName); +export async function checkSheetForChanges(sheetId, sheetName) { + const rows = await getFromSheet(sheetId, sheetName); const hasChanged = previousRows !== undefined && !areArraysEqual(rows, previousRows); previousRows = rows; return { @@ -74,13 +74,13 @@ export async function checkSheetForChanges(sheetId, pageName) { /** * Get the data from the sheet in the initial range. * @param {String} sheetId - ID from the sheet from which the data should be pulled. - * @param {String} pageName - Name of the excel page to check + * @param {String} sheetName - Name of the excel page to check * @return {Array} 2D-array containing the data from the sheet. */ -async function getFromSheet(sheetId, pageName) { +async function getFromSheet(sheetId, sheetName) { const response = await sheets.spreadsheets.values.get({ spreadsheetId: sheetId, - range: pageName + range: sheetName }); return response.data.values; From c547d5c37963644bdfe5e8568d83aa0b470037e7 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 08:46:04 +0200 Subject: [PATCH 19/42] variable name to camel case --- src/main.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.js b/src/main.js index 996175b..1cf41d7 100644 --- a/src/main.js +++ b/src/main.js @@ -190,11 +190,11 @@ async function startFromFile(configPath, rulesPath) { console.log("Synchronisation cold start completed"); // Pod -> Sheet sync - let websocket_endpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); + let websocketEndpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); - if (websocket_endpoints.length > 0 && websocket_endpoints[0].length > 0 && (!config.debug_noWebSockets)) { + if (websocketEndpoints.length > 0 && websocketEndpoints[0].length > 0 && (!config.debug_noWebSockets)) { // listen using websockets - let url = websocket_endpoints[0] + let url = websocketEndpoints[0] let requestOptions = getWebsocketRequestOptions(config.source) let response = await (await fetch(url, requestOptions)).json() From 84e1f3b0aee1100c234a9fd122519bd1cfa52e30 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:03:24 +0200 Subject: [PATCH 20/42] better compareArrays function --- src/util.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/util.js b/src/util.js index 079ed77..00f42df 100644 --- a/src/util.js +++ b/src/util.js @@ -26,14 +26,15 @@ export function shallowEqual(obj1, obj2) { * Compares 2 arrays and checks if they contain the same objects (order doesn't matter) * @param {Object[]} first * @param {Object[]} second + * @param {function(Object, Object): boolean } comparator * @returns {boolean} Boolean that indicates if the arrays are equal (not counting order) */ -export function compareArrays(first, second) { +export function compareArrays(first, second, comparator = shallowEqual) { if (first.length !== second.length) { return false; } first.forEach((element) => { - if (second.filter((entry) => shallowEqual(entry, element)).length === 0) { + if (second.filter((entry) => comparator(entry, element)).length === 0) { return false; } }) @@ -50,17 +51,12 @@ export function getWebsocketRequestOptions(source) { myHeaders.append("Content-Type", "application/ld+json"); let raw = JSON.stringify({ - "@context": [ - "https://www.w3.org/ns/solid/notification/v1" - ], + "@context": ["https://www.w3.org/ns/solid/notification/v1"], "type": "http://www.w3.org/ns/solid/notifications#WebSocketChannel2023", "topic": source }); return { - method: 'POST', - headers: myHeaders, - body: raw, - redirect: 'follow' + method: 'POST', headers: myHeaders, body: raw, redirect: 'follow' }; } \ No newline at end of file From 4943ee4fee0780b2477c1fc93636f5f6b105a7df Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:04:25 +0200 Subject: [PATCH 21/42] more docs --- src/util.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util.js b/src/util.js index 00f42df..cfd1c0a 100644 --- a/src/util.js +++ b/src/util.js @@ -1,7 +1,7 @@ /** * Compares objects - checks keys and values - * @param {Object} obj1 - * @param {Object} obj2 + * @param {Object} obj1 - first object to compare + * @param {Object} obj2 - second object to compare * @returns {boolean} Boolean that indicates if the objects are equal */ export function shallowEqual(obj1, obj2) { @@ -24,9 +24,9 @@ export function shallowEqual(obj1, obj2) { /** * Compares 2 arrays and checks if they contain the same objects (order doesn't matter) - * @param {Object[]} first - * @param {Object[]} second - * @param {function(Object, Object): boolean } comparator + * @param {Object[]} first - first array of objects to compare + * @param {Object[]} second - second array of objects to compare + * @param {function(Object, Object): boolean } comparator - comparator function to use. Defaults to shallowEqual * @returns {boolean} Boolean that indicates if the arrays are equal (not counting order) */ export function compareArrays(first, second, comparator = shallowEqual) { From 1818f1967c0d30bb421eb3a8bb90eaef8af9df53 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:18:01 +0200 Subject: [PATCH 22/42] fix discarded return value foreach --- src/util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util.js b/src/util.js index cfd1c0a..93e02f9 100644 --- a/src/util.js +++ b/src/util.js @@ -33,11 +33,11 @@ export function compareArrays(first, second, comparator = shallowEqual) { if (first.length !== second.length) { return false; } - first.forEach((element) => { + for (const element of first) { if (second.filter((entry) => comparator(entry, element)).length === 0) { return false; } - }) + } return true; } From f7cccc13be102cb0d20bccaee931edc4e5be6a95 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:19:18 +0200 Subject: [PATCH 23/42] refactor to not use previousMap variable --- src/main.js | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main.js b/src/main.js index 1cf41d7..27039ff 100644 --- a/src/main.js +++ b/src/main.js @@ -11,7 +11,6 @@ let config = {}; // Array containing all quads on the sheet when the last change was detected. let previousQuads; -let previousMap; /** * Parse YAML data and store it in the configuration object. @@ -183,13 +182,12 @@ async function startFromFile(configPath, rulesPath) { await makeClient(); const rows = await writeToSheet(arrays, config.sheetid); const maps = rowsToObjects(rows); - previousMap = maps; previousQuads = await objectsToRdf({data: maps}, rml); console.log("Synchronisation cold start completed"); - // Pod -> Sheet sync + // Pod -> Sheet sync let websocketEndpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); if (websocketEndpoints.length > 0 && websocketEndpoints[0].length > 0 && (!config.debug_noWebSockets)) { @@ -206,10 +204,10 @@ async function startFromFile(configPath, rulesPath) { const {results} = await queryResource(config, true); const arrays = mapsTo2DArray(results); const maps = rowsToObjects(arrays); - if (!compareArrays(maps, previousMap)) { + const quads = await objectsToRdf({data: maps}, rml); + if (!compareArrays(quads, previousQuads, compareQuads)) { const rows = await writeToSheet(arrays, config.sheetid); const maps2 = rowsToObjects(rows); - previousMap = maps2; previousQuads = await objectsToRdf({data: maps2}, rml); } else { console.log("got notified but the latest changes are already present"); @@ -222,10 +220,10 @@ async function startFromFile(configPath, rulesPath) { const {results} = await queryResource(config, true); const arrays = mapsTo2DArray(results); const maps = rowsToObjects(arrays); - if (!compareArrays(maps, previousMap)) { + const quads = await objectsToRdf({data: maps}, rml); + if (!compareArrays(quads, previousQuads, compareQuads)) { const rows = await writeToSheet(arrays, config.sheetid); const maps2 = rowsToObjects(rows); - previousMap = maps2; previousQuads = await objectsToRdf({data: maps2}, rml); } }, config.interval); @@ -237,7 +235,6 @@ async function startFromFile(configPath, rulesPath) { if (hasChanged) { console.log("Changes detected. Synchronizing..."); const maps = rowsToObjects(rows); - previousMap = maps; const quads = await objectsToRdf({data: maps}, rml); From 705fa353e0631c39805e31097f5738eaeac0732d Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:23:44 +0200 Subject: [PATCH 24/42] docs for getNotificationChannelTypes --- src/solid.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/solid.js b/src/solid.js index 59bb144..776929b 100644 --- a/src/solid.js +++ b/src/solid.js @@ -41,6 +41,11 @@ export async function queryResource(config, noCache = false) { }); } +/** + * Query the available websocket channels that may be listed in a given endpoint + * @param {string} url - host to query (e.g. http://localhost:3000/.well-known/solid/) + * @returns {Promise} list of available endpoints to request a websocket connection + */ export async function getNotificationChannelTypes(url){ const myEngine = new QueryEngine(); const result = await (await myEngine.queryBindings(` From 68ff9b3bf028d709f9c834b35e2f1791eebd7eb5 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:25:59 +0200 Subject: [PATCH 25/42] fix return types in docs --- src/google.js | 2 +- src/rdf-generation.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google.js b/src/google.js index 6950d6b..fc88f85 100644 --- a/src/google.js +++ b/src/google.js @@ -75,7 +75,7 @@ export async function checkSheetForChanges(sheetId, sheetName) { * Get the data from the sheet in the initial range. * @param {String} sheetId - ID from the sheet from which the data should be pulled. * @param {String} sheetName - Name of the excel page to check - * @return {Array} 2D-array containing the data from the sheet. + * @return {Promise} 2D-array containing the data from the sheet. */ async function getFromSheet(sheetId, sheetName) { const response = await sheets.spreadsheets.values.get({ diff --git a/src/rdf-generation.js b/src/rdf-generation.js index 91901ad..42498c4 100644 --- a/src/rdf-generation.js +++ b/src/rdf-generation.js @@ -31,7 +31,7 @@ export async function objectsToRdf(data, rml) { /** * Convert a String containing RDF data into quad objects. * @param {String} text - A string containing the RDF data. - * @return {[quad]} Parsed quad objects. + * @return {Promise<[quad]>} Parsed quad objects. */ async function convertRdfToQuads(text) { const parser = new Parser(); From 4cf51a90da8c8fb0c839671e4934fc6544be47c0 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:32:19 +0200 Subject: [PATCH 26/42] update readme + example configs --- README.md | 22 ++++++++++++++++------ config.fields.example.yml | 3 ++- config.query.example.yml | 3 ++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index de4ecac..ff7d656 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,6 @@ The `sheet` section of the configuration file contains settings related to a spe #### id (string) This parameter allows you to specify an id for the Google sheet that should be read and/or altered. -example: -```yaml -sheet: - id: "ABCD1234" -``` - To find the id of your Google sheet, look at the URL of the Google Sheet in the address bar of your web browser. The URL should look something like this: ``` @@ -92,6 +86,19 @@ https://docs.google.com/spreadsheets/d/DOCUMENT_ID/edit#gid=0 Here, "DOCUMENT_ID" will be a long string of characters, letters, and numbers. This is the unique identifier for the Google Sheet. +#### name (string) +This parameter allows you to specify a name for the google sheet that should be read and/or altered. \ +This is the name of the tab on the bottom left that you want to read and/or alter + +example: + +```yaml +sheet: + id: "ABCD1234" + name: "Sheet1" +``` + + ### Using Fields for Data Retrieving Instead of using a single, user defined SPARQL query as in the previous method, the user can use the `fields` option @@ -154,3 +161,6 @@ write back changes from the Google Sheet back to a single destination. ### Public read/write authorization It is required for the resource specified in the configuration file to have public read and write access, as the agent has no support for authentication. + +### No 2 applications write at the same time. +Currently, it is not handled when the sheet and the resource in the pod are updated at exactly the same time. \ No newline at end of file diff --git a/config.fields.example.yml b/config.fields.example.yml index 57647bb..dda61ef 100644 --- a/config.fields.example.yml +++ b/config.fields.example.yml @@ -8,4 +8,5 @@ fields: - logo: "" sheet: - id: "ABCD1234" \ No newline at end of file + id: "ABCD1234" + name: "Sheet1" \ No newline at end of file diff --git a/config.query.example.yml b/config.query.example.yml index f1215c0..26c4d7b 100644 --- a/config.query.example.yml +++ b/config.query.example.yml @@ -8,4 +8,5 @@ query: > } sheet: - id: "ABCD1234" \ No newline at end of file + id: "ABCD1234" + name: "Sheet1" \ No newline at end of file From a826b5f906c341b84a3919e761c2b37009233826 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:35:41 +0200 Subject: [PATCH 27/42] update readme for debug configurations --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index ff7d656..c15d481 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,16 @@ fields: - logo: "" ``` +### Debug configurations + +#### websockets +This parameter allows you to turn of websockets when you want explicit polling every X seconds. + +example: +```yaml +debug: + websockets: "false" +``` ### Full examples Full configuration examples that incorporate either the query or fields method are present in From 7fabfcd40619641f9f93c673cab8bbf579b7e732 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 09:51:43 +0200 Subject: [PATCH 28/42] fix comparison --- src/google.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/google.js b/src/google.js index fc88f85..34b5baf 100644 --- a/src/google.js +++ b/src/google.js @@ -63,6 +63,8 @@ export async function writeToSheet(array, sheetId) { */ export async function checkSheetForChanges(sheetId, sheetName) { const rows = await getFromSheet(sheetId, sheetName); + console.log(rows.slice(rows.length - 2, rows.length)); + console.log(previousRows.slice(previousRows.length - 2, previousRows.length)); const hasChanged = previousRows !== undefined && !areArraysEqual(rows, previousRows); previousRows = rows; return { @@ -117,6 +119,9 @@ function areArraysEqual(arr1, arr2) { } for (let i = 0; i < arr1.length; i++) { + if (arr1[i].length !== arr2[i].length) { + return false; + } for (let j = 0; j < arr1[i].length; j++) { if (arr1[i][j] !== arr2[i][j]) { return false; From 2423de493e147850bd851d7398a660768a06bae7 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 10:01:06 +0200 Subject: [PATCH 29/42] add host to config --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c15d481..cc32e28 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,15 @@ The synchronisation application is configured through the `config.yml` file. This parameter allows a user to specify a resource. This resource should be represented as a URI to a Solid pod from which the data will be fetched. + +#### host (string) +This parameter allows a user to specify the host of a resource. +This is required to use the websocket protocol to listen for changes on the resource. + example: ```yaml resource: "https://data.knows.idlab.ugent.be/person/office/software" +host: "http://data.knows.idlab.ugent.be" ``` #### query (string) From 2e67217a6413025759a60f4be3cc925e0ad19942 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 11:00:02 +0200 Subject: [PATCH 30/42] remove console.log --- src/google.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/google.js b/src/google.js index 34b5baf..1e1dda0 100644 --- a/src/google.js +++ b/src/google.js @@ -63,8 +63,6 @@ export async function writeToSheet(array, sheetId) { */ export async function checkSheetForChanges(sheetId, sheetName) { const rows = await getFromSheet(sheetId, sheetName); - console.log(rows.slice(rows.length - 2, rows.length)); - console.log(previousRows.slice(previousRows.length - 2, previousRows.length)); const hasChanged = previousRows !== undefined && !areArraysEqual(rows, previousRows); previousRows = rows; return { From be94f3622f386e128b9d1fefced8f22be15904c8 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:15:54 +0200 Subject: [PATCH 31/42] fix faulty url --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc32e28..382d559 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ This is required to use the websocket protocol to listen for changes on the reso example: ```yaml -resource: "https://data.knows.idlab.ugent.be/person/office/software" -host: "http://data.knows.idlab.ugent.be" +resource: "http://localhost:3000/example/software" +host: "http://localhost:3000" ``` #### query (string) From 0191662a5778770953e2180876d1a686579ab76e Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:17:31 +0200 Subject: [PATCH 32/42] capitalize + usage sync + add interval to readme + websockets --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 382d559..44915e3 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,13 @@ Here, "DOCUMENT_ID" will be a long string of characters, letters, and numbers. This is the unique identifier for the Google Sheet. #### name (string) -This parameter allows you to specify a name for the google sheet that should be read and/or altered. \ -This is the name of the tab on the bottom left that you want to read and/or alter +This parameter allows you to specify a name for the Google sheet that should be read and/or altered. \ +This is the name of the tab on the bottom left that you want to sync. + + +#### interval (int) +This parameter allows you to specify the number of milliseconds between polls. +This number will be used for polling the sheet for changes and polling the pod for changes when websockets aren't used. example: @@ -102,6 +107,7 @@ example: sheet: id: "ABCD1234" name: "Sheet1" + interval: 1000 ``` @@ -136,7 +142,8 @@ fields: ### Debug configurations #### websockets -This parameter allows you to turn of websockets when you want explicit polling every X seconds. +This parameter allows you to turn off websockets when you want explicit polling every 5 seconds. +This value can de modified using the `interval` option from the Google Sheet configuration. example: ```yaml From 465a76dffee604fa35a9edfcecc6a51ff01093ff Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:20:51 +0200 Subject: [PATCH 33/42] fix active voice --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44915e3..4f43113 100644 --- a/README.md +++ b/README.md @@ -186,4 +186,4 @@ It is required for the resource specified in the configuration file to have publ as the agent has no support for authentication. ### No 2 applications write at the same time. -Currently, it is not handled when the sheet and the resource in the pod are updated at exactly the same time. \ No newline at end of file +Currently, it is not handled when the sheet and another application try to update the resource in the pod at exactly the same time. \ No newline at end of file From 6aa75ff39fb3c0d227a8b4e4537e40446f012463 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:21:17 +0200 Subject: [PATCH 34/42] add curl request --- TESTS.md | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/TESTS.md b/TESTS.md index fb08403..dea65cf 100644 --- a/TESTS.md +++ b/TESTS.md @@ -35,18 +35,12 @@ Set the `id` section to the id of an existing Google Sheet. - Follow and execute all steps in the "cold start" test above. ### Steps -Using postman or another software to make requests, send +The pod can be updated with the following request: +```curl +curl --location --request PATCH 'http://localhost:3000/example/software' --header 'Content-Type: text/n3' --data-raw '@prefix solid: . @prefix software: . @prefix schema: . _:rename a solid:InsertDeletePatch; solid:inserts { software:test schema:name "test"; schema:description "abracadabra". }.' ``` -@prefix solid: . -@prefix software: . -@prefix schema: . -_:rename a solid:InsertDeletePatch; -solid:inserts { software:test schema:name "test"; schema:description "abracadabra". }. -``` -The resource endpoint ([http://localhost:3000/example/softwate](http://localhost:3000/example/softwate)). - When using websockets, the change should be almost immediately be shown, -otherwise wait at least the configured amount of milliseconds as configured under interval in the configuration file (default 5000). +otherwise wait at least the configured amount of milliseconds (default 5000ms). ### Postconditions - The changes are correctly converted and visible in the google sheet \ No newline at end of file From 0d3bbaacc19bb3c53c0ef51a0d42e3c6fef2be09 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:21:38 +0200 Subject: [PATCH 35/42] docs on function naming --- src/util.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util.js b/src/util.js index 93e02f9..db311d9 100644 --- a/src/util.js +++ b/src/util.js @@ -1,5 +1,6 @@ /** * Compares objects - checks keys and values + * Called shallowEqual because javascript's `===` checks equality from objects on memory level, and not value level. * @param {Object} obj1 - first object to compare * @param {Object} obj2 - second object to compare * @returns {boolean} Boolean that indicates if the objects are equal From f1efe993f186caf149d4ad7549b738e161688fad Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:21:56 +0200 Subject: [PATCH 36/42] add interval to example config --- config.fields.example.yml | 3 ++- config.query.example.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.fields.example.yml b/config.fields.example.yml index dda61ef..283325e 100644 --- a/config.fields.example.yml +++ b/config.fields.example.yml @@ -9,4 +9,5 @@ fields: sheet: id: "ABCD1234" - name: "Sheet1" \ No newline at end of file + name: "Sheet1" + interval: 10000 \ No newline at end of file diff --git a/config.query.example.yml b/config.query.example.yml index 26c4d7b..8bf6e74 100644 --- a/config.query.example.yml +++ b/config.query.example.yml @@ -9,4 +9,5 @@ query: > sheet: id: "ABCD1234" - name: "Sheet1" \ No newline at end of file + name: "Sheet1" + interval: 10000 \ No newline at end of file From 1def2a5b52bb53f56ec155e114fb24eed9c52b5a Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:25:21 +0200 Subject: [PATCH 37/42] passive to active voice + typo fix --- TESTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/TESTS.md b/TESTS.md index dea65cf..d855905 100644 --- a/TESTS.md +++ b/TESTS.md @@ -27,7 +27,7 @@ Set the `id` section to the id of an existing Google Sheet. 2. Wait at least the configured amount of milliseconds as configured under `interval` in the configuration file (default 5000). ### Postconditions -- The changes are correctly converted and written back to the resource `http://localhost:3000/example/software`. +- The resource `http://localhost:3000/example/software` contains the new data. ## Test if changes on the Pod are synced back to the Google Sheet @@ -39,8 +39,8 @@ The pod can be updated with the following request: ```curl curl --location --request PATCH 'http://localhost:3000/example/software' --header 'Content-Type: text/n3' --data-raw '@prefix solid: . @prefix software: . @prefix schema: . _:rename a solid:InsertDeletePatch; solid:inserts { software:test schema:name "test"; schema:description "abracadabra". }.' ``` -When using websockets, the change should be almost immediately be shown, +When using websockets, the change should be almost immediately shown, otherwise wait at least the configured amount of milliseconds (default 5000ms). ### Postconditions -- The changes are correctly converted and visible in the google sheet \ No newline at end of file +- The Google sheet contains the new data. \ No newline at end of file From 821c4b348e445b3c617c8b31203040836bbdebec Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 14:26:18 +0200 Subject: [PATCH 38/42] excel => sheet --- src/google.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/google.js b/src/google.js index 1e1dda0..bc3057e 100644 --- a/src/google.js +++ b/src/google.js @@ -57,7 +57,7 @@ export async function writeToSheet(array, sheetId) { /** * Pull data from the sheet and check if there are any changes with the previously pulled data. * @param {String} sheetId - ID of the Google sheet from which the data should be pulled and checked. - * @param {String} sheetName - Name of the excel page to check + * @param {String} sheetName - Name of the Sheet page to check * @return {Promise<{Boolean, Array}>} - 2D-array containing the latest data from the sheet * and a boolean indicating a possible change. */ @@ -74,7 +74,7 @@ export async function checkSheetForChanges(sheetId, sheetName) { /** * Get the data from the sheet in the initial range. * @param {String} sheetId - ID from the sheet from which the data should be pulled. - * @param {String} sheetName - Name of the excel page to check + * @param {String} sheetName - Name of the Sheet page to check * @return {Promise} 2D-array containing the data from the sheet. */ async function getFromSheet(sheetId, sheetName) { From 822e783d2f7e8cc220d1f4a17e14794e8d285940 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 16:41:11 +0200 Subject: [PATCH 39/42] add missing host value to yaml files --- config.fields.example.yml | 1 + config.query.example.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/config.fields.example.yml b/config.fields.example.yml index 283325e..c073c7c 100644 --- a/config.fields.example.yml +++ b/config.fields.example.yml @@ -1,4 +1,5 @@ resource: "http://localhost:3000/example/software" +host: "http://localhost:3000" fields: required: diff --git a/config.query.example.yml b/config.query.example.yml index 8bf6e74..8cc8b2d 100644 --- a/config.query.example.yml +++ b/config.query.example.yml @@ -1,4 +1,5 @@ resource: "http://localhost:3000/example/software" +host: "http://localhost:3000" query: > SELECT DISTINCT * WHERE { From 9f48615bb06748597284bf3d098611044c831616 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 16:49:01 +0200 Subject: [PATCH 40/42] fixes readme --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4f43113..c6c0084 100644 --- a/README.md +++ b/README.md @@ -49,12 +49,12 @@ The synchronisation app can now read and use these tokes to access the Google Sh The synchronisation application is configured through the `config.yml` file. -#### resource (string) +### resource (string) This parameter allows a user to specify a resource. This resource should be represented as a URI to a Solid pod from which the data will be fetched. -#### host (string) +### host (string) This parameter allows a user to specify the host of a resource. This is required to use the websocket protocol to listen for changes on the resource. @@ -64,7 +64,7 @@ resource: "http://localhost:3000/example/software" host: "http://localhost:3000" ``` -#### query (string) +### query (string) This parameter allows a user to define a SPARQL query that will be used to retrieve data from the specified data sources. example: @@ -99,7 +99,8 @@ This is the name of the tab on the bottom left that you want to sync. #### interval (int) This parameter allows you to specify the number of milliseconds between polls. -This number will be used for polling the sheet for changes and polling the pod for changes when websockets aren't used. +The code will poll the sheet for changes after the specified number of milliseconds. +The code will also poll the pod after this amount of milliseconds when websockets aren't used. example: @@ -143,7 +144,7 @@ fields: #### websockets This parameter allows you to turn off websockets when you want explicit polling every 5 seconds. -This value can de modified using the `interval` option from the Google Sheet configuration. +The `interval` option from the Google Sheet configuration changes this value. example: ```yaml @@ -185,5 +186,5 @@ write back changes from the Google Sheet back to a single destination. It is required for the resource specified in the configuration file to have public read and write access, as the agent has no support for authentication. -### No 2 applications write at the same time. +### No 2 applications write at the same time Currently, it is not handled when the sheet and another application try to update the resource in the pod at exactly the same time. \ No newline at end of file From 6b3534e3135a3fef675e4c74e05c5f58e87c8236 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Thu, 17 Aug 2023 16:50:37 +0200 Subject: [PATCH 41/42] change markdown type to shell --- TESTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TESTS.md b/TESTS.md index d855905..786c851 100644 --- a/TESTS.md +++ b/TESTS.md @@ -36,7 +36,7 @@ Set the `id` section to the id of an existing Google Sheet. ### Steps The pod can be updated with the following request: -```curl +```shell curl --location --request PATCH 'http://localhost:3000/example/software' --header 'Content-Type: text/n3' --data-raw '@prefix solid: . @prefix software: . @prefix schema: . _:rename a solid:InsertDeletePatch; solid:inserts { software:test schema:name "test"; schema:description "abracadabra". }.' ``` When using websockets, the change should be almost immediately shown, From 5d83c40999d316fe21a6d937e0938a50730cf478 Mon Sep 17 00:00:00 2001 From: sevrijss Date: Fri, 18 Aug 2023 08:55:07 +0200 Subject: [PATCH 42/42] remove debug part from config yaml, moved websockets to yaml root --- src/main.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main.js b/src/main.js index 27039ff..95e6480 100644 --- a/src/main.js +++ b/src/main.js @@ -67,10 +67,8 @@ function ymlContentToConfig(ymlContent) { throw new Error("Error parsing YAML: host value should be specified") } - if (configJson.debug){ - if (configJson.debug.websockets) { - config.debug_noWebSockets = configJson.debug.websockets === "false" - } + if (configJson.websockets) { + config.noWebsockets = configJson.websockets === "false" } config.interval = configJson.sheet.interval ? configJson.sheet.interval : 5000; @@ -190,7 +188,7 @@ async function startFromFile(configPath, rulesPath) { // Pod -> Sheet sync let websocketEndpoints = await getNotificationChannelTypes(config.host + "/.well-known/solid"); - if (websocketEndpoints.length > 0 && websocketEndpoints[0].length > 0 && (!config.debug_noWebSockets)) { + if (websocketEndpoints.length > 0 && websocketEndpoints[0].length > 0 && (!config.noWebsockets)) { // listen using websockets let url = websocketEndpoints[0] let requestOptions = getWebsocketRequestOptions(config.source)