Merge branch 'master' into pico

build/makefiles/lin/serial2xsbug.mk

@@ -41,7 +41,7 @@ else
 	C_OPTIONS += -O3
 endif
 
-LIBRARIES =
+LIBRARIES = -lpthread
 LINK_OPTIONS =
 
 OBJECTS = \

build/makefiles/lin/tools.mk

@@ -230,7 +230,7 @@ else
 	C_FLAGS += -D_RELEASE=1 -O3
 endif
 
-LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs gio-2.0) -latomic
+LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs gio-2.0) -latomic -lpthread
 
 LINK_FLAGS = -fPIC
 

documentation/tools/manifest.md

@@ -1,7 +1,7 @@
 # Manifest
 
 Copyright 2017-2021 Moddable Tech, Inc.<BR>
-Revised: March 3, 2021
+Revised: March 11, 2021
 
 A manifest is a JSON file that describes the modules and resources necessary to build a Moddable app. This document explains the properties of the JSON object and how manifests are processed by the Moddable SDK build tools.
 
@@ -108,7 +108,7 @@ The Moddable SDK sdkconfig defaults files are located in the `$MODDABLE/build/de
 	When applications specify optional sdkconfig files using the `SDKCONFIGPATH` manifest environment variable, the merge processing additionally includes the following:
 
 4. On debug builds, the application `sdkconfig.defaults` file, when provided, is merged on top of the base Moddable SDK `sdkconfig.defaults` file.
-5. On release builds, the application `sdkconfig.defaults.release` options, when provided,  are merged on top of the merge performed in step 2.
+5. On release builds, the application `sdkconfig.defaults.release` options, when provided,  are merged on top of the merge performed in step 4.
 6. On release instrumented builds, the `sdkconfig.inst` options, when provided, are merged on top of the merge performed in step 5.
 
 ***

documentation/xs/XS Marshalling.md

@@ -2,36 +2,37 @@
 
 Copyright 2021 Moddable Tech, Inc.
 
-Revised: March 9, 2021
+Revised: March 15, 2021
 
 Warning: These notes are preliminary. Omissions and errors are likely. If you encounter problems, please ask for assistance.
 
-
 ## Introduction
 
-For the sake of exchanging data between machines that can run in separate threads, XS always supported marshalling thru its C programmming interface. 
+To exchange data between machines that can run in separate threads, XS always supported marshalling thru the **XS in C** programming interface. 
 
-Even if you do not write C code, this document is useful if you write JavaScript code based on the standard `Worker` programming interface provided by the [worker](https://github.com/Moddable-OpenSource/moddable/blob/public/documentation/base/worker.md) module of the Moddable SDK.
+Even if you do not write C code, this document is useful if you write JavaScript code using the standard `Worker` programming interface provided by the [worker](../base/worker.md) module of the Moddable SDK.
 
 ## What Is Marshalling?
 
-Similarly to what `JSON.stringify` and `JSON.parse` do with a string, *marshalling* creates a memory block from a JavaScript value and *demarshalling* creates a JavaScript value from a memory block. The format of the memory block is binary. 
+Similar to what `JSON.stringify` and `JSON.parse` do with a string, *marshalling* creates a memory block from a JavaScript value and *demarshalling* creates a JavaScript value from a memory block. The format of the memory block is binary. 
+
+XS machines can be created from scratch or cloned from a read-only machine. Typically on micro-controllers, multiple small machines can be created in RAM based based on one large read-only machine in ROM.
 
-XS machines can be created from scratch or based on a read-only machine. Typically on micro-controllers, multiple tiny machines can be created in RAM based based on one huge machine in ROM.
+> For those familiar with Secure ECMAScript (SES), a cloned machine is in a state analogous to the post-lockdown state in SES. In this state, all built-ins are deeply frozen. An XS created from scratch is fully mutable and therefore analogous to the pre-lockdown state in SES.
 
-When two machines are created from scratch or based on different read-only machines, they are **alien** and the corresponding programmming interface must be used:
+When two machines are created from scratch or cloned from different read-only machines, they are **alien** and the alien programming interface must be used:
 
 	void* xsMarshallAlien(xsSlot slot);
 	xsSlot xsDemarshallAlien(void* data);
 
 > In **mcsim**, the simulator and the app are alien machines. They communicate using the standard `Worker` programming interface, which is implemented here with `xsMarshallAlien` and `xsDemarshallAlien`.
 
-Otherwise the marshalling can take advantage of what is shared by the two machines and the **full** programmming interface can be used:
+When two machines are cloned from the same read-ony machinee, marshalling can take advantage of what is shared by the two machines and the **full** programming interface can be used:
 
 	void* xsMarshall(xsSlot slot);
 	xsSlot xsDemarshall(void* data);
 
-> The **worker** module creates threads and machines based on the same read-only machine. They communicate using the standard `Worker` programming interface, which is implemented here with `xsMarshall` and `xsDemarshall`.
+> The **worker** module clones all machines from the same read-only machine. They communicate using the standard `Worker` programming interface, which is implemented here with `xsMarshall` and `xsDemarshall`.
 
 ## Alien Marshalling
 
@@ -48,9 +49,9 @@ Firstly, everything that can be exchanged thru JSON:
 	- `Object`
 	- `Array`
 
-> Usually, the marshalled memory block is smaller that the JSON string and `xsMarshallAlien`/`xsDemarshallAlien` are faster than `JSON.stringify`/ `JSON.parse`.
+> Usually, the marshalled memory block is smaller that the equivalent JSON string and `xsMarshallAlien`/`xsDemarshallAlien` are faster than `JSON.stringify`/ `JSON.parse`.
 
-Then XS can marshall other values:
+XS can also marshall other values not possible using JSON:
 
 - `undefined` 
 - bigints
@@ -64,7 +65,6 @@ Then XS can marshall other values:
 	- `ArrayBuffer`, `SharedArrayBuffer`, `DataView`
 	- `Proxy`
 
-
 Furthermore, XS can marshall cyclic references:
 
 ##### main.js
@@ -91,7 +91,7 @@ Symbols are consistent inside a marshalled memory block:
 		trace(`${m.object[m.symbol]}\n`); // null
 	}
 
-But successive marshallings of the same symbol result in successive demarshalling of different symbols:
+However, successive marshallings of the same symbol result in successive demarshalling of different symbols:
 
 ##### main.js
 	let step = 0;
@@ -109,11 +109,11 @@ But successive marshallings of the same symbol result in successive demarshallin
 			trace(`${symbol === m.symbol}`) // false
 	}
 
-### No Way*
+### No Way
 
-Marshalling is about data, what is related to code, or to the execution of code, cannot be marshalled: accessors, arguments, classes, functions, generators, modules and promises.
+Marshalling is for data. Objects related to code, or to the execution of code, cannot be marshalled: accessors, arguments, classes, functions, generators, modules and promises.
 
-Also, what is related to garbage collection cannot be marshalled: finalization registries, weak references, weak maps and weak sets. And, of course, what is outside XS cannot be marshalled: host objects and host functions.
+Also, objects related to garbage collection cannot be marshalled: finalization registries, weak references, weak maps and weak sets. And, of course, objects implemented outside XS cannot be marshalled: host objects and host functions.
 
 In these cases, XS tries to report a meaningful error:
 
@@ -132,9 +132,9 @@ breaks into **xsbug** with
 
 ## Full Marshalling
 
-What can be exchanged between machines based on the same read-only machine?
+What can be exchanged between machines cloned from the same read-only machine?
 
-Firsly, everything that can be marshalled between alien machines. Then XS takes advantage of what is shared in the read-only machine to complete the marshalled instances.
+Firstly, everything that can be marshalled between alien machines. Then XS takes advantage of what is shared in the read-only machine to complete the marshalled instances.
 
 In the following examples, *preload.js* is a module that is preloaded by the XS linker. Its body is executed at link time to define classes and objects in the read-only machine.
 
@@ -231,7 +231,7 @@ References to instances in the read-only machine are preserved:
 		trace(`${m.o === o}\n`); // true
 	}
 
-That is especially useful to exchange references to objects with methods, like the handler of a proxy. Here is an example inspired by the [MDN Proxy documentation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy):
+That is especially useful for exchanging references to objects with methods, like the handler of a proxy. Here is an example inspired by the [MDN Proxy documentation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy):
 
 ##### preload.js
 	export const handler = Object.freeze({
@@ -258,3 +258,6 @@ That is especially useful to exchange references to objects with methods, like t
 	}
 
 
+### Exchange Data, Share Code
+
+Marshalling is a way to safely exchange **data** between machines. Prototypes, classes, and proxy handlers in the read-only machine are ways to safely share **code** between cloned machines.

examples/network/mqtt/mqttconnection/main.js

@@ -19,8 +19,7 @@ import WiFi from "wifi/connection";
 import config from "mc/config";
 
 const mqtt = new Client({
-	host: "test.mosquitto.org",
-	id: "moddable_" + Net.get("MAC"),
+	host: "test.mosquitto.org"
 });
 
 mqtt.onReady = function() {
@@ -54,8 +53,10 @@ new WiFi(
 	},
 	function (msg) {
 		trace(`Wi-Fi ${msg}\n`);
-		if (WiFi.gotIP === msg)
+		if (WiFi.gotIP === msg) {
+			mqtt.id = "moddable_" + Net.get("MAC");
 			mqtt.wait(false);		// connected
+		}
 		else if (WiFi.disconnected === msg)
 			mqtt.wait(true);		// lost connection
 	}

examples/network/mqtt/mqttconnection/mqttconnection.js

@@ -24,7 +24,7 @@
 
 		- tries to reconnect on connection dropped (0.5 second delay)
 		- 10 second timeout on establishing connection (until onReady)
-		- call wait(false) if no network to turn off reconnect attempts and wait(true) to restart
+		- call wait(false) if no network connection to turn off reconnect attempts and wait(true) to restart
 		- onConnected and onReady called on each reconnection
 		- onClose may be called more than once between connections
 		- any subscriptions are lost on disconnect and must be restablished on reconnect
@@ -62,7 +62,7 @@ class Connection {
 			Timer.clear(this.#timeout);
 		this.#timeout = undefined;
 
-		return mqtt?.close();
+		mqtt?.close();
 	}
 	publish(topic, data) {
 		if (!this.#ready)
@@ -158,6 +158,16 @@ class Connection {
 		else if (!this.#ready && !this.#timeout && !this.#reconnect)
 			this.#restart();
 	}
+	get id() {
+		return this.#options.id;
+	}
+	set id(value) {
+		value = value.toString();
+		if ((undefined !== this.#options.id) && (value !== this.#options.id))
+			throw new Error("may only set once");
+
+		this.#options.id = value;
+	}
 }
 
 export default Connection;

tools/mcconfig/make.lin.mk

@@ -120,7 +120,7 @@ else
 #	C_FLAGS += -DMC_MEMORY_DEBUG=1
 endif
 
-LINK_LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs gio-2.0)
+LINK_LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs gio-2.0) -lpthread
 
 LINK_OPTIONS = -fPIC -shared -Wl,-Bdynamic\,-Bsymbolic
 

tools/mcconfig/make.x-lin.mk

@@ -118,7 +118,7 @@ else
 #	C_FLAGS += -DMC_MEMORY_DEBUG=1
 endif
 
-LINK_LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs freetype2 gtk+-3.0) -ldl -latomic
+LINK_LIBRARIES = -lm -lc $(shell $(PKGCONFIG) --libs freetype2 gtk+-3.0) -ldl -latomic -lpthread
 
 LINK_OPTIONS = -fPIC
 

tools/mcmanifest.js

@@ -156,12 +156,16 @@ export class MakeFile extends FILE {
 		// Merge any application sdkconfig files
 		if (tool.environment.SDKCONFIGPATH != baseConfigDirectory) {
 			let appConfigFile = tool.environment.SDKCONFIGPATH + tool.slash + "sdkconfig.defaults";
-			if ((false === tool.debug) && (1 == tool.isDirectoryOrFile(appConfigFile + ".release")))
-				appConfigFile += ".release";
 			if (1 == tool.isDirectoryOrFile(appConfigFile)) {
 				let entries = tool.readFileString(appConfigFile);
 				mergedConfig = mergedConfig.concat(entries.split(regex));
 			}
+
+			if ((false === tool.debug) && (1 == tool.isDirectoryOrFile(appConfigFile + ".release"))) {
+				let entries = tool.readFileString(appConfigFile + ".release");
+				mergedConfig = mergedConfig.concat(entries.split(regex));
+			}
+
 			if (tool.debug === false && tool.instrument === true) {
 				appConfigFile = tool.environment.SDKCONFIGPATH + tool.slash + "sdkconfig.inst";
 				if (1 == tool.isDirectoryOrFile(appConfigFile)) {

xs/makefiles/lin/xsc.mk

@@ -72,7 +72,7 @@ else
 	C_OPTIONS += -O3
 endif
 
-LIBRARIES = -lm -ldl -latomic
+LIBRARIES = -lm -ldl -latomic -lpthread
 
 LINK_OPTIONS = -rdynamic
 

xs/makefiles/lin/xsid.mk

@@ -53,7 +53,7 @@ else
 	C_OPTIONS += -O3
 endif
 
-LIBRARIES = -lm -ldl -latomic
+LIBRARIES = -lm -ldl -latomic -lpthread
 
 LINK_OPTIONS = -rdynamic
 

xs/makefiles/lin/xsl.mk

@@ -81,7 +81,7 @@ else
 	C_OPTIONS += -O3
 endif
 
-LIBRARIES = -ldl -lm -latomic
+LIBRARIES = -ldl -lm -latomic -lpthread
 
 LINK_OPTIONS = -rdynamic
 

xs/sources/xsAPI.c

@@ -266,7 +266,7 @@ void fxStringX(txMachine* the, txSlot* theSlot, txString theValue)
 
 void fxStringBuffer(txMachine* the, txSlot* theSlot, txString theValue, txSize theSize)
 {
-	theSlot->value.string = (txString)fxNewChunk(the, theSize + 1);
+	theSlot->value.string = (txString)fxNewChunk(the, fxAddChunkSizes(the, theSize, 1));
 	if (theValue)
 		c_memcpy(theSlot->value.string, theValue, theSize);
 	else
@@ -490,7 +490,7 @@ void fxArrayCacheEnd(txMachine* the, txSlot* reference)
 	txIndex length = array->value.array.length;
 	if (length) {
 		txSlot *srcSlot, *dstSlot;
-		array->value.array.address = (txSlot*)fxNewChunk(the, length * sizeof(txSlot));
+		array->value.array.address = (txSlot*)fxNewChunk(the, fxMultiplyChunkSizes(the, length, sizeof(txSlot)));
 		srcSlot = array->next;
 		dstSlot = array->value.array.address + length;
 		while (srcSlot) {

xs/sources/xsAll.c

@@ -42,7 +42,7 @@ txString fxAdornStringC(txMachine* the, txString prefix, txSlot* string, txStrin
 	txSize stringSize = c_strlen(string->value.string);
 	txSize prefixSize = prefix ? c_strlen(prefix) : 0;
 	txSize suffixSize = suffix ? c_strlen(suffix) : 0;
-	txSize resultSize = stringSize + prefixSize + suffixSize + 1;
+	txSize resultSize = fxAddChunkSizes(the, fxAddChunkSizes(the, fxAddChunkSizes(the, stringSize, prefixSize), suffixSize), 1);
 	txString result = (txString)fxNewChunk(the, resultSize);
 	if (prefix && prefixSize)
 		c_memcpy(result, prefix, prefixSize);
@@ -159,7 +159,8 @@ txString fxConcatString(txMachine* the, txSlot* a, txSlot* b)
 {
 	txSize aSize = c_strlen(a->value.string);
 	txSize bSize = c_strlen(b->value.string);
-	txString result = (txString)fxNewChunk(the, aSize + bSize + 1);
+	txSize resultSize = fxAddChunkSizes(the, fxAddChunkSizes(the, aSize, bSize), 1);
+	txString result = (txString)fxNewChunk(the, resultSize);
 	c_memcpy(result, a->value.string, aSize);
 	c_memcpy(result + aSize, b->value.string, bSize + 1);
 	a->value.string = result;
@@ -171,7 +172,7 @@ txString fxConcatStringC(txMachine* the, txSlot* a, txString b)
 {
 	txSize aSize = c_strlen(a->value.string);
 	txSize bSize = c_strlen(b);
-	txSize resultSize = aSize + bSize + 1;
+	txSize resultSize = fxAddChunkSizes(the, fxAddChunkSizes(the, aSize, bSize), 1);
 	txString result = C_NULL;
 	if (a->kind == XS_STRING_KIND)
 		result = (txString)fxRenewChunk(the, a->value.string, resultSize);
@@ -196,8 +197,9 @@ txString fxCopyString(txMachine* the, txSlot* a, txSlot* b)
 txString fxCopyStringC(txMachine* the, txSlot* a, txString b)
 {
 	txSize bSize = c_strlen(b);
-	txString result = (txString)fxNewChunk(the, bSize + 1);
-	c_memcpy(result, b, bSize + 1);
+	txSize resultSize = fxAddChunkSizes(the, bSize, 1);
+	txString result = (txString)fxNewChunk(the, resultSize);
+	c_memcpy(result, b, resultSize);
 	a->value.string = result;
 	a->kind = XS_STRING_KIND;
 	return result;

xs/sources/xsAll.h

@@ -748,11 +748,13 @@ extern txBoolean fxIsSameSlot(txMachine* the, txSlot* a, txSlot* b);
 extern txBoolean fxIsSameValue(txMachine* the, txSlot* a, txSlot* b, txBoolean zero);
 
 /* xsMemory.c */
+extern txSize fxAddChunkSizes(txMachine* the, txSize a, txSize b);
 extern void fxCheckStack(txMachine* the, txSlot* slot);
 extern void fxAllocate(txMachine* the, txCreation* theCreation);
 extern void fxCollect(txMachine* the, txBoolean theFlag);
 mxExport txSlot* fxDuplicateSlot(txMachine* the, txSlot* theSlot);
 extern void fxFree(txMachine* the);
+extern txSize fxMultiplyChunkSizes(txMachine* the, txSize a, txSize b);
 mxExport void* fxNewChunk(txMachine* the, txSize theSize);
 extern txSlot* fxNewSlot(txMachine* the);
 mxExport void* fxRenewChunk(txMachine* the, void* theData, txSize theSize);

xs/sources/xsArray.c

@@ -219,7 +219,7 @@ void fxCacheArray(txMachine* the, txSlot* instance)
 	txSlot* array = instance->next;
 	txIndex length = array->value.array.length;
 	if (length) {
-		txSlot* address = (txSlot *)fxNewChunk(the, length * sizeof(txSlot));
+		txSlot* address = (txSlot *)fxNewChunk(the, fxMultiplyChunkSizes(the, length, sizeof(txSlot)));
 		txSlot* srcSlot = array->next;
 		txSlot* dstSlot = address;
 		txIndex index = 0;
@@ -640,7 +640,7 @@ txBoolean fxSetArrayLength(txMachine* the, txSlot* array, txIndex length)
 		if (result < limit) {
 			if (result > address) {
 				size = result - address;
-				chunk = (txSlot*)fxNewChunk(the, size * sizeof(txSlot));
+				chunk = (txSlot*)fxNewChunk(the, fxMultiplyChunkSizes(the, size, sizeof(txSlot)));
 				address = array->value.array.address;
 				c_memcpy(chunk, address, size * sizeof(txSlot));
 			}
@@ -1563,13 +1563,13 @@ void fx_Array_prototype_join(txMachine* the)
 			string = fxToString(the, slot);
 			slot->kind += XS_KEY_KIND - XS_STRING_KIND;
 			slot->value.key.sum = c_strlen(string);
-			size += slot->value.key.sum;
+			size = fxAddChunkSizes(the, size, slot->value.key.sum);
 		}
 		mxPop();
 		index++;
 	}
 	mxPop();
-	string = mxResult->value.string = fxNewChunk(the, size + 1);
+	string = mxResult->value.string = fxNewChunk(the, fxAddChunkSizes(the, size, 1));
 	slot = list->next;
 	while (slot) {
 		c_memcpy(string, slot->value.key.string, slot->value.key.sum);
@@ -2321,12 +2321,12 @@ void fx_Array_prototype_toLocaleString(txMachine* the)
 			string = fxToString(the, slot);
 			slot->kind += XS_KEY_KIND - XS_STRING_KIND;
 			slot->value.key.sum = c_strlen(string);
-			size += slot->value.key.sum;
+			size = fxAddChunkSizes(the, size, slot->value.key.sum);
 		}
 		mxPop();
 		index++;
 	}
-	string = mxResult->value.string = fxNewChunk(the, size + 1);
+	string = mxResult->value.string = fxNewChunk(the, fxAddChunkSizes(the, size, 1));
 	slot = list->next;
 	while (slot) {
 		c_memcpy(string, slot->value.key.string, slot->value.key.sum);