diff --git a/.github/workflows/deploy-pages.yaml b/.github/workflows/deploy-pages.yaml index bda1e08e..fe2635a8 100644 --- a/.github/workflows/deploy-pages.yaml +++ b/.github/workflows/deploy-pages.yaml @@ -18,12 +18,11 @@ jobs: - name: Checkout uses: actions/checkout@v2.3.1 - name: Install and Build 🔧 - # Ensure public files are available when hosted as a web application + # Last step ensures public files are available when hosting the web application run: | - rm -rf dist/ - cp -r public/ dist yarn install - yarn build + yarn build:app + cp -r public/* dist - name: Deploy 🚀 uses: JamesIves/github-pages-deploy-action@4.1.5 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 245ce0f9..59b718bd 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -14,7 +14,7 @@ jobs: - name: Install and Build 🔧 run: | yarn install - yarn build + yarn build:library - name: Publish @dodona/papyros to NPM uses: JS-DevTools/npm-publish@v1 with: diff --git a/.gitignore b/.gitignore index a7043600..a27cf1c6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ yarn-error.log* # script outputs translationIssues.txt + +src/workers/python/python_package/ diff --git a/package.json b/package.json index 5ab29d21..c1933cbf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dodona/papyros", - "version": "0.1.90", + "version": "0.1.925-tar", "private": false, "homepage": ".", "devDependencies": { @@ -33,9 +33,10 @@ "ts-jest": "^27.0.7", "ts-loader": "^9.2.6", "typescript": "^4.1.2", - "webpack": "^5.64.0", - "webpack-cli": "^4.9.1", - "webpack-dev-server": "^4.6.0", + "url-loader": "^4.1.1", + "webpack": "^5.70.0", + "webpack-cli": "^4.9.2", + "webpack-dev-server": "^4.7.4", "worker-loader": "^3.0.8" }, "dependencies": { @@ -53,13 +54,17 @@ "@codemirror/search": "^0.19.3", "@codemirror/state": "^0.19.6", "comlink": "^4.3.1", + "comsync": "^0.0.7", "escape-html": "^1.0.3", "i18n-js": "^3.8.0", - "sync-message": "^0.0.8" + "pyodide-worker-runner": "^0.0.8", + "sync-message": "^0.0.9" }, "scripts": { - "start": "webpack serve --mode development", - "build": "webpack build --mode production", + "start": "yarn setup && webpack serve --mode development", + "setup": "bash scripts/setup.sh", + "build:app": "bash scripts/build_package.sh development", + "build:library": "bash scripts/build_package.sh production", "test": "echo No tests defined yet", "lint": "eslint --ext '.js' --ext '.ts' src", "validate:translations": "node scripts/ValidateTranslations.js" diff --git a/scripts/build_package.sh b/scripts/build_package.sh new file mode 100644 index 00000000..b99dbcf3 --- /dev/null +++ b/scripts/build_package.sh @@ -0,0 +1,4 @@ +#!/bin/bash +rm -rf dist +bash ./scripts/setup.sh +webpack build --mode=$1 diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 00000000..375fdc05 --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd src/workers/python +python3.9 build_package.py + + diff --git a/src/App.ts b/src/App.ts index 6f427f6b..f2e037b7 100644 --- a/src/App.ts +++ b/src/App.ts @@ -5,6 +5,8 @@ import { import { Papyros } from "./Papyros"; import { InputMode } from "./InputManager"; import { getElement } from "./util/Util"; +import { papyrosLog, LogType } from "./util/Logging"; +import { BackendManager } from "./BackendManager"; async function startPapyros(): Promise { @@ -28,11 +30,12 @@ async function startPapyros(): Promise { } }); // Try to configure synchronous input mechanism - if (!await papyros.configureInput(location.href, DEFAULT_SERVICE_WORKER, false)) { + if (!await papyros.configureInput(location.href, DEFAULT_SERVICE_WORKER)) { getElement(MAIN_APP_ID).innerHTML = "Your browser is unsupported.\n" + "Please use a modern version of Chrome, Safari, Firefox, ..."; } else { // Start actual application + papyrosLog(LogType.Debug, "Using channel: ", BackendManager.channel); papyros.launch(); } } diff --git a/src/Backend.ts b/src/Backend.ts index 20abf63c..3643be7d 100644 --- a/src/Backend.ts +++ b/src/Backend.ts @@ -1,7 +1,7 @@ -import { PapyrosEvent } from "./PapyrosEvent"; -import { Channel, readMessage, uuidv4 } from "sync-message"; -import { parseData } from "./util/Util"; import { CompletionContext, CompletionResult } from "@codemirror/autocomplete"; +import { BackendEvent, BackendEventType } from "./BackendEvent"; +import { papyrosLog, LogType } from "./util/Logging"; +import { syncExpose, SyncExtras } from "comsync"; /** * Interface to represent the CodeMirror CompletionContext in a worker @@ -37,72 +37,54 @@ export interface WorkerAutocompleteContext { } | null; } -export abstract class Backend { - protected onEvent: (e: PapyrosEvent) => any; - protected runId: number; - +export abstract class Backend { + protected extras: Extras; + protected onEvent: (e: BackendEvent) => any; /** - * Constructor is limited as it is meant to be used as a WebWorker - * These are then exposed via Comlink. Proper initialization occurs - * in the launch method when the worker is started + * Constructor is limited as it is meant to be used as a WebWorker + * Proper initialization occurs in the launch method when the worker is started + * Synchronously exposing methods should be done here */ constructor() { + this.extras = {} as Extras; // eslint-disable-next-line @typescript-eslint/no-empty-function this.onEvent = () => { }; - this.runId = 0; + this.runCode = this.syncExpose()(this.runCode.bind(this)); + } + + /** + * @return {any} The function to expose methods for Comsync to allow interrupting + */ + protected syncExpose(): any { + return syncExpose; } /** * Initialize the backend by doing all setup-related work - * @param {function(PapyrosEvent):void} onEvent Callback for when events occur - * @param {Channel} channel for communication with the main thread + * @param {function(BackendEvent):void} onEvent Callback for when events occur * @return {Promise} Promise of launching */ launch( - onEvent: (e: PapyrosEvent) => void, - channel: Channel + onEvent: (e: BackendEvent) => void ): Promise { - // Input messages are handled in a special way - // In order to link input requests to their responses - // An ID is required to make the connection - // The message must be read in the worker to not stall the main thread - const onInput = (e: PapyrosEvent): string => { - const inputData = parseData(e.data, e.contentType); - const messageId = uuidv4(); - inputData.messageId = messageId; - e.data = JSON.stringify(inputData); - e.contentType = "text/json"; + this.onEvent = (e: BackendEvent) => { onEvent(e); - return readMessage(channel, messageId); - }; - this.onEvent = (e: PapyrosEvent) => { - e.runId = this.runId; - if (e.type === "input") { - return onInput(e); - } else { - return onEvent(e); + if (e.type === BackendEventType.Sleep) { + return this.extras.syncSleep(e.data); + } else if (e.type === BackendEventType.Input) { + return this.extras.readMessage(); } }; return Promise.resolve(); } - /** - * Internal helper method that actually executes the code - * Results or Errors must be passed by using the onEvent-callback - * @param code The code to run - */ - protected abstract _runCodeInternal(code: string): Promise; - /** * Executes the given code + * @param {Extras} extras Helper properties to run code * @param {string} code The code to run - * @param {string} runId The uuid for this execution * @return {Promise} Promise of execution */ - async runCode(code: string, runId: number): Promise { - this.runId = runId; - return await this._runCodeInternal(code); - } + abstract runCode(extras: Extras, code: string): Promise; /** * Converts the context to a cloneable object containing useful properties @@ -110,7 +92,6 @@ export abstract class Backend { * Class instances are not passable to workers, so we extract the useful information * @param {CompletionContext} context Current context to autocomplete for * @param {RegExp} expr Expression to match the previous token with - * default a word with an optional dot to represent property access * @return {WorkerAutocompleteContext} Completion context that can be passed as a message */ static convertCompletionContext(context: CompletionContext, expr = /\w*(\.)?/): @@ -120,7 +101,7 @@ export abstract class Backend { return [line.number, (range.head - line.from)]; })[0]; const beforeMatch = context.matchBefore(expr); - return { + const ret = { explicit: context.explicit, before: beforeMatch, pos: context.pos, @@ -128,6 +109,8 @@ export abstract class Backend { line: lineNr, text: context.state.doc.toString() }; + papyrosLog(LogType.Debug, "Worker completion context:", ret); + return ret; } /** diff --git a/src/BackendEvent.ts b/src/BackendEvent.ts new file mode 100644 index 00000000..586e6b1d --- /dev/null +++ b/src/BackendEvent.ts @@ -0,0 +1,38 @@ +/** + * Enum representing all possible types for supported events + */ +export enum BackendEventType { + Start = "start", + End = "end", + Input = "input", + Output = "output", + Sleep = "sleep", + Error = "error", + Interrupt = "interrupt" +} +/** + * All possible types for ease of iteration + */ +export const BACKEND_EVENT_TYPES = [ + BackendEventType.Start, BackendEventType.End, + BackendEventType.Input, BackendEventType.Output, + BackendEventType.Sleep, BackendEventType.Error, + BackendEventType.Interrupt +]; +/** + * Interface for events used for communication between threads + */ +export interface BackendEvent { + /** + * The type of action generating this event + */ + type: BackendEventType; + /** + * The actual data stored in this event + */ + data: any; + /** + * The format used for the data to help with parsing + */ + contentType?: string; +} diff --git a/src/BackendManager.ts b/src/BackendManager.ts index 17b408b2..47bd3605 100644 --- a/src/BackendManager.ts +++ b/src/BackendManager.ts @@ -1,45 +1,115 @@ -import { releaseProxy, Remote, wrap } from "comlink"; import { Backend } from "./Backend"; import { ProgrammingLanguage } from "./ProgrammingLanguage"; import PythonWorker from "./workers/python/PythonWorker.worker"; import JavaScriptWorker from "./workers/javascript/JavaScriptWorker.worker"; -// Store Worker per Backend as Comlink proxy has no explicit reference to the Worker -// We need the Worker itself to be able to terminate it (@see stopBackend) -const BACKEND_MAP: Map, Worker> = new Map(); - -const CREATE_WORKER_MAP: Map Worker> = new Map([ - [ProgrammingLanguage.Python, () => new PythonWorker()], - [ProgrammingLanguage.JavaScript, () => new JavaScriptWorker()] -]); - +import { BackendEvent, BackendEventType } from "./BackendEvent"; +import { LogType, papyrosLog } from "./util/Logging"; +import { Channel, makeChannel } from "sync-message"; +import { SyncClient } from "comsync"; +import { PyodideClient } from "pyodide-worker-runner"; /** - * Start a backend for the given language, while storing the worker - * @param {ProgrammingLanguage} language The programming language supported by the backend - * @return {Remote} A Comlink proxy for the Backend + * Callback type definition for subscribers + * @param {BackendEvent} e The published event */ -export function startBackend(language: ProgrammingLanguage): Remote { - if (CREATE_WORKER_MAP.has(language)) { - const worker = CREATE_WORKER_MAP.get(language)!(); - const backend = wrap(worker); - // store worker itself in the map - BACKEND_MAP.set(backend, worker); - return backend; - } else { - throw new Error(`${language} is not yet supported.`); - } -} +type BackendEventListener = (e: BackendEvent) => void; /** - * Stop a backend by terminating the worker and releasing memory - * @param {Remote} backend The proxy for the backend to stop + * Abstract class to implement the singleton pattern + * Static methods group functionality */ -export function stopBackend(backend: Remote): void { - if (BACKEND_MAP.has(backend)) { - const toStop = BACKEND_MAP.get(backend)!; - toStop.terminate(); - backend[releaseProxy](); - BACKEND_MAP.delete(backend); - } else { - throw new Error(`Unknown backend supplied for backend ${JSON.stringify(backend)}`); +export abstract class BackendManager { + /** + * Map programming languages to Backend constructors + */ + private static createBackendMap: Map SyncClient>; + /** + * Map to cache Backends per ProgrammingLanguage + */ + private static backendMap: Map>; + /** + * Map an event type to interested subscribers + * Uses an Array to maintain order of subscription + */ + static subscriberMap: Map>; + /** + * The channel used to communicate with the SyncClients + */ + static channel: Channel; + + /** + * @param {ProgrammingLanguage} language The language to support + * @param {Function} backendCreator The constructor for a SyncClient + */ + static registerBackend(language: ProgrammingLanguage, + backendCreator: () => SyncClient): void { + BackendManager.createBackendMap.set(language, backendCreator); + BackendManager.backendMap.delete(language); + } + + /** + * Start a backend for the given language and cache for reuse + * @param {ProgrammingLanguage} language The programming language supported by the backend + * @return {SyncClient} A SyncClient for the Backend + */ + static startBackend(language: ProgrammingLanguage): SyncClient { + if (this.backendMap.has(language)) { // Cached + return this.backendMap.get(language)!; + } else if (this.createBackendMap.has(language)) { + // Create and then cache + const syncClient = this.createBackendMap.get(language)!(); + this.backendMap.set(language, syncClient); + return syncClient; + } else { + throw new Error(`${language} is not yet supported.`); + } + } + + /** + * Register a callback for when an event of a certain type is published + * @param {BackendEventType} type The type of event to subscribe to + * @param {BackendEventListener} subscriber Callback for when an event + * of the given type is published + */ + static subscribe(type: BackendEventType, subscriber: BackendEventListener): void { + if (!this.subscriberMap.has(type)) { + this.subscriberMap.set(type, []); + } + const subscribers = this.subscriberMap.get(type)!; + if (!subscribers.includes(subscriber)) { + subscribers.push(subscriber); + } + } + + /** + * Publish an event, notifying all listeners for its type + * @param {BackendEventType} e The event to publish + */ + static publish(e: BackendEvent): void { + papyrosLog(LogType.Debug, "Publishing event: ", e); + if (this.subscriberMap.has(e.type)) { + this.subscriberMap.get(e.type)!.forEach(cb => cb(e)); + } + } + + /** + * Initialise the fields and setup the maps + */ + static { + BackendManager.channel = makeChannel()!; + BackendManager.createBackendMap = new Map(); + BackendManager.backendMap = new Map(); + BackendManager.subscriberMap = new Map(); + BackendManager.registerBackend(ProgrammingLanguage.Python, + () => new PyodideClient( + () => new PythonWorker(), + BackendManager.channel + ) + ); + BackendManager.registerBackend(ProgrammingLanguage.JavaScript, + () => new SyncClient( + () => new JavaScriptWorker(), + BackendManager.channel + ) + ); } } diff --git a/src/CodeEditor.ts b/src/CodeEditor.ts index 7bb3a89a..9c184d15 100644 --- a/src/CodeEditor.ts +++ b/src/CodeEditor.ts @@ -26,7 +26,7 @@ import { rectangularSelection } from "@codemirror/rectangular-selection"; import { defaultHighlightStyle } from "@codemirror/highlight"; import { lintKeymap } from "@codemirror/lint"; import { showPanel } from "@codemirror/panel"; -import { RenderOptions, renderWithOptions } from "./util/Util"; +import { RenderOptions, renderWithOptions, t } from "./util/Util"; /** * Component that provides useful features to users writing code @@ -64,15 +64,14 @@ export class CodeEditor { * @param {string} initialCode The initial code to display * @param {number} indentLength The length in spaces for the indent unit */ - constructor(language: ProgrammingLanguage, - editorPlaceHolder: string, initialCode = "", indentLength = 4) { + constructor(initialCode = "", indentLength = 4) { this.editorView = new EditorView( { state: EditorState.create({ doc: initialCode, extensions: [ - this.languageCompartment.of(CodeEditor.getLanguageSupport(language)), + this.languageCompartment.of([]), this.autocompletionCompartment.of( autocompletion() ), @@ -80,7 +79,7 @@ export class CodeEditor { indentUnit.of(CodeEditor.getIndentUnit(indentLength)) ), keymap.of([indentWithTab]), - this.placeholderCompartment.of(placeholder(editorPlaceHolder)), + this.placeholderCompartment.of([]), this.panelCompartment.of(showPanel.of(null)), ...CodeEditor.getExtensions() ] @@ -105,20 +104,29 @@ export class CodeEditor { } /** - * Set the language that is currently used, with a corresponding placeholder + * Set the language that is currently used * @param {ProgrammingLanguage} language The language to use - * @param {CompletionSource} completionSource Function to generate autocomplete results - * @param {string} editorPlaceHolder Placeholder when empty */ - setLanguage(language: ProgrammingLanguage, completionSource: CompletionSource, - editorPlaceHolder: string): void { + setLanguage(language: ProgrammingLanguage) + : void { this.editorView.dispatch({ effects: [ this.languageCompartment.reconfigure(CodeEditor.getLanguageSupport(language)), + this.placeholderCompartment.reconfigure(placeholder(t("Papyros.code_placeholder", + { programmingLanguage: language }))) + ] + }); + } + + /** + * @param {CompletionSource} completionSource Function to obtain autocomplete results + */ + setCompletionSource(completionSource: CompletionSource): void { + this.editorView.dispatch({ + effects: [ this.autocompletionCompartment.reconfigure( autocompletion({ override: [completionSource] }) - ), - this.placeholderCompartment.reconfigure(placeholder(editorPlaceHolder)) + ) ] }); } diff --git a/src/CodeRunner.ts b/src/CodeRunner.ts new file mode 100644 index 00000000..4c64bdc9 --- /dev/null +++ b/src/CodeRunner.ts @@ -0,0 +1,277 @@ +import { proxy } from "comlink"; +import { SyncClient } from "comsync"; +import { Backend } from "./Backend"; +import { BackendEvent, BackendEventType } from "./BackendEvent"; +import { BackendManager } from "./BackendManager"; +import { CodeEditor } from "./CodeEditor"; +import { + APPLICATION_STATE_TEXT_ID, RUN_BTN_ID, + STATE_SPINNER_ID, STOP_BTN_ID +} from "./Constants"; +import { InputManager } from "./InputManager"; +import { ProgrammingLanguage } from "./ProgrammingLanguage"; +import { svgCircle } from "./util/HTMLShapes"; +import { LogType, papyrosLog } from "./util/Logging"; +import { + addListener, ButtonOptions, renderButton, + RenderOptions, renderWithOptions, getElement, + t +} from "./util/Util"; + +interface DynamicButton { + id: string; + buttonHTML: string; + onClick: () => void; +} + +/** + * Enum representing the possible states while processing code + */ +export enum RunState { + Loading = "loading", + Running = "running", + AwaitingInput = "awaiting_input", + Stopping = "stopping", + Ready = "ready" +} +/** + * Helper component to manage and visualize the current RunState + */ +export class CodeRunner { + /** + * The currently used programming language + */ + private programmingLanguage: ProgrammingLanguage; + /** + * The editor in which the code is written + */ + readonly editor: CodeEditor; + /** + * Component to request and handle input from the user + */ + readonly inputManager: InputManager; + /** + * The backend that executes the code asynchronously + */ + private backend: Promise>; + /** + * Current state of the program + */ + private state: RunState; + /** + * Buttons managed by this component + */ + private buttons: Array; + + /** + * Construct a new RunStateManager with the given listeners + * @param {ProgrammingLanguage} programmingLanguage The language to use + */ + constructor(programmingLanguage: ProgrammingLanguage) { + this.programmingLanguage = programmingLanguage; + this.editor = new CodeEditor(); + this.inputManager = new InputManager(async (input: string) => { + (await this.backend).writeMessage(input); + this.setState(RunState.Running); + }); + this.backend = Promise.resolve({} as SyncClient); + this.buttons = []; + this.addButton({ + id: RUN_BTN_ID, + buttonText: t("Papyros.run"), + extraClasses: "text-white bg-blue-500" + }, () => this.runCode()); + this.addButton({ + id: STOP_BTN_ID, + buttonText: t("Papyros.stop"), + extraClasses: "text-white bg-red-500" + }, () => this.stop()); + BackendManager.subscribe(BackendEventType.Input, + () => this.setState(RunState.AwaitingInput)); + this.state = RunState.Ready; + } + + /** + * Start the backend to enable running code + */ + async start(): Promise { + this.setState(RunState.Loading); + const backend = BackendManager.startBackend(this.programmingLanguage); + this.editor.setLanguage(this.programmingLanguage); + // Use a Promise to immediately enable running while downloading + this.backend = new Promise(resolve => { + return backend.workerProxy + // Allow passing messages between worker and main thread + .launch(proxy((e: BackendEvent) => BackendManager.publish(e))) + .then(() => { + this.editor.setCompletionSource(async context => { + const completionContext = Backend.convertCompletionContext(context); + return backend.workerProxy.autocomplete(completionContext); + }); + return resolve(backend); + }); + }); + this.editor.focus(); + this.setState(RunState.Ready); + } + + /** + * Interrupt the currently running code + * @return {Promise} Promise of stopping + */ + async stop(): Promise { + this.setState(RunState.Stopping); + BackendManager.publish({ + type: BackendEventType.End, + data: "User cancelled run", contentType: "text/plain" + }); + await this.backend.then(b => b.interrupt()); + } + + /** + * Set the used programming language to the given one to allow editing and running code + * @param {ProgrammingLanguage} programmingLanguage The language to use + */ + async setProgrammingLanguage(programmingLanguage: ProgrammingLanguage): Promise { + if (this.programmingLanguage !== programmingLanguage) { // Expensive, so ensure it is needed + await this.backend.then(b => b.interrupt()); + this.programmingLanguage = programmingLanguage; + await this.start(); + } + } + + getProgrammingLanguage(): ProgrammingLanguage { + return this.programmingLanguage; + } + + /** + * Get the button to run the code + */ + get runButton(): HTMLButtonElement { + return getElement(RUN_BTN_ID); + } + + /** + * Get the button to interrupt the code + */ + get stopButton(): HTMLButtonElement { + return getElement(STOP_BTN_ID); + } + + /** + * Show or hide the spinning circle, representing a running animation + * @param {boolean} show Whether to show the spinner + */ + showSpinner(show: boolean): void { + getElement(STATE_SPINNER_ID).style.display = show ? "" : "none"; + } + + /** + * Show the current state of the program to the user + * @param {RunState} state The current state of the run + * @param {string} message Optional message to indicate the state + */ + setState(state: RunState, message?: string): void { + this.state = state; + this.stopButton.disabled = [RunState.Ready, RunState.Loading].includes(state); + if (state === RunState.Ready) { + this.showSpinner(false); + this.runButton.disabled = false; + } else { + this.showSpinner(true); + this.runButton.disabled = true; + } + getElement(APPLICATION_STATE_TEXT_ID).innerText = + message || t(`Papyros.states.${state}`); + } + + getState(): RunState { + return this.state; + } + + /** + * Add a button to display to the user + * @param {ButtonOptions} options Options for rendering the button + * @param {function} onClick Listener for click events on the button + */ + addButton(options: ButtonOptions, onClick: () => void): void { + this.buttons.push({ + id: options.id, + buttonHTML: renderButton(options), + onClick: onClick + }); + } + + /** + * Render the RunStateManager with the given options + * @param {RenderOptions} statusPanelOptions Options for rendering the panel + * @param {RenderOptions} inputOptions Options for rendering the InputManager + * @param {RenderOptions} codeEditorOptions Options for rendering the editor + * @return {HTMLElement} The rendered RunStateManager + */ + render(statusPanelOptions: RenderOptions, + inputOptions: RenderOptions, + codeEditorOptions: RenderOptions): HTMLElement { + const rendered = renderWithOptions(statusPanelOptions, ` +
+
+ ${this.buttons.map(b => b.buttonHTML).join("\n")} +
+
+
+ ${svgCircle(STATE_SPINNER_ID, "red")} +
+
`); + // Buttons are freshly added to the DOM, so attach listeners now + this.buttons.forEach(b => addListener(b.id, b.onClick, "click")); + this.inputManager.render(inputOptions); + this.editor.render(codeEditorOptions, rendered); + return rendered; + } + + /** + * Run the code that is currently present in the editor + * @return {Promise} Promise of running the code + */ + async runCode(): Promise { + // Setup pre-run + this.setState(RunState.Running); + BackendManager.publish({ + type: BackendEventType.Start, + data: "User started run", contentType: "text/plain" + }); + papyrosLog(LogType.Debug, "Running code in Papyros, sending to backend"); + const start = new Date().getTime(); + let endMessage = "Program finishd normally"; + let terminated = false; + try { + await (this.backend).then(b => b.call( + b.workerProxy.runCode, this.editor.getCode() + )); + } catch (error: any) { + papyrosLog(LogType.Debug, "Error during code run", error); + if (error.type === "InterruptError") { + // Error signaling forceful interrupt + terminated = true; + } + BackendManager.publish({ + type: BackendEventType.Error, + data: JSON.stringify(error), + contentType: "text/json" + }); + endMessage = "Program terminated due to error: " + error.constructor.name; + } finally { + if (this.state !== RunState.Stopping) { // Was interrupted + BackendManager.publish({ + type: BackendEventType.End, + data: endMessage, contentType: "text/plain" + }); + } + const end = new Date().getTime(); + this.setState(RunState.Ready, t("Papyros.finished", { time: (end - start) / 1000 })); + if (terminated) { + await this.start(); + } + } + } +} diff --git a/src/Constants.ts b/src/Constants.ts index dc92d6c7..44fdc9b7 100644 --- a/src/Constants.ts +++ b/src/Constants.ts @@ -8,7 +8,7 @@ import { ProgrammingLanguage } from "./ProgrammingLanguage"; function addPapyrosPrefix(s: string): string { return `__papyros-${s}`; } -/* Default HT%M ids for various elements */ +/* Default HTML ids for various elements */ export const MAIN_APP_ID = addPapyrosPrefix("papyros"); export const OUTPUT_TA_ID = addPapyrosPrefix("code-output-area"); export const INPUT_AREA_WRAPPER_ID = addPapyrosPrefix("code-input-area-wrapper"); diff --git a/src/InputManager.ts b/src/InputManager.ts index 043bf8cd..d57e9972 100644 --- a/src/InputManager.ts +++ b/src/InputManager.ts @@ -1,19 +1,18 @@ import { t } from "i18n-js"; import { SWITCH_INPUT_MODE_A_ID, - INPUT_TA_ID, SEND_INPUT_BTN_ID, USER_INPUT_WRAPPER_ID + USER_INPUT_WRAPPER_ID } from "./Constants"; -import { PapyrosEvent } from "./PapyrosEvent"; +import { BackendEvent, BackendEventType } from "./BackendEvent"; import { papyrosLog, LogType } from "./util/Logging"; import { - addListener, parseData, + addListener, RenderOptions, renderWithOptions } from "./util/Util"; -import { Channel, makeChannel, writeMessage } from "sync-message"; import { InteractiveInputHandler } from "./input/InteractiveInputHandler"; import { UserInputHandler } from "./input/UserInputHandler"; import { BatchInputHandler } from "./input/BatchInputHandler"; -import { RunListener } from "./RunListener"; +import { BackendManager } from "./BackendManager"; export enum InputMode { Interactive = "interactive", @@ -22,50 +21,45 @@ export enum InputMode { export const INPUT_MODES = [InputMode.Batch, InputMode.Interactive]; -export interface InputData { - prompt: string; - messageId: string; -} - -export class InputManager implements RunListener { - private _inputMode: InputMode; +export class InputManager { + private inputMode: InputMode; private inputHandlers: Map; private renderOptions: RenderOptions; - _waiting: boolean; - prompt: string; - - onSend: () => void; - channel: Channel; - messageId = ""; - - constructor(onSend: () => void, inputMode: InputMode) { - this._inputMode = inputMode; - this.channel = makeChannel()!; // by default we try to use Atomics - this.onSend = onSend; - this._waiting = false; - this.prompt = ""; + private waiting: boolean; + private prompt: string; + + private sendInput: (input: string) => void; + + constructor(sendInput: (input: string) => void) { this.inputHandlers = this.buildInputHandlerMap(); + this.inputMode = InputMode.Interactive; + this.sendInput = sendInput; + this.waiting = false; + this.prompt = ""; this.renderOptions = {} as RenderOptions; + BackendManager.subscribe(BackendEventType.Start, () => this.onRunStart()); + BackendManager.subscribe(BackendEventType.End, () => this.onRunEnd()); + BackendManager.subscribe(BackendEventType.Input, e => this.onInputRequest(e)); } private buildInputHandlerMap(): Map { const interactiveInputHandler: UserInputHandler = - new InteractiveInputHandler(() => this.sendLine(), INPUT_TA_ID, SEND_INPUT_BTN_ID); + new InteractiveInputHandler(() => this.onUserInput()); const batchInputHandler: UserInputHandler = - new BatchInputHandler(() => this.sendLine(), INPUT_TA_ID); + new BatchInputHandler(() => this.onUserInput()); return new Map([ [InputMode.Interactive, interactiveInputHandler], [InputMode.Batch, batchInputHandler] ]); } - get inputMode(): InputMode { - return this._inputMode; + getInputMode(): InputMode { + return this.inputMode; } - set inputMode(inputMode: InputMode) { + setInputMode(inputMode: InputMode): void { this.inputHandler.onToggle(false); - this._inputMode = inputMode; + this.inputMode = inputMode; this.render(this.renderOptions); this.inputHandler.onToggle(true); } @@ -76,60 +70,61 @@ export class InputManager implements RunListener { render(options: RenderOptions): void { this.renderOptions = options; + let switchMode = ""; const otherMode = this.inputMode === InputMode.Interactive ? InputMode.Batch : InputMode.Interactive; + switchMode = ` + ${t(`Papyros.input_modes.switch_to_${otherMode}`)} + `; + renderWithOptions(options, `
- - ${t(`Papyros.input_modes.switch_to_${otherMode}`)} -`); - addListener(SWITCH_INPUT_MODE_A_ID, im => this.inputMode = im, +${switchMode}`); + addListener(SWITCH_INPUT_MODE_A_ID, im => this.setInputMode(im), "click", "data-value"); + this.inputHandler.render({ parentElementId: USER_INPUT_WRAPPER_ID }); - this.inputHandler.waitWithPrompt(this._waiting, this.prompt); + this.inputHandler.waitWithPrompt(this.waiting, this.prompt); } - set waiting(waiting: boolean) { - this._waiting = waiting; - this.inputHandler.waitWithPrompt(waiting, this.prompt); + waitWithPrompt(waiting: boolean, prompt=""): void { + this.waiting = waiting; + this.prompt = prompt; + this.inputHandler.waitWithPrompt(this.waiting, this.prompt); } - async sendLine(): Promise { + async onUserInput(): Promise { if (this.inputHandler.hasNext()) { const line = this.inputHandler.next(); papyrosLog(LogType.Debug, "Sending input to user: " + line); - await writeMessage(this.channel, line, this.messageId); - this.waiting = false; - this.onSend(); + this.sendInput(line); + this.waitWithPrompt(false); } else { papyrosLog(LogType.Debug, "Had no input to send, still waiting!"); - this.waiting = true; + this.waitWithPrompt(true, this.prompt); } } /** * Asynchronously handle an input request by prompting the user for input - * @param {PapyrosEvent} e Event containing the input data + * @param {BackendEvent} e Event containing the input data * @return {Promise} Promise of handling the request */ - async onInput(e: PapyrosEvent): Promise { + async onInputRequest(e: BackendEvent): Promise { papyrosLog(LogType.Debug, "Handling input request in Papyros"); - const data = parseData(e.data, e.contentType) as InputData; - this.messageId = data.messageId; - this.prompt = data.prompt; - return this.sendLine(); + this.prompt = e.data; + return await this.onUserInput(); } onRunStart(): void { - this.waiting = false; + this.waitWithPrompt(false); this.inputHandler.onRunStart(); } onRunEnd(): void { - this.prompt = ""; this.inputHandler.onRunEnd(); - this.waiting = false; + this.waitWithPrompt(false); } } diff --git a/src/InputServiceWorker.ts b/src/InputServiceWorker.ts index d7b864c1..11609e1f 100644 --- a/src/InputServiceWorker.ts +++ b/src/InputServiceWorker.ts @@ -6,14 +6,15 @@ import { InputWorker } from "./workers/input/InputWorker"; // Strip away the filename of the script to obtain the scope let domain = location.href; -domain = domain.slice(0, domain.lastIndexOf("/")+1); +domain = domain.slice(0, domain.lastIndexOf("/") + 1); const inputHandler = new InputWorker(domain); addEventListener("fetch", async function (event: FetchEvent) { if (!await inputHandler.handleInputRequest(event)) { // Not a Papyros-specific request // Fetch as we would handle a normal request - return; // Default to nothing, browser will handle fetch itself + // Default action is to let browser handle it by not responding here + return; } }); // Prevent needing to reload page to have working input diff --git a/src/Library.ts b/src/Library.ts index aae23b6d..266781de 100644 --- a/src/Library.ts +++ b/src/Library.ts @@ -1,21 +1,19 @@ +import { BackendEvent } from "./BackendEvent"; import { CodeEditor } from "./CodeEditor"; import { InputManager, InputMode } from "./InputManager"; import { OutputManager } from "./OutputManager"; import { Papyros } from "./Papyros"; -import { PapyrosEvent } from "./PapyrosEvent"; -import { RunStateManager, RunState } from "./RunStateManager"; -import { InputWorker } from "./workers/input/InputWorker"; +import { CodeRunner, RunState } from "./CodeRunner"; export * from "./ProgrammingLanguage"; -export type { PapyrosEvent }; +export type { BackendEvent }; export { Papyros, CodeEditor, RunState, - RunStateManager, + CodeRunner, InputManager, InputMode, OutputManager, - InputWorker }; diff --git a/src/OutputManager.ts b/src/OutputManager.ts index 39982b7a..d0d80ea5 100644 --- a/src/OutputManager.ts +++ b/src/OutputManager.ts @@ -1,11 +1,12 @@ import escapeHTML from "escape-html"; -import { PapyrosEvent } from "./PapyrosEvent"; -import { RunListener } from "./RunListener"; +import { BackendEvent, BackendEventType } from "./BackendEvent"; +import { BackendManager } from "./BackendManager"; import { inCircle } from "./util/HTMLShapes"; import { getElement, parseData, RenderOptions, renderWithOptions, t } from "./util/Util"; +import { LogType, papyrosLog } from "./util/Logging"; /** * Shape of Error objects that are easy to interpret @@ -40,9 +41,16 @@ export interface FriendlyError { /** * Component for displaying code output or errors to the user */ -export class OutputManager implements RunListener { +export class OutputManager { // Store options to allow re-rendering - options: RenderOptions = { parentElementId: "" }; + options: RenderOptions; + + constructor() { + BackendManager.subscribe(BackendEventType.Start, () => this.reset()); + BackendManager.subscribe(BackendEventType.Output, e => this.showOutput(e)); + BackendManager.subscribe(BackendEventType.Error, e => this.showError(e)); + this.options = { parentElementId: "" }; + } /** * Retrieve the parent element containing all output parts @@ -78,12 +86,12 @@ export class OutputManager implements RunListener { /** * Display output to the user, based on its content type - * @param {PapyrosEvent} output Event containing the output data + * @param {BackendEvent} output Event containing the output data */ - showOutput(output: PapyrosEvent): void { + showOutput(output: BackendEvent): void { const data = parseData(output.data, output.contentType); - if (output.contentType === "img/png;base64") { - this.renderNextElement(``); + if (output.contentType && output.contentType.startsWith("img")) { + this.renderNextElement(``); } else { this.renderNextElement(this.spanify(data, false)); } @@ -91,11 +99,12 @@ export class OutputManager implements RunListener { /** * Display an error to the user - * @param {PapyrosEvent} error Event containing the error data + * @param {BackendEvent} error Event containing the error data */ - showError(error: PapyrosEvent): void { + showError(error: BackendEvent): void { let errorHTML = ""; const errorData = parseData(error.data, error.contentType); + papyrosLog(LogType.Debug, "Showing error: ", errorData); if (typeof (errorData) === "string") { errorHTML = this.spanify(errorData, false, "text-red-500"); } else { @@ -144,12 +153,4 @@ export class OutputManager implements RunListener { reset(): void { this.render(this.options); } - - onRunStart(): void { - this.reset(); - } - - onRunEnd(): void { - // currently empty - } } diff --git a/src/Papyros.css b/src/Papyros.css index 6310c420..179c2ed0 100644 --- a/src/Papyros.css +++ b/src/Papyros.css @@ -15,7 +15,8 @@ white-space: pre; overflow: scroll; resize: both; - max-width: none !important; /* override default max width of 300px */ + /* override default max width of 300px to allow resizing */ + max-width: 800px !important; } /* Add placeholder to an empty element */ diff --git a/src/Papyros.ts b/src/Papyros.ts index 341f91a3..bf73d539 100644 --- a/src/Papyros.ts +++ b/src/Papyros.ts @@ -1,16 +1,11 @@ /* eslint-disable max-len */ import "./Papyros.css"; -import { proxy, Remote } from "comlink"; import I18n from "i18n-js"; -import { Backend } from "./Backend"; -import { startBackend, stopBackend } from "./BackendManager"; -import { CodeEditor } from "./CodeEditor"; import { EDITOR_WRAPPER_ID, PROGRAMMING_LANGUAGE_SELECT_ID, OUTPUT_TA_ID, LOCALE_SELECT_ID, INPUT_AREA_WRAPPER_ID, EXAMPLE_SELECT_ID, PANEL_WRAPPER_ID } from "./Constants"; -import { InputManager, InputMode } from "./InputManager"; -import { PapyrosEvent } from "./PapyrosEvent"; +import { InputMode } from "./InputManager"; import { ProgrammingLanguage } from "./ProgrammingLanguage"; import { LogType, papyrosLog } from "./util/Logging"; import { @@ -19,39 +14,17 @@ import { RenderOptions, renderWithOptions, addListener, ButtonOptions, getElement } from "./util/Util"; -import { RunState, RunStateManager } from "./RunStateManager"; +import { RunState, CodeRunner } from "./CodeRunner"; import { getCodeForExample, getExampleNames } from "./examples/Examples"; import { OutputManager } from "./OutputManager"; import { makeChannel } from "sync-message"; -import { RunListener } from "./RunListener"; +import { BackendManager } from "./BackendManager"; const LANGUAGE_MAP = new Map([ ["python", ProgrammingLanguage.Python], ["javascript", ProgrammingLanguage.JavaScript] ]); -/** - * Groups values related to running code - */ -interface PapyrosCodeState { - /** - * The currently used programming language - */ - programmingLanguage: ProgrammingLanguage; - /** - * The editor in which the code is written - */ - editor: CodeEditor; - /** - * The backend that executes the code asynchronously - */ - backend: Remote; - /** - * The identifier for the current run - */ - runId: number; -} - /** * Configuration options for this instance of Papyros */ @@ -109,25 +82,13 @@ export class Papyros { */ config: PapyrosConfig; /** - * Component to manage and visualize the state of the program - */ - stateManager: RunStateManager; - /** - * Component to request and handle input from the user + * Component to run code entered by the user */ - inputManager: InputManager; + codeRunner: CodeRunner; /** * Component to handle output generated by the user's code */ outputManager: OutputManager; - /** - * Groups all internal properties related to running code - */ - codeState: PapyrosCodeState; - /** - * Listeners to changes in the running state - */ - runListeners: Array; /** * Whether this instance has been launched */ @@ -139,53 +100,21 @@ export class Papyros { */ constructor(config: PapyrosConfig) { this.launched = false; - this.runListeners = []; this.config = config; // Load translations as other components depend on them loadTranslations(); I18n.locale = config.locale; const { programmingLanguage } = this.config; - this.codeState = { - programmingLanguage: programmingLanguage, - editor: new CodeEditor( - programmingLanguage, - t("Papyros.code_placeholder", { programmingLanguage })), - backend: {} as Remote, - runId: 0 - }; this.outputManager = new OutputManager(); - this.stateManager = new RunStateManager(() => this.runCode(), () => this.stop()); - this.inputManager = new InputManager(() => this.stateManager.setState(RunState.Running), config.inputMode); - this.addRunListener(this.inputManager); - this.addRunListener(this.outputManager); + this.codeRunner = new CodeRunner(programmingLanguage); } /** - * Register a listener to be notified when code runs start or end - * @param {RunListener} listener The new listener + * @return {RunState} The current state of the user's code */ - addRunListener(listener: RunListener): void { - this.runListeners.push(listener); - } - - /** - * Inform the listeners about the current run - * @param {boolean} start Whether the run started or ended - */ - private notifyListeners(start: boolean): void { - if (start) { - this.runListeners.forEach(l => l.onRunStart()); - } else { - this.runListeners.forEach(l => l.onRunEnd()); - } - } - - /** - * Getter for the current state of the program - */ - get state(): RunState { - return this.stateManager.state; + getState(): RunState { + return this.codeRunner.getState(); } /** @@ -194,10 +123,7 @@ export class Papyros { */ async launch(): Promise { if (!this.launched) { - const start = new Date().getTime(); - await this.startBackend(); - papyrosLog(LogType.Important, `Finished loading backend after ${new Date().getTime() - start} ms`); - this.codeState.editor.focus(); + await this.codeRunner.start(); } return this; } @@ -207,42 +133,21 @@ export class Papyros { * @param {ProgrammingLanguage} programmingLanguage The language to use */ async setProgrammingLanguage(programmingLanguage: ProgrammingLanguage): Promise { - if (this.codeState.programmingLanguage !== programmingLanguage) { // Expensive, so ensure it is needed - stopBackend(this.codeState.backend); - this.codeState.programmingLanguage = programmingLanguage; - await this.startBackend(); - } + await this.codeRunner.setProgrammingLanguage(programmingLanguage); } /** * @param {string} code The code to use in the editor */ setCode(code: string): void { - this.codeState.editor.setCode(code); + this.codeRunner.editor.setCode(code); } /** * @return {string} The currently written code */ getCode(): string { - return this.codeState.editor.getCode(); - } - - /** - * Start up the backend for the current programming language - */ - private async startBackend(): Promise { - const programmingLanguage = this.codeState.programmingLanguage; - this.stateManager.setState(RunState.Loading); - const backend = startBackend(programmingLanguage); - // Allow passing messages between worker and main thread - await backend.launch(proxy(e => this.onMessage(e)), this.inputManager.channel); - this.codeState.backend = backend; - this.codeState.editor.setLanguage(programmingLanguage, - async context => await this.codeState.backend.autocomplete(Backend.convertCompletionContext(context)), - t("Papyros.code_placeholder", { programmingLanguage }) - ); - this.stateManager.setState(RunState.Ready); + return this.codeRunner.editor.getCode(); } /** @@ -256,118 +161,22 @@ export class Papyros { * This allows using SharedArrayBuffers without configuring the HTTP headers yourself * @return {Promise} Promise of configuring input */ - async configureInput(serviceWorkerRoot?: string, serviceWorkerName?: string, - allowReload = false): Promise { - const RELOAD_STORAGE_KEY = "__papyros_reloading"; - if (allowReload && window.localStorage.getItem(RELOAD_STORAGE_KEY)) { - // We are the result of the page reload, so we can start - window.localStorage.removeItem(RELOAD_STORAGE_KEY); - return true; - } else { - if (typeof SharedArrayBuffer === "undefined") { - papyrosLog(LogType.Important, "SharedArrayBuffers are not available. "); - if (!serviceWorkerRoot || !serviceWorkerName || !("serviceWorker" in navigator)) { - papyrosLog(LogType.Important, "Unable to register service worker. Please specify all required parameters and ensure service workers are supported."); - return false; - } - // Ensure there is a slash at the end to allow the script to be resolved - const rootWithSlash = serviceWorkerRoot.endsWith("/") ? serviceWorkerRoot : serviceWorkerRoot + "/"; - const serviceWorkerUrl = rootWithSlash + serviceWorkerName; - papyrosLog(LogType.Important, `Registering service worker: ${serviceWorkerUrl}`); - await window.navigator.serviceWorker.register(serviceWorkerUrl); - this.inputManager.channel = makeChannel({ serviceWorker: { scope: rootWithSlash } })!; - if (allowReload) { // Store that we are reloading, to prevent the next load from doing all this again - window.localStorage.setItem(RELOAD_STORAGE_KEY, RELOAD_STORAGE_KEY); - // service worker adds new headers that may allow SharedArrayBuffers to be used - window.location.reload(); - } - return true; - } else { - return true; + async configureInput(serviceWorkerRoot?: string, serviceWorkerName?: string) + : Promise { + if (typeof SharedArrayBuffer === "undefined") { + papyrosLog(LogType.Important, "SharedArrayBuffers are not available. "); + if (!serviceWorkerRoot || !serviceWorkerName || !("serviceWorker" in navigator)) { + papyrosLog(LogType.Important, "Unable to register service worker. Please specify all required parameters and ensure service workers are supported."); + return false; } + // Ensure there is a slash at the end to allow the script to be resolved + const rootWithSlash = serviceWorkerRoot.endsWith("/") ? serviceWorkerRoot : serviceWorkerRoot + "/"; + const serviceWorkerUrl = rootWithSlash + serviceWorkerName; + papyrosLog(LogType.Important, `Registering service worker: ${serviceWorkerUrl}`); + await window.navigator.serviceWorker.register(serviceWorkerUrl); + BackendManager.channel = makeChannel({ serviceWorker: { scope: rootWithSlash } })!; } - } - - /** - * Process PapyrosEvents with type="error" - * @param {PapyrosEvent} e The error-event - */ - private onError(e: PapyrosEvent): void { - papyrosLog(LogType.Debug, "Got error in Papyros: ", e); - this.outputManager.showError(e); - } - /** - * Process PapyrosEvents with type="input" - * @param {PapyrosEvent} e The input-event - */ - private async onInput(e: PapyrosEvent): Promise { - papyrosLog(LogType.Debug, "Received onInput event in Papyros: ", e); - this.stateManager.setState(RunState.AwaitingInput); - await this.inputManager.onInput(e); - } - - /** - * Generic handler function to pass PapyrosEvents to the relevant method - * @param {PapyrosEvent} e The event ro process - */ - onMessage(e: PapyrosEvent): void { - papyrosLog(LogType.Debug, "received event in onMessage", e); - if (e.runId === this.codeState.runId) { // Only process relevant messages - if (e.type === "output") { - this.outputManager.showOutput(e); - } else if (e.type === "input") { - this.onInput(e); - } else if (e.type === "error") { - this.onError(e); - } - } else { - papyrosLog(LogType.Debug, "Received event with outdated runId: ", e); - } - } - - /** - * Run the code that is currently present in the editor - * @return {Promise} Promise of running the code - */ - async runCode(): Promise { - if (this.state !== RunState.Ready) { - papyrosLog(LogType.Error, `Run code called from invalid state: ${this.state}`); - return; - } - // Setup pre-run - this.codeState.runId += 1; - this.stateManager.setState(RunState.Running); - this.notifyListeners(true); - - papyrosLog(LogType.Debug, "Running code in Papyros, sending to backend"); - const start = new Date().getTime(); - try { - await this.codeState.backend.runCode(this.getCode(), this.codeState.runId); - } catch (error: any) { - this.onError(error); - } finally { - const end = new Date().getTime(); - this.stateManager.setState(RunState.Ready, t("Papyros.finished", { time: (end - start) / 1000 })); - this.notifyListeners(false); - } - } - - /** - * Interrupt the currently running code - * @return {Promise} Promise of stopping - */ - async stop(): Promise { - if (![RunState.Running, RunState.AwaitingInput].includes(this.state)) { - papyrosLog(LogType.Error, `Stop called from invalid state: ${this.state}`); - return; - } - papyrosLog(LogType.Debug, "Stopping backend!"); - this.codeState.runId += 1; // ignore messages coming from last run - this.stateManager.setState(RunState.Stopping); - this.notifyListeners(false); - // Since we use workers, the old one must be entirely replaced to interrupt it - stopBackend(this.codeState.backend); - return this.startBackend(); + return true; } /** @@ -447,22 +256,20 @@ export class Papyros { removeSelection(EXAMPLE_SELECT_ID); // Modify search query params without reloading page history.pushState(null, "", `?locale=${I18n.locale}&language=${pl}`); - } + }, "change", "value" ); addListener(LOCALE_SELECT_ID, locale => { - document.location.href = `?locale=${locale}&language=${this.codeState.programmingLanguage}`; - }); + document.location.href = `?locale=${locale}&language=${this.codeRunner.getProgrammingLanguage()}`; + }, "change", "value"); addListener(EXAMPLE_SELECT_ID, name => { - const code = getCodeForExample(this.codeState.programmingLanguage, name); + const code = getCodeForExample(this.codeRunner.getProgrammingLanguage(), name); this.setCode(code); - }, "input"); + }, "change", "value"); // Ensure there is no initial selection removeSelection(EXAMPLE_SELECT_ID); } - - this.inputManager.render(renderOptions.inputOptions); - const runStatePanel = this.stateManager.render(renderOptions.statusPanelOptions); - this.codeState.editor.render(renderOptions.codeEditorOptions, runStatePanel); + this.codeRunner.render(renderOptions.statusPanelOptions, + renderOptions.inputOptions, renderOptions.codeEditorOptions); this.outputManager.render(renderOptions.outputOptions); } @@ -472,7 +279,7 @@ export class Papyros { * @param {function} onClick Listener for click events on the button */ addButton(options: ButtonOptions, onClick: () => void): void { - this.stateManager.addButton(options, onClick); + this.codeRunner.addButton(options, onClick); } /** diff --git a/src/PapyrosEvent.ts b/src/PapyrosEvent.ts deleted file mode 100644 index 7b97eae9..00000000 --- a/src/PapyrosEvent.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Interface for events used for communication between threads - */ -export interface PapyrosEvent { - /** - * The type of action generating this event - */ - type: "input" | "output" | "success" | "error"; - /** - * The identifier for the run this message is associated with - * This allows discarding outdated events that were delayed - */ - runId: number; - /** - * The actual data stored in this event - */ - data: string; - /** - * The format used for the data to help with parsing - */ - contentType: string; -} diff --git a/src/RunStateManager.ts b/src/RunStateManager.ts deleted file mode 100644 index 61e3039b..00000000 --- a/src/RunStateManager.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { APPLICATION_STATE_TEXT_ID, RUN_BTN_ID, STATE_SPINNER_ID, STOP_BTN_ID } from "./Constants"; -import { svgCircle } from "./util/HTMLShapes"; -import { - addListener, ButtonOptions, renderButton, - RenderOptions, renderWithOptions, getElement, - t -} from "./util/Util"; - -interface DynamicButton { - id: string; - buttonHTML: string; - onClick: () => void; -} - -/** - * Enum representing the possible states while processing code - */ -export enum RunState { - Loading = "loading", - Running = "running", - AwaitingInput = "awaiting_input", - Stopping = "stopping", - Ready = "ready" -} - -/** - * Helper component to manage and visualize the current RunState - */ -export class RunStateManager { - /** - * Current state of the program - */ - state: RunState; - /** - * Buttons managed by this component - */ - buttons: Array; - - /** - * Construct a new RunStateManager with the given listeners - * @param {function} onRunClicked Callback for when the run button is clicked - * @param {function} onStopClicked Callback for when the stop button is clicked - */ - constructor(onRunClicked: () => void, onStopClicked: () => void) { - this.buttons = []; - this.addButton({ - id: RUN_BTN_ID, - buttonText: t("Papyros.run"), - extraClasses: "text-white bg-blue-500" - }, onRunClicked); - this.addButton({ - id: STOP_BTN_ID, - buttonText: t("Papyros.stop"), - extraClasses: "text-white bg-red-500" - }, onStopClicked); - this.state = RunState.Ready; - } - - /** - * Get the button to run the code - */ - get runButton(): HTMLButtonElement { - return getElement(RUN_BTN_ID); - } - - /** - * Get the button to interrupt the code - */ - get stopButton(): HTMLButtonElement { - return getElement(STOP_BTN_ID); - } - - /** - * Show or hide the spinning circle, representing a running animation - * @param {boolean} show Whether to show the spinner - */ - showSpinner(show: boolean): void { - getElement(STATE_SPINNER_ID).style.display = show ? "" : "none"; - } - - - /** - * Show the current state of the program to the user - * @param {RunState} state The current state of the run - * @param {string} message Optional message to indicate the state - */ - setState(state: RunState, message?: string): void { - this.state = state; - this.stopButton.disabled = [RunState.Ready, RunState.Loading].includes(state); - if (state === RunState.Ready) { - this.showSpinner(false); - this.runButton.disabled = false; - } else { - this.showSpinner(true); - this.runButton.disabled = true; - } - getElement(APPLICATION_STATE_TEXT_ID).innerText = - message || t(`Papyros.states.${state}`); - } - - /** - * Add a button to display to the user - * @param {ButtonOptions} options Options for rendering the button - * @param {function} onClick Listener for click events on the button - */ - addButton(options: ButtonOptions, onClick: () => void): void { - this.buttons.push({ - id: options.id, - buttonHTML: renderButton(options), - onClick: onClick - }); - } - - /** - * Render the RunStateManager with the given options - * @param {RenderOptions} options Options for rendering - * @return {HTMLElement} The rendered RunStateManager - */ - render(options: RenderOptions): HTMLElement { - const rendered = renderWithOptions(options, ` -
-
- ${this.buttons.map(b => b.buttonHTML).join("\n")} -
-
-
- ${svgCircle(STATE_SPINNER_ID, "red")} -
-
`); - // Buttons are freshly added to the DOM, so attach listeners now - this.buttons.forEach(b => addListener(b.id, b.onClick, "click")); - return rendered; - } -} diff --git a/src/examples/PythonExamples.ts b/src/examples/PythonExamples.ts index 7cda4061..fa548f66 100644 --- a/src/examples/PythonExamples.ts +++ b/src/examples/PythonExamples.ts @@ -128,5 +128,10 @@ x = np.linspace(0, 10, 1000) plt.plot(x, np.sin(x)); plt.show() +`, + "Sleep": `import time +print("See you in a few seconds!") +time.sleep(3) +print("Good to see you again!") ` }; diff --git a/src/input/BatchInputHandler.ts b/src/input/BatchInputHandler.ts index ba6c9958..2697e3ed 100644 --- a/src/input/BatchInputHandler.ts +++ b/src/input/BatchInputHandler.ts @@ -1,3 +1,4 @@ +import { INPUT_TA_ID } from "../Constants"; import { InputMode } from "../InputManager"; import { RenderOptions, renderWithOptions } from "../util/Util"; import { UserInputHandler } from "./UserInputHandler"; @@ -15,16 +16,15 @@ export class BatchInputHandler extends UserInputHandler { /** * Construct a new BatchInputHandler - * @param {function()} onInput Callback for when the user has entered a value - * @param {string} inputAreaId HTML identifier for the used HTML input field + * @param {function()} inputCallback Callback for when the user has entered a value */ - constructor(onInput: () => void, inputAreaId: string) { - super(onInput, inputAreaId); + constructor(inputCallback: () => void) { + super(inputCallback); this.lineNr = 0; this.previousInput = ""; } - onToggle(active: boolean): void { + override onToggle(active: boolean): void { if (active) { this.inputArea.value = this.previousInput; } else { @@ -32,14 +32,14 @@ export class BatchInputHandler extends UserInputHandler { } } - getInputMode(): InputMode { + override getInputMode(): InputMode { return InputMode.Batch; } /** * Retrieve the lines of input that the user has given so far * @return {Array} The entered lines */ - private get lines(): Array { + protected get lines(): Array { const l = this.inputArea.value.split("\n"); if (!l[l.length - 1]) { // last line is empty l.splice(l.length - 1); // do not consider it valid input @@ -47,27 +47,27 @@ export class BatchInputHandler extends UserInputHandler { return l; } - hasNext(): boolean { + override hasNext(): boolean { return this.lineNr < this.lines.length; } - next(): string { + override next(): string { const nextLine = this.lines[this.lineNr]; this.lineNr += 1; return nextLine; } - onRunStart(): void { + override onRunStart(): void { this.lineNr = 0; } - onRunEnd(): void { + override onRunEnd(): void { // Intentionally empty } render(options: RenderOptions): HTMLElement { const rendered = renderWithOptions(options, ` -`); @@ -77,7 +77,7 @@ focus:outline-none focus:ring-1 focus:ring-blue-500" rows="5"> if (this.lines.length < this.lineNr) { this.lineNr = this.lines.length - 1; } - this.onInput(); + this.inputCallback(); } }); return rendered; diff --git a/src/input/InteractiveInputHandler.ts b/src/input/InteractiveInputHandler.ts index 988e7353..83aed0b5 100644 --- a/src/input/InteractiveInputHandler.ts +++ b/src/input/InteractiveInputHandler.ts @@ -1,46 +1,34 @@ +import { INPUT_TA_ID, SEND_INPUT_BTN_ID } from "../Constants"; import { InputMode } from "../InputManager"; -import { getElement, RenderOptions, renderWithOptions, t } from "../util/Util"; +import { addListener, getElement, RenderOptions, renderWithOptions, t } from "../util/Util"; import { UserInputHandler } from "./UserInputHandler"; +/** + * Input handler that takes input from the user in an interactive fashion + */ export class InteractiveInputHandler extends UserInputHandler { - /** - * HTML identifier for the used HTML button - */ - private sendButtonId: string; - /** - * Construct a new InteractiveInputHandler - * @param {function()} onInput Callback for when the user has entered a value - * @param {string} inputAreaId HTML identifier for the used HTML input field - * @param {string} sendButtonId HTML identifier for the used HTML button - */ - constructor(onInput: () => void, inputAreaId: string, - sendButtonId: string) { - super(onInput, inputAreaId); - this.sendButtonId = sendButtonId; - } - /** * Retrieve the button that users can click to send their input */ get sendButton(): HTMLButtonElement { - return getElement(this.sendButtonId); + return getElement(SEND_INPUT_BTN_ID); } - getInputMode(): InputMode { + override getInputMode(): InputMode { return InputMode.Interactive; } - hasNext(): boolean { + override hasNext(): boolean { return this.waiting; // Allow sending empty lines when the user does this explicitly } - next(): string { + override next(): string { const value = this.inputArea.value; - this.inputArea.value = ""; + this.reset(); return value; } - waitWithPrompt(waiting: boolean, prompt?: string): void { + override waitWithPrompt(waiting: boolean, prompt?: string): void { super.waitWithPrompt(waiting, prompt); this.sendButton.disabled = !waiting; this.inputArea.disabled = !waiting; @@ -51,36 +39,35 @@ export class InteractiveInputHandler extends UserInputHandler { } } - onToggle(): void { + override onToggle(): void { this.reset(); } - onRunStart(): void { + override onRunStart(): void { this.reset(); } - onRunEnd(): void { + override onRunEnd(): void { // Intentionally empty } render(options: RenderOptions): HTMLElement { const rendered = renderWithOptions(options, `
- -
`); - getElement(this.sendButtonId) - .addEventListener("click", () => this.onInput()); + addListener(SEND_INPUT_BTN_ID, () => this.inputCallback(), "click"); this.inputArea.addEventListener("keydown", (ev: KeyboardEvent) => { - if (this.waiting && ev.key.toLowerCase() === "enter") { - this.onInput(); + if (this.waiting && ev.key && ev.key.toLowerCase() === "enter") { + this.inputCallback(); } }); return rendered; diff --git a/src/input/UserInputHandler.ts b/src/input/UserInputHandler.ts index 594b36d4..b8d3e852 100644 --- a/src/input/UserInputHandler.ts +++ b/src/input/UserInputHandler.ts @@ -1,34 +1,27 @@ +import { INPUT_TA_ID } from "../Constants"; import { InputMode } from "../InputManager"; -import { RunListener } from "../RunListener"; import { getElement, RenderOptions, t } from "../util/Util"; /** * Base class for components that handle input from the user */ -export abstract class UserInputHandler implements RunListener { +export abstract class UserInputHandler { /** * Whether we are waiting for the user to input data */ protected waiting: boolean; - /** - * Callback for when the user has entered a value - */ - protected onInput: () => void; - /** - * HTML identifier for the used HTML input field - */ - protected inputAreaId: string; + + protected inputCallback: () => void; /** * Construct a new UserInputHandler - * @param {function()} onInput Callback for when the user has entered a value - * @param {string} inputAreaId HTML identifier for the used HTML input field + * @param {function()} inputCallback Callback for when the user has entered a value */ - constructor(onInput: () => void, inputAreaId: string) { + constructor(inputCallback: () => void) { this.waiting = false; - this.onInput = onInput; - this.inputAreaId = inputAreaId; + this.inputCallback = inputCallback; } + /** * Whether this handler has input ready */ @@ -66,7 +59,7 @@ export abstract class UserInputHandler implements RunListener { * Retrieve the HTMLInputElement for this InputHandler */ get inputArea(): HTMLInputElement { - return getElement(this.inputAreaId); + return getElement(INPUT_TA_ID); } /** @@ -89,7 +82,7 @@ export abstract class UserInputHandler implements RunListener { } /** - * Helper method to reset internal state when needed + * Helper method to reset internal state */ protected reset(): void { this.inputArea.value = ""; diff --git a/src/util/Util.ts b/src/util/Util.ts index f44b965d..9ec1e5a5 100644 --- a/src/util/Util.ts +++ b/src/util/Util.ts @@ -192,11 +192,14 @@ export function renderWithOptions( /** * Parse the data contained within a PapyrosEvent using its contentType * Supported content types are: text/plain, text/json, img/png;base64 - * @param {unknown} data The data to parse + * @param {string} data The data to parse * @param {string} contentType The content type of the data * @return {any} The parsed data */ -export function parseData(data: unknown, contentType: string): any { +export function parseData(data: string, contentType?: string): any { + if (!contentType) { + return data; + } const [baseType, specificType] = contentType.split("/"); switch (baseType) { case "text": { @@ -205,7 +208,13 @@ export function parseData(data: unknown, contentType: string): any { return data; } case "json": { - return JSON.parse(data as string); + return JSON.parse(data); + } + case "integer": { + return parseInt(data); + } + case "float": { + return parseFloat(data); } } break; @@ -218,6 +227,10 @@ export function parseData(data: unknown, contentType: string): any { } break; } + case "application": { + // Content such as application/json does not need parsing as it is in the correct shape + return data; + } } papyrosLog(LogType.Important, `Unhandled content type: ${contentType}`); return data; diff --git a/src/workers/javascript/JavaScriptWorker.worker.ts b/src/workers/javascript/JavaScriptWorker.worker.ts index 0aa6c50a..7683532e 100644 --- a/src/workers/javascript/JavaScriptWorker.worker.ts +++ b/src/workers/javascript/JavaScriptWorker.worker.ts @@ -1,13 +1,15 @@ -import { expose } from "comlink"; +import * as Comlink from "comlink"; import { Backend, WorkerAutocompleteContext } from "../../Backend"; import { CompletionResult } from "@codemirror/autocomplete"; import { javascriptLanguage } from "@codemirror/lang-javascript"; +import { BackendEventType } from "../../BackendEvent"; +import { SyncExtras } from "comsync"; /** * Implementation of a JavaScript backend for Papyros * by using eval and overriding some builtins */ -class JavaScriptWorker extends Backend { +class JavaScriptWorker extends Backend { /** * Convert varargs to a string, similar to how the console does it * @param {any[]} args The values to join into a string @@ -41,10 +43,9 @@ class JavaScriptWorker extends Backend { */ private prompt(text = ""): string { return this.onEvent({ - type: "input", - data: JavaScriptWorker.stringify({ prompt: text }), - contentType: "text/json", - runId: this.runId + type: BackendEventType.Input, + data: text, + contentType: "text/plain" }); } @@ -54,9 +55,8 @@ class JavaScriptWorker extends Backend { */ private consoleLog(...args: any[]): void { this.onEvent({ - type: "output", + type: BackendEventType.Output, data: JavaScriptWorker.stringify(...args) + "\n", - runId: this.runId, contentType: "text/plain" }); } @@ -67,10 +67,9 @@ class JavaScriptWorker extends Backend { */ private consoleError(...args: any[]): void { this.onEvent({ - type: "error", + type: BackendEventType.Error, data: JavaScriptWorker.stringify(...args) + "\n", - contentType: "text/plain", - runId: this.runId + contentType: "text/plain" }); } @@ -121,7 +120,8 @@ class JavaScriptWorker extends Backend { return ret; } - override _runCodeInternal(code: string): Promise { + override runCode(extras: SyncExtras, code: string): Promise { + this.extras = extras; // Builtins to store before execution and restore afterwards // Workers do not have access to prompt const oldContent = { @@ -144,14 +144,13 @@ class JavaScriptWorker extends Backend { } catch (error: any) { // try to create a friendly traceback Error.captureStackTrace(error); return Promise.resolve(this.onEvent({ - type: "error", - runId: this.runId, - contentType: "text/json", - data: JSON.stringify({ + type: BackendEventType.Error, + contentType: "application/json", + data: { name: error.constructor.name, what: error.message, traceback: error.stack - }) + } })); } finally { // restore the old builtins new Function("ctx", @@ -163,5 +162,6 @@ class JavaScriptWorker extends Backend { // Default export to be recognized as a TS module export default {} as any; -// Expose handles the actual export -expose(new JavaScriptWorker()); + +// Comlink and Comsync handle the actual export +Comlink.expose(new JavaScriptWorker()); diff --git a/src/workers/python/Pyodide.ts b/src/workers/python/Pyodide.ts deleted file mode 100644 index 7f0ce5f1..00000000 --- a/src/workers/python/Pyodide.ts +++ /dev/null @@ -1,30 +0,0 @@ -/** - * TypeScript interface for used Pyodide methods - */ -export interface Pyodide { - /** - * Runs a string of Python code from JavaScript. - */ - runPython: (code: string, globals?: any) => any; - /** - * Runs Python code using PyCF_ALLOW_TOP_LEVEL_AWAIT. - */ - runPythonAsync: (code: string) => Promise; - /** - * Inspect a Python code chunk and use pyodide.loadPackage() - * to install any known packages that the code chunk imports. - */ - loadPackagesFromImports: (code: string) => Promise; - /** - * Load a package or a list of packages over the network. - */ - loadPackage: (names: string | string[]) => Promise; - /** - * An alias to the global Python namespace. - */ - globals: Map; -} -// Version and locations of code required to run Pyodide in the browser -const PYODIDE_VERSION = "v0.19.0"; -export const PYODIDE_INDEX_URL = `https://cdn.jsdelivr.net/pyodide/${PYODIDE_VERSION}/full/`; -export const PYODIDE_JS_URL = PYODIDE_INDEX_URL + "pyodide.js"; diff --git a/src/workers/python/PythonWorker.worker.ts b/src/workers/python/PythonWorker.worker.ts index 7e9afbfc..908be3a0 100644 --- a/src/workers/python/PythonWorker.worker.ts +++ b/src/workers/python/PythonWorker.worker.ts @@ -1,86 +1,71 @@ -import { expose } from "comlink"; +import * as Comlink from "comlink"; import { Backend, WorkerAutocompleteContext } from "../../Backend"; -import { PapyrosEvent } from "../../PapyrosEvent"; -import { LogType, papyrosLog } from "../../util/Logging"; -import { Pyodide, PYODIDE_INDEX_URL, PYODIDE_JS_URL } from "./Pyodide"; -import { Channel } from "sync-message"; import { CompletionResult } from "@codemirror/autocomplete"; -import { parseData } from "../../util/Util"; +import { BackendEvent } from "../../BackendEvent"; +import { + pyodideExpose, Pyodide, + loadPyodideAndPackage, + PyodideExtras +} from "pyodide-worker-runner"; /* eslint-disable-next-line */ -const initPythonString = require("!!raw-loader!./init.py").default; +const pythonPackageUrl = require("!!url-loader!./python_package.tar.gz.load_by_url").default; -// Load in the Pyodide initialization script -importScripts(PYODIDE_JS_URL); -// Now loadPyodide is available -declare function loadPyodide(args: { indexURL: string; fullStdLib: boolean }): Promise; +async function getPyodide(): Promise { + return await loadPyodideAndPackage({ url: pythonPackageUrl, format: ".tgz" }); +} +const pyodidePromise = getPyodide(); /** * Implementation of a Python backend for Papyros * Powered by Pyodide (https://pyodide.org/) */ -class PythonWorker extends Backend { +class PythonWorker extends Backend { pyodide: Pyodide; - initialized: boolean; - + papyros: any; constructor() { super(); this.pyodide = {} as Pyodide; - this.initialized = false; } - private convert(data: any): any { - let converted = data; - if ("toJs" in data) { - converted = data.toJs(); - } - return Object.fromEntries(converted); + private static convert(data: any): any { + return data.toJs ? data.toJs({ dict_converter: Object.fromEntries }) : data; + } + + /** + * @return {any} Function to expose a method with Pyodide support + */ + protected override syncExpose(): any { + return pyodideExpose; } override async launch( - onEvent: (e: PapyrosEvent) => void, - channel: Channel + onEvent: (e: BackendEvent) => void ): Promise { - await super.launch(onEvent, channel); - this.pyodide = await loadPyodide({ - indexURL: PYODIDE_INDEX_URL, - fullStdLib: false - }); - // Load our own modules to connect Papyros and Pyodide - await this._runCodeInternal(initPythonString); + await super.launch(onEvent); + this.pyodide = await pyodidePromise; // Python calls our function with a PyProxy dict or a Js Map, // These must be converted to a PapyrosEvent (JS Object) to allow message passing - const eventCallback = (data: any): void => { - return this.onEvent(this.convert(data)); - }; - // Initialize our loaded Papyros module with the callback - this.pyodide.globals.get("init_papyros")(eventCallback); - this.initialized = true; + this.papyros = await this.pyodide.pyimport("papyros").Papyros.callKwargs( + { + callback: (e: any) => { + const converted = PythonWorker.convert(e); + return this.onEvent(converted); + } + } + ); } - override async _runCodeInternal(code: string): Promise { - if (this.initialized) { - try { - // Sometimes a SyntaxError can cause imports to fail - // We want the SyntaxError to be handled by process_code as well - await this.pyodide.loadPackagesFromImports(code); - } catch (e) { - papyrosLog(LogType.Debug, "Something went wrong while loading imports: ", e); - } - await this.pyodide.globals.get("process_code")(code); - } else { - // Don't use loadPackagesFromImports here because it loads matplotlib immediately - await this.pyodide.loadPackage("micropip"); - return this.pyodide.runPythonAsync(code); + async runCode(extras: PyodideExtras, code: string): Promise { + this.extras = extras; + if (extras.interruptBuffer) { + this.pyodide.setInterruptBuffer(extras.interruptBuffer); } + await this.papyros.run_async(code); } override async autocomplete(context: WorkerAutocompleteContext): Promise { - // Do not await as not strictly required to compute autocompletions - this.pyodide.loadPackagesFromImports(context.text); - const result = this.convert(await this.pyodide.globals.get("autocomplete")(context)); - result.options = parseData(result.options, result.contentType); - delete result.contentType; + const result = PythonWorker.convert(await this.papyros.autocomplete(context)); result.span = /^[\w$]*$/; return result; } @@ -88,5 +73,6 @@ class PythonWorker extends Backend { // Default export to be recognized as a TS module export default {} as any; -// Expose handles the actual export -expose(new PythonWorker()); + +// Comlink and Comsync handle the actual export +Comlink.expose(new PythonWorker()); diff --git a/src/workers/python/build_package.py b/src/workers/python/build_package.py new file mode 100644 index 00000000..ef0e2568 --- /dev/null +++ b/src/workers/python/build_package.py @@ -0,0 +1,49 @@ + +import tarfile +import shutil +import os +import subprocess +import sys + +def tarfile_filter(tar_info): + name = tar_info.name + if any( + x in name + for x in [ + "__pycache__", + "friendly_traceback/locales", + "dist-info" + ] + ) or name.endswith(".pyc"): + return None + return tar_info + +def create_package(package_name, dependencies, extra_deps): + shutil.rmtree(package_name, ignore_errors=True) + install_dependencies(dependencies.split(" "), package_name) + try: + shutil.copytree(extra_deps, + os.path.join(package_name, extra_deps), dirs_exist_ok=True) + except Exception as e: + # Always seems to result in a harmless permission denied error + print(e) + pass + tar_name = f"{package_name}.tar.gz.load_by_url" + if os.path.exists(tar_name): + os.remove(tar_name) + with tarfile.open(tar_name, "w:gz") as tar: + tar.add(package_name, arcname="", recursive=True, filter=tarfile_filter) + shutil.rmtree(package_name) + +def install_dependencies(packages, out_dir): + if not isinstance(packages, list): + packages = [packages] + subprocess.check_call([sys.executable, "-m", "pip", "install", "-t", out_dir, *packages]) + +def check_tar(tarname): + with open(tarname, "rb") as t: + shutil.unpack_archive(tarname, ".", 'gztar') + + +if __name__ == "__main__": + create_package("python_package", "python-runner friendly_traceback jedi", extra_deps="papyros") diff --git a/src/workers/python/init.py b/src/workers/python/init.py deleted file mode 100644 index df5a8e54..00000000 --- a/src/workers/python/init.py +++ /dev/null @@ -1,188 +0,0 @@ -import sys -import json -import os -from collections.abc import Awaitable - -import micropip -from pyodide import to_js - -await micropip.install("python_runner") -import python_runner - -ft = micropip.install("friendly_traceback") -jedi_install = micropip.install("jedi") -# Otherwise `import matplotlib` fails while assuming a browser backend -os.environ["MPLBACKEND"] = "AGG" - -# Code is executed in a worker with less resources than ful environment -sys.setrecursionlimit(500) - -# Global Papyros instance -papyros = None - - -class Papyros(python_runner.PatchedStdinRunner): - def override_matplotlib(self): - # workaround from https://github.com/pyodide/pyodide/issues/1518 - import base64 - from io import BytesIO - import matplotlib.pyplot - - def show(): - buf = BytesIO() - matplotlib.pyplot.savefig(buf, format="png") - buf.seek(0) - # encode to a base64 str - img = base64.b64encode(buf.read()).decode("utf-8") - matplotlib.pyplot.clf() - self.output("img", img, contentType="img/png;base64") - - matplotlib.pyplot.show = show - - async def run_async(self, source_code, mode="exec", top_level_await=True): - """ - Mostly a copy of the parent `run_async` with `await ft` in case of an exception, - because `serialize_traceback` isn't async. - """ - with self._execute_context(source_code): - try: - code_obj = self.pre_run( - source_code, mode, top_level_await=top_level_await) - if code_obj: - result = self.execute(code_obj, source_code, mode) - while isinstance(result, Awaitable): - result = await result - return result - except: - await ft - # Let `_execute_context` and `serialize_traceback` - # handle the exception - raise - - def serialize_syntax_error(self, exc, source_code): - raise # Rethrow to ensure FriendlyTraceback library is imported correctly - - def serialize_traceback(self, exc, source_code): - import friendly_traceback # Delay import for faster startup - from friendly_traceback.core import FriendlyTraceback - - # Allow friendly_traceback to inspect the code - friendly_traceback.source_cache.cache.add(self.filename, source_code) - - # Initialize traceback - fr = FriendlyTraceback(type(exc), exc, exc.__traceback__) - fr.assign_generic() - fr.assign_cause() - # Translate properties to FriendlyError interface - tb = fr.info.get("shortened_traceback", "") - info = fr.info.get("generic", "") - why = fr.info.get("cause", "") - what = fr.info.get("message", "") - - name = type(exc).__name__ - user_start = 0 - tb_lines = tb.split("\n") - # Find first line in traceback that involves code from the user - while user_start < len(tb_lines) and self.filename not in tb_lines[user_start]: - user_start += 1 - # Find line containing Exception name, denoting end of location of issue - user_end = user_start + 1 - while user_end < len(tb_lines) and name not in tb_lines[user_end]: - user_end += 1 - where = "\n".join(tb_lines[user_start:user_end]) or "" - # Format for callback - return dict( - text=json.dumps( - dict( - name=name, - traceback=tb, - info=info, - why=why, - where=where, - what=what - ) - ) - ) - - -def init_papyros(event_callback): - global papyros - - def runner_callback(event_type, data): - def cb(typ, dat, **kwargs): - return event_callback(to_js(dict(type=typ, data=dat, **kwargs))) - - # Translate python_runner events to papyros events - if event_type == "output": - for part in data["parts"]: - typ = part["type"] - if typ in ["stderr", "traceback", "syntax_error"]: - cb("error", part["text"], contentType="text/json") - elif typ == "stdout": - cb("output", part["text"], contentType="text/plain") - elif typ == "img": - cb("output", part["text"], contentType=part["contentType"]) - elif typ in ["input", "input_prompt"]: - continue - else: - raise ValueError(f"Unknown output part type {typ}") - elif event_type == "input": - return cb("input", json.dumps(dict(prompt=data["prompt"])), contentType="text/json") - else: - raise ValueError(f"Unknown event type {event_type}") - - papyros = Papyros(callback=runner_callback) - - -async def process_code(code, filename="my_code.py"): - with open(filename, "w") as f: - f.write(code) - papyros.filename = filename - - try: - import matplotlib - except ModuleNotFoundError: - pass - else: - # Only override matplotlib when required by the code - papyros.override_matplotlib() - - await papyros.run_async(code) - -def convert_completion(completion, index): - converted = dict(type=completion.type, label=completion.name_with_symbols) - # if completion.get_signatures(): - # converted["detail"] = completion.get_signatures()[0].description - # converted["detail"] = f"{completion.parent().name} ({completion.type})" - if completion.type != "keyword": - # Keywords have obvious meanings yet non-useful docstrings - converted["info"] = completion.docstring().replace("\n", "\r\n") - # Jedi does sorting, so give earlier element highest boost - converted["boost"] = -index - return converted - -async def autocomplete(context): - context = context.to_py() - if context["before"]: - before = context["before"] - else: - before = dict(text=None) - before["from"] = context["pos"] - - complete_from = before["from"] - if not context["explicit"] and \ - (context["before"] is not None and not context["before"]["text"]): - # If user did not request completions, don't complete for the empty string - options = [] - else: - await jedi_install - import jedi - s = jedi.Script(context["text"]) - # Convert Jedi completions to CodeMirror objects - options = [convert_completion(c, i) - for (i, c) in enumerate(s.complete(line=context["line"], column=context["column"]))] - if "." in before["text"]: - complete_from = before["to"] - results = dict(options=json.dumps(options), contentType="text/json") - results["from"] = complete_from - return to_js(results) diff --git a/src/workers/python/papyros/__init__.py b/src/workers/python/papyros/__init__.py new file mode 100644 index 00000000..760fe982 --- /dev/null +++ b/src/workers/python/papyros/__init__.py @@ -0,0 +1 @@ +from .papyros import Papyros diff --git a/src/workers/python/papyros/autocomplete.py b/src/workers/python/papyros/autocomplete.py new file mode 100644 index 00000000..35539c79 --- /dev/null +++ b/src/workers/python/papyros/autocomplete.py @@ -0,0 +1,30 @@ +from jedi import Script + +def convert_completion(completion, index): + converted = dict(type=completion.type, label=completion.name_with_symbols) + if completion.type != "keyword": + # Keywords have obvious meanings yet non-useful docstrings + converted["info"] = completion.docstring().replace("\n", "\r\n") + # Jedi does sorting, so give earlier element highest boost + converted["boost"] = -index + return converted + +async def autocomplete(context): + # Ensure before-match is not None + before = context.get("before", {"text": "", "from": context["pos"]}) + complete_from = before["from"] + if context["explicit"] or before["text"]: + completions = Script(context["text"]).complete(line=context["line"], column=context["column"]) + # Convert Jedi completions to CodeMirror objects + options = [convert_completion(c, i) + for (i, c) in enumerate(completions)] + if "." in before["text"]: + # Completing a property access, so complete from end of match + complete_from = before["to"] + else: + # Don't complete for emptry string unless asked for + options = [] + return { + "options": options, + "from": complete_from + } diff --git a/src/workers/python/papyros/papyros.py b/src/workers/python/papyros/papyros.py new file mode 100644 index 00000000..04f744de --- /dev/null +++ b/src/workers/python/papyros/papyros.py @@ -0,0 +1,164 @@ +import os +import sys +import json +import python_runner +import friendly_traceback + +from friendly_traceback.core import FriendlyTraceback +from collections.abc import Awaitable +from pyodide_worker_runner import install_imports + + +from .util import to_py +from .autocomplete import autocomplete + +SYS_RECURSION_LIMIT = 500 + +# Global Papyros instance +papyros = None + +class Papyros(python_runner.PyodideRunner): + def __init__( + self, + *, + source_code="", + filename="my_program.py", + callback=None, + limit=SYS_RECURSION_LIMIT + ): + if callback is None: + raise ValueError("Callback must not be None") + super().__init__(source_code=source_code, filename=filename) + self.limit = limit + self.override_globals() + self.set_event_callback(callback) + + def set_event_callback(self, event_callback): + def runner_callback(event_type, data): + def cb(typ, dat, contentType=None, **kwargs): + return event_callback(dict(type=typ, data=dat, contentType=contentType or "text/plain", **kwargs)) + + if event_type == "output": + for part in data["parts"]: + typ = part["type"] + if typ in ["stderr", "traceback", "syntax_error"]: + cb("error", part["text"], contentType=part.get("contentType")) + elif typ in ["input", "input_prompt"]: + # Do not display values entered by user for input + continue + else: + cb("output", part["text"], contentType=part.get("contentType")) + elif event_type == "input": + return cb("input", data["prompt"]) + elif event_type == "sleep": + return cb("sleep", data["seconds"]*1000, contentType="application/number") + else: + return cb(event_type, data.get("data", ""), contentType=data.get("contentType")) + + self.set_callback(runner_callback) + + def override_globals(self): + # Code is executed in a worker with less resources than ful environment + sys.setrecursionlimit(self.limit) + # Otherwise `import matplotlib` fails while assuming a browser backend + os.environ["MPLBACKEND"] = "AGG" + try: + import matplotlib + except ModuleNotFoundError: + pass + else: + # Only override matplotlib when required by the code + self.override_matplotlib() + + def override_matplotlib(self): + # workaround from https://github.com/pyodide/pyodide/issues/1518 + import base64 + from io import BytesIO + import matplotlib.pyplot + + def show(): + buf = BytesIO() + matplotlib.pyplot.savefig(buf, format="png") + buf.seek(0) + # encode to a base64 str + img = base64.b64encode(buf.read()).decode("utf-8") + matplotlib.pyplot.clf() + self.output("img", img, contentType="img/png;base64") + + matplotlib.pyplot.show = show + + async def install_imports(self, source_code, ignore_missing=True): + try: + await install_imports(source_code) + except ValueError: + if not ignore_missing: + raise + + def pre_run(self, source_code, mode="exec", top_level_await=False): + self.override_globals() + return super().pre_run(source_code, mode=mode, top_level_await=top_level_await) + + async def run_async(self, source_code, mode="exec", top_level_await=True): + with self._execute_context(): + try: + await self.install_imports(source_code, ignore_missing=False) + code_obj = self.pre_run(source_code, mode=mode, top_level_await=top_level_await) + if code_obj: + result = self.execute(code_obj, mode) + while isinstance(result, Awaitable): + result = await result + return result + except Exception as e: + # Sometimes KeyboardInterrupt is caught by Pyodide and raised as a PythonError + # with a js_error containing the reason + js_error = str(getattr(e, "js_error", "")) + if isinstance(e, KeyboardInterrupt) or "KeyboardInterrupt" in js_error: + self.callback("interrupt", data="KeyboardInterrupt", contentType="text/plain") + else: + raise + + def serialize_syntax_error(self, exc): + raise # Rethrow to ensure FriendlyTraceback library is imported correctly + + def serialize_traceback(self, exc): + # Allow friendly_traceback to inspect the code + friendly_traceback.source_cache.cache.add(self.filename, self.source_code) + + # Initialize traceback + fr = FriendlyTraceback(type(exc), exc, exc.__traceback__) + fr.assign_generic() + fr.assign_cause() + # Translate properties to FriendlyError interface + tb = fr.info.get("shortened_traceback", "") + info = fr.info.get("generic", "") + why = fr.info.get("cause", "") + what = fr.info.get("message", "") + + name = type(exc).__name__ + user_start = 0 + tb_lines = tb.split("\n") + # Find first line in traceback that involves code from the user + while user_start < len(tb_lines) and self.filename not in tb_lines[user_start]: + user_start += 1 + # Find line containing Exception name, denoting end of location of issue + user_end = user_start + 1 + while user_end < len(tb_lines) and name not in tb_lines[user_end]: + user_end += 1 + where = "\n".join(tb_lines[user_start:user_end]) or "" + # Format for callback + return dict( + text=json.dumps(dict( + name=name, + traceback=tb, + info=info, + why=why, + where=where, + what=what + )), + contentType="text/json" + ) + + async def autocomplete(self, context): + context = to_py(context) + await self.install_imports(context["text"], ignore_missing=True) + return await autocomplete(context) diff --git a/src/workers/python/papyros/util.py b/src/workers/python/papyros/util.py new file mode 100644 index 00000000..d22d8992 --- /dev/null +++ b/src/workers/python/papyros/util.py @@ -0,0 +1,4 @@ +def to_py(arg): + if hasattr(arg, "to_py"): + arg = arg.to_py() + return arg diff --git a/src/workers/python/python_package.tar.gz.load_by_url b/src/workers/python/python_package.tar.gz.load_by_url new file mode 100644 index 00000000..1c425e7e Binary files /dev/null and b/src/workers/python/python_package.tar.gz.load_by_url differ diff --git a/webpack.config.js b/webpack.config.js index 10bd8f56..32c7a94e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,24 +1,33 @@ const path = require("path"); -const glob = require("glob"); const TerserPlugin = require('terser-webpack-plugin'); const PUBLIC_DIR = "public"; const LIBRARY_DIR = "dist"; const DEVELOPMENT_PORT = 8080; -module.exports = function (webpackEnv) { - const mode = webpackEnv["mode"]; +module.exports = function (webpackEnv, argv) { + let mode = argv.mode; + if (!mode) { + mode = webpackEnv.WEBPACK_SERVE ? 'development' : 'production' + } // In development, the bundle is loaded from the public folder // In production, node_modules typically use the dist folder - const outFolder = mode === "development" ? PUBLIC_DIR : LIBRARY_DIR; + let outFolder = ""; + let entries = {}; + if (mode === "development") { + outFolder = PUBLIC_DIR; + entries = Object.fromEntries([ + ["App", "./src/App.ts"], + ["InputServiceWorker", "./src/InputServiceWorker.ts"] + ]); + } else { + outFolder = LIBRARY_DIR; + entries = Object.fromEntries([ + ["Library", "./src/Library.ts"], + ["/workers/input/InputWorker", "./src/workers/input/InputWorker.ts"] + ]); + } return { - entry: Object.fromEntries( - glob.sync("./src/**/*.{ts,js}") // All js and ts file - // But not already typed files or worker files (those get inlined) - .filter(n => !n.includes(".d.ts") && !n.includes(".worker.ts")) - // Strip src folder and extension - // Obtain [name, actual path] - .map(v => [v.split("./src/")[1].split(".")[0], v]) - ), + entry: entries, module: { rules: [ // Inline bundle worker-scripts to prevent bundle resolution errors diff --git a/yarn.lock b/yarn.lock index c7f6c4e2..0e82e68d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1093,7 +1093,7 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== -"@types/retry@^0.12.0": +"@types/retry@^0.12.0", "@types/retry@^0.12.1": version "0.12.1" resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.1.tgz#d8f1c0d0dc23afad6dc16a9e993a0865774b4065" integrity sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g== @@ -2063,6 +2063,11 @@ compression@^1.7.4: safe-buffer "5.1.2" vary "~1.1.2" +comsync@^0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/comsync/-/comsync-0.0.7.tgz#fd148e98815402db7ef27782f5992c719aa8cd87" + integrity sha512-/MarUXxRBvSg3m38qxg8Kx0g8pgc3ENbYXUOL2Hf2jysjniEKD/DFxlQNdiNDWDbeVUrpMcM5KZq+oqh01lgcA== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -2412,7 +2417,7 @@ encoding@^0.1.12: dependencies: iconv-lite "^0.6.2" -enhanced-resolve@^5.0.0, enhanced-resolve@^5.8.3: +enhanced-resolve@^5.0.0: version "5.9.1" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.9.1.tgz#e898cea44d9199fd92137496cff5691b910fb43e" integrity sha512-jdyZMwCQ5Oj4c5+BTnkxPgDZO/BJzh/ADDmKebayyzNwjVX1AFCeGkOfxNx0mHi2+8BKC5VxUYiw3TIvoT7vhw== @@ -2420,6 +2425,14 @@ enhanced-resolve@^5.0.0, enhanced-resolve@^5.8.3: graceful-fs "^4.2.4" tapable "^2.2.0" +enhanced-resolve@^5.9.2: + version "5.9.2" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.9.2.tgz#0224dcd6a43389ebfb2d55efee517e5466772dd9" + integrity sha512-GIm3fQfwLJ8YZx2smuHpBKkXC1yOk+OBEmKckVyL0i/ea8mqDEykK3ld5dgH1QYPNyT/lIllxV2LULnxCHaHkA== + dependencies: + graceful-fs "^4.2.4" + tapable "^2.2.0" + enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" @@ -4617,6 +4630,14 @@ p-retry@^4.5.0: "@types/retry" "^0.12.0" retry "^0.13.1" +p-retry@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-5.1.0.tgz#a436926f54a15d833e3c39bbee9a73036716cfd4" + integrity sha512-zh8em2ciphCu4eZYzatLp4bTYkAhyi8PwMIOyQyh1b5bxunYNe6nwumHPkUBtvmEfIfnTYzhOq1+vWf46Qii+w== + dependencies: + "@types/retry" "^0.12.1" + retry "^0.13.1" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -4903,6 +4924,13 @@ purgecss@^4.0.3: postcss "^8.3.5" postcss-selector-parser "^6.0.6" +pyodide-worker-runner@^0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/pyodide-worker-runner/-/pyodide-worker-runner-0.0.8.tgz#8469060750ee098e4be194bb51efa71403710514" + integrity sha512-476A8teV5fI9kidqF5ElDpfAxKpbFmVUszEIxy9phqapJsyhtmZeKV8AjfM7Qy8bQHZ/2TxC3IodOnra1rijpQ== + dependencies: + p-retry "^5.0.0" + qs@6.9.7: version "6.9.7" resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.7.tgz#4610846871485e1e048f44ae3b94033f0e675afe" @@ -5497,10 +5525,10 @@ symbol-tree@^3.2.4: resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== -sync-message@^0.0.8: - version "0.0.8" - resolved "https://registry.yarnpkg.com/sync-message/-/sync-message-0.0.8.tgz#ca20723cd85fe819284ce82de3797ca108b02659" - integrity sha512-2DiQIfLgf6OnBU2cNEqV38UZyMD9nUkXyaWXUPFIykdbfpXl3V5x9VmJH4s7M66IK5G1C7wY6KB9W/44E+vVvQ== +sync-message@^0.0.9: + version "0.0.9" + resolved "https://registry.yarnpkg.com/sync-message/-/sync-message-0.0.9.tgz#8da80c1103b5eb688729e22d9480b8a59c83aa5d" + integrity sha512-mA7WltTNV+kHaM8SkDExirvwH7/+IXPR8+2JSYa2ahKdP+mdatO9/vZuuTKMiBcPAk9zQ5gfAcbSOFwKQeQEIA== table@^6.0.9: version "6.8.0" @@ -5781,6 +5809,15 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +url-loader@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.1.tgz#28505e905cae158cf07c92ca622d7f237e70a4e2" + integrity sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA== + dependencies: + loader-utils "^2.0.0" + mime-types "^2.1.27" + schema-utils "^3.0.0" + util-deprecate@^1.0.1, util-deprecate@^1.0.2, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -5866,7 +5903,7 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== -webpack-cli@^4.9.1: +webpack-cli@^4.9.2: version "4.9.2" resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.9.2.tgz#77c1adaea020c3f9e2db8aad8ea78d235c83659d" integrity sha512-m3/AACnBBzK/kMTcxWHcZFPrw/eQuY4Df1TxvIWfWM2x7mRqBQCqKEd96oCUa9jkapLBaFfRce33eGDb4Pr7YQ== @@ -5895,7 +5932,7 @@ webpack-dev-middleware@^5.3.1: range-parser "^1.2.1" schema-utils "^4.0.0" -webpack-dev-server@^4.6.0: +webpack-dev-server@^4.7.4: version "4.7.4" resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-4.7.4.tgz#d0ef7da78224578384e795ac228d8efb63d5f945" integrity sha512-nfdsb02Zi2qzkNmgtZjkrMOcXnYZ6FLKcQwpxT7MvmHKc+oTtDsBju8j+NMyAygZ9GW1jMEUpy3itHtqgEhe1A== @@ -5944,10 +5981,10 @@ webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.64.0: - version "5.69.1" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.69.1.tgz#8cfd92c192c6a52c99ab00529b5a0d33aa848dc5" - integrity sha512-+VyvOSJXZMT2V5vLzOnDuMz5GxEqLk7hKWQ56YxPW/PQRUuKimPqmEIJOx8jHYeyo65pKbapbW464mvsKbaj4A== +webpack@^5.70.0: + version "5.70.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.70.0.tgz#3461e6287a72b5e6e2f4872700bc8de0d7500e6d" + integrity sha512-ZMWWy8CeuTTjCxbeaQI21xSswseF2oNOwc70QSKNePvmxE7XW36i7vpBMYZFAUHPwQiEbNGCEYIOOlyRbdGmxw== dependencies: "@types/eslint-scope" "^3.7.3" "@types/estree" "^0.0.51" @@ -5958,7 +5995,7 @@ webpack@^5.64.0: acorn-import-assertions "^1.7.6" browserslist "^4.14.5" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.8.3" + enhanced-resolve "^5.9.2" es-module-lexer "^0.9.0" eslint-scope "5.1.1" events "^3.2.0"