diff --git a/pxtcompiler/emitter/driver.ts b/pxtcompiler/emitter/driver.ts index b9c11e4e5f13..de3fdf2e92a0 100644 --- a/pxtcompiler/emitter/driver.ts +++ b/pxtcompiler/emitter/driver.ts @@ -271,6 +271,7 @@ namespace ts.pxtc { if (opts.ast || opts.forceEmit || res.diagnostics.length == 0) { const binOutput = compileBinary(program, opts, res, entryPoint); + res.times["compilebinary"] = U.cpuUs() - emitStart res.diagnostics = res.diagnostics.concat(patchUpDiagnostics(binOutput.diagnostics)) } diff --git a/pxtcompiler/emitter/hexfile.ts b/pxtcompiler/emitter/hexfile.ts index 9be9a76df009..bef33dc7b891 100644 --- a/pxtcompiler/emitter/hexfile.ts +++ b/pxtcompiler/emitter/hexfile.ts @@ -1083,6 +1083,7 @@ _stored_program: .hex ${res} src = asmHeader(bin) + src } if (opts.embedBlob) { + bin.packedSource = packSource(opts.embedMeta, ts.pxtc.decodeBase64(opts.embedBlob)) // TODO more dynamic check for source size if (!bin.target.noSourceInFlash && bin.packedSource.length < 40000) { diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 600014b792b4..f7addc208f35 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -630,7 +630,7 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, packagedFiles?: pxt.Map): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -641,9 +641,15 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); - if (!ghPkg) { - throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); + let ghPkg: CachedPackage; + let caughtError: any; + try { + ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + } catch (e) { + caughtError = e; + } + if ((!ghPkg || caughtError) && !packagedFiles?.[`${extSrc}-backup.json`]) { + throw caughtError || new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } } ) diff --git a/pxtlib/main.ts b/pxtlib/main.ts index 81451cc05338..8e869d1f17a3 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -439,7 +439,7 @@ namespace pxt { export interface Host { readFile(pkg: Package, filename: string, skipAdditionalFiles?: boolean): string; writeFile(pkg: Package, filename: string, contents: string, force?: boolean): void; - downloadPackageAsync(pkg: Package, deps?: string[]): Promise; + downloadPackageAsync(pkg: Package, deps?: string[]/**, fallback?: () => void **/): Promise; getHexInfoAsync(extInfo: pxtc.ExtensionInfo): Promise; cacheStoreAsync(id: string, val: string): Promise; cacheGetAsync(id: string): Promise; // null if not found diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 9ca53e06ccc1..4f7521cea791 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -142,12 +142,35 @@ namespace pxt { const proto = this.verProtocol() let files: Map; - if (proto == "pub") { - files = await Cloud.downloadScriptFilesAsync(this.verArgument()) - } else if (proto == "github") { - const config = await pxt.packagesConfigAsync(); - const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) - files = resp.files; + if (proto == "pub" || proto == "github") { + let caughtError: any; + try { + if (proto == "pub") { + files = await Cloud.downloadScriptFilesAsync(this.verArgument()) + } else { + const config = await pxt.packagesConfigAsync(); + const resp = await pxt.github.downloadPackageAsync(this.verArgument(), config) + files = resp.files; + } + } catch (e) { + caughtError = e ; + } + + // attempt to grab from top level package if `${dep._verspec}-backup.json` defined + let p = this.parent; + while (p && p != this) { + const backupFile = p.readFile(`${this._verspec}-backup.json`); + if (backupFile) { + files = JSON.parse(backupFile); + break; + } + p = p.parent; + } + + if (!files && caughtError) { + // no backup found, rethrow + throw caughtError || new Error("Could not download."); + } } else if (proto == "embed") { files = pxt.getEmbeddedScript(this.verArgument()) } else if (proto == "pkg") { @@ -307,27 +330,70 @@ namespace pxt { return Promise.resolve(v) } - private downloadAsync() { - return this.resolveVersionAsync() - .then(verNo => { - if (this.invalid()) { - pxt.debug(`skip download of invalid package ${this.id}`); - return undefined; - } - if (!/^embed:/.test(verNo) && this.installedVersion == verNo) - return undefined; - pxt.debug('downloading ' + verNo) - return this.host().downloadPackageAsync(this) - .then(() => { - this.loadConfig(); - - if (this.isAssetPack()) { - this.writeAssetPackFiles(); - } - pxt.debug(`installed ${this.id} /${verNo}`) - }) + /** + * strip out at beginning of hex file import + * push them into hex, github cache if those values are not already there + * flag attached that says they're temporary / backup only + * treat the cache as expired by time + * keep trying to fetch new one but return it in place until network is available + **/ + private async downloadAsync(): Promise { + const verNo = await this.resolveVersionAsync(); + if (this.invalid()) { + pxt.debug(`skip download of invalid package ${this.id}`); + return undefined; + } + if (!/^embed:/.test(verNo) && this.installedVersion == verNo) + return undefined; + pxt.debug('downloading ' + verNo) + await this.host().downloadPackageAsync( + this, + // undefined, + // () => { + // let p = this.parent; + // while (p && p != this) { + // const backupFile = p.readFile(`${this._verspec}-backup.json`); + // if (backupFile) { + // return JSON.parse(backupFile); + // } + // p = p.parent; + // } + + // return undefined; + // } + ); + // try { + // } catch (e) { + // let files: Map; + // let p = this.parent; + // while (p && p != this) { + // const backupFile = p.readFile(`${this._verspec}-backup.json`); + // if (backupFile) { + // files = JSON.parse(backupFile); + // break; + // } + // p = p.parent; + // } + + // if (files) { + // // this.writeFile(pxt.CONFIG_NAME, files[pxt.CONFIG_NAME]) + // // for (const fname of Object.keys(files)) { + // // this.writeFile(fname, files[fname]); + // // } + // // this + // } else { + // // no backup found, rethrow + // throw e; + // } + // } + + this.loadConfig(); + + if (this.isAssetPack()) { + this.writeAssetPackFiles(); + } - }) + pxt.debug(`installed ${this.id} /${verNo}`) } loadConfig() { @@ -1340,6 +1406,8 @@ namespace pxt { for (const dep of depsToPack) { // todo; swap this over to just one json blob under files, // not keyed off -backup.json, to make extracting / hiding more obvious. key goes in pxtlib/main.ts + + // include c++ pxtc.HexInfo as well? if (files[`${dep._verspec}-backup.json`]) continue; const packed: Map = {} diff --git a/webapp/src/app.tsx b/webapp/src/app.tsx index 6e355c0b3eb8..ce28ff565aad 100644 --- a/webapp/src/app.tsx +++ b/webapp/src/app.tsx @@ -2868,7 +2868,7 @@ export class ProjectView } files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - await pxt.github.cacheProjectDependenciesAsync(cfg); + await pxt.github.cacheProjectDependenciesAsync(cfg, files); const hd = await workspace.installAsync({ name: cfg.name, diff --git a/webapp/src/package.ts b/webapp/src/package.ts index b4a83de08f0d..1e6c5a4b760f 100644 --- a/webapp/src/package.ts +++ b/webapp/src/package.ts @@ -456,7 +456,13 @@ export class EditorPackage { sortedFiles(): File[] { let lst = Util.values(this.files) if (!pxt.options.debug) - lst = lst.filter(f => f.name != pxt.github.GIT_JSON && f.name != pxt.SIMSTATE_JSON && f.name != pxt.SERIAL_EDITOR_FILE && f.name != pxt.PALETTES_FILE) + lst = lst.filter(f => + f.name != pxt.github.GIT_JSON + && f.name != pxt.SIMSTATE_JSON + && f.name != pxt.SERIAL_EDITOR_FILE + && f.name != pxt.PALETTES_FILE + && !/(pub|github):.+-backup.json/.test(f.name) + ); lst.sort((a, b) => a.weight() - b.weight() || Util.strcmp(a.name, b.name)) return lst } @@ -701,7 +707,7 @@ class Host .then(v => v.val, e => null) } - downloadPackageAsync(pkg: pxt.Package): Promise { + downloadPackageAsync(pkg: pxt.Package, deps?: string[]/**, fallback?: () => void**/): Promise { let proto = pkg.verProtocol() let epkg = getEditorPkg(pkg) @@ -723,13 +729,25 @@ class Host } }) + const handleMissingNetwork = (e: any) => { + // const files = fallback(); + const files = mainPkg.readFile(`${pkg._verspec}-backup.json`); + if (files) { + epkg.setFiles(JSON.parse(files)); + } else { + throw e; + } + } + if (proto == "pub") { // make sure it sits in cache return workspace.getPublishedScriptAsync(pkg.verArgument()) .then(files => epkg.setFiles(files)) + .catch(handleMissingNetwork); } else if (proto == "github") { return workspace.getPublishedScriptAsync(pkg.version()) .then(files => epkg.setFiles(files)) + .catch(handleMissingNetwork); } else if (proto == "workspace") { return fromWorkspaceAsync(pkg.verArgument()) } else if (proto == "file") { diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index bfbde0a7a5fb..7c42409172cb 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -646,7 +646,8 @@ export function installAsync(h0: InstallHeader, text: ScriptText, dontOverwriteI pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - return pxt.github.cacheProjectDependenciesAsync(cfg) + + return pxt.github.cacheProjectDependenciesAsync(cfg, text) .then(() => importAsync(h, text)) .then(() => h); }