Supporting more build environments

.vscode/launch.json

@@ -9,8 +9,8 @@
       "request": "launch",
       "name": "Run tests",
       "program": "${workspaceFolder}/node_modules/.bin/jest",
-      "cwd": "${workspaceFolder}/packages/core",
-      "args": ["--runInBand", "--testPathPattern", "tests/import-adder.test.js"],
+      "cwd": "${workspaceFolder}/packages/compat",
+      "args": ["--runInBand", "--testPathPattern", "tests/stage2  .test.js"],
       "outputCapture": "std"
     },
     {

packages/compat/package.json

@@ -57,7 +57,7 @@
     "semver": "^7.3.2",
     "symlink-or-copy": "^1.2.0",
     "tree-sync": "^2.0.0",
-    "typescript-memoize": "^1.0.0-alpha.3",
+    "typescript-memoize": "^1.0.0",
     "walk-sync": "^1.1.3",
     "yargs": "^16.1.0"
   },

packages/compat/src/compat-adapters/@ember-data/model.ts

@@ -6,7 +6,7 @@ export default class EmberDataModel extends V1Addon {
     if (!meta['implicit-modules']) {
       meta['implicit-modules'] = [];
     }
-    meta['implicit-modules'].push('./-private');
+    meta['implicit-modules'].push('./-private.js');
     return meta;
   }
 }

packages/compat/src/compat-adapters/ember-asset-loader.ts

@@ -1,5 +1,6 @@
 import V1Addon from '../v1-addon';
 import Funnel from 'broccoli-funnel';
+import cloneDeep from 'lodash/cloneDeep';
 
 // ember-asset-loader's ManifestGenerator (which is used as the Addon base class
 // for by ember-engines) has an "all" postprocessTree hook. We can't / won't run
@@ -14,7 +15,16 @@ import Funnel from 'broccoli-funnel';
 export default class extends V1Addon {
   get v2Tree() {
     return new Funnel(super.v2Tree, {
-      exclude: ['_app/config/asset-manifest.js', '_app_/instance-initializers/load-asset-manifest.js'],
+      exclude: ['_app_/config/asset-manifest.js', '_app_/instance-initializers/load-asset-manifest.js'],
     });
   }
+  get packageMeta() {
+    let meta = super.packageMeta;
+    if (meta['app-js']) {
+      meta = cloneDeep(meta);
+      delete meta['app-js']!['./instance-initializers/load-asset-manifest.js'];
+      delete meta['app-js']!['./config/asset-manifest.js'];
+    }
+    return meta;
+  }
 }

packages/compat/src/compat-adapters/ember-source.ts

@@ -9,6 +9,22 @@ import semver from 'semver';
 export default class extends V1Addon {
   private useRealModules = semver.satisfies(this.packageJSON.version, '>=3.27.0', { includePrerelease: true });
 
+  get newPackageJSON() {
+    let extra = {
+      './dist/ember-template-compiler': './dist/ember-template-compiler.js',
+    };
+    let pkg = JSON.parse(JSON.stringify(super.newPackageJSON));
+    if (!pkg.exports) {
+      pkg.exports = extra;
+    } else {
+      Object.assign(pkg.exports, extra);
+    }
+    return pkg;
+  }
+  get v2Tree() {
+    return mergeTrees([super.v2Tree, new Funnel(this.rootTree, { include: ['dist/ember-template-compiler.js'] })]);
+  }
+
   // when using real modules, we're replacing treeForAddon and treeForVendor
   customizes(treeName: string) {
     return (

packages/compat/src/compat-app.ts

@@ -13,7 +13,7 @@ import {
   Package,
   TemplateCompilerPlugins,
   Resolver,
-  TemplateCompiler,
+  NodeTemplateCompiler,
   AddonPackage,
 } from '@embroider/core';
 import V1InstanceCache from './v1-instance-cache';
@@ -398,7 +398,7 @@ class CompatAppAdapter implements AppAdapter<TreeNames> {
     // It's ok that this isn't a fully configured template compiler. We're only
     // using it to parse component snippets out of rules.
     resolver.astTransformer(
-      new TemplateCompiler({
+      new NodeTemplateCompiler({
         compilerPath: resolveSync(this.templateCompilerPath(), { basedir: this.root }),
         EmberENV: {},
         plugins: {},

packages/compat/src/rewrite-package-json.ts

@@ -2,37 +2,15 @@ import Plugin from 'broccoli-plugin';
 import { Node } from 'broccoli-node-api';
 import { writeFileSync } from 'fs';
 import { join } from 'path';
-import { AddonMeta } from '@embroider/core';
-
-type GetMeta = () => Partial<AddonMeta>;
 
 export default class RewritePackageJSON extends Plugin {
-  constructor(inputTree: Node, private getMeta: GetMeta, private originalPackageJSON: any) {
+  constructor(inputTree: Node, private getPackageJSON: () => any) {
     super([inputTree], {
       annotation: 'embroider:core:rewrite-package-json',
     });
   }
 
   build() {
-    let pkg = this.originalPackageJSON;
-    let meta: AddonMeta = Object.assign(
-      {},
-      pkg.meta,
-      {
-        version: 2,
-        'auto-upgraded': true,
-        type: 'addon',
-      } as AddonMeta,
-      this.getMeta()
-    );
-    pkg['ember-addon'] = meta;
-
-    // classic addons don't get to customize their entrypoints like this. We
-    // always rewrite them so their entrypoint is index.js, so whatever was here
-    // is just misleading to stage3 packagers that might look (rollup does).
-    delete pkg.main;
-    delete pkg.module;
-
-    writeFileSync(join(this.outputPath, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8');
+    writeFileSync(join(this.outputPath, 'package.json'), JSON.stringify(this.getPackageJSON(), null, 2), 'utf8');
   }
 }

packages/compat/src/template-compiler-broccoli-plugin.ts

@@ -0,0 +1,31 @@
+import type { Node } from 'broccoli-node-api';
+import Filter from 'broccoli-persistent-filter';
+import type { TemplateCompiler } from '@embroider/core';
+import { join } from 'path';
+
+export default class TemplateCompileTree extends Filter {
+  constructor(inputTree: Node, private templateCompiler: TemplateCompiler, private stage: 1 | 3) {
+    super(inputTree, {
+      name: `embroider-template-compile-stage${stage}`,
+      persist: true,
+      extensions: ['hbs', 'handlebars'],
+      // in stage3 we are changing the file extensions from hbs to js. In
+      // stage1, we are just keeping hbs.
+      targetExtension: stage === 3 ? 'js' : undefined,
+    });
+  }
+
+  processString(source: string, relativePath: string) {
+    if (this.stage === 1) {
+      return this.templateCompiler.applyTransforms(relativePath, source);
+    } else {
+      return this.templateCompiler.compile(relativePath, source);
+    }
+  }
+  cacheKeyProcessString(source: string, relativePath: string) {
+    return `${this.stage}-${this.templateCompiler.cacheKey}` + super.cacheKeyProcessString(source, relativePath);
+  }
+  baseDir() {
+    return join(__dirname, '..');
+  }
+}

packages/compat/src/v1-addon.ts

@@ -11,7 +11,7 @@ import mergeTrees from 'broccoli-merge-trees';
 import semver from 'semver';
 import rewriteAddonTree from './rewrite-addon-tree';
 import { mergeWithAppend } from './merges';
-import { AddonMeta, TemplateCompiler, debug, PackageCache, Resolver, extensionsPattern } from '@embroider/core';
+import { AddonMeta, NodeTemplateCompiler, debug, PackageCache, Resolver, extensionsPattern } from '@embroider/core';
 import Options from './options';
 import walkSync from 'walk-sync';
 import ObserveTree from './observe-tree';
@@ -24,6 +24,7 @@ import writeFile from 'broccoli-file-creator';
 import SynthesizeTemplateOnlyComponents from './synthesize-template-only-components';
 import { isEmberAutoImportDynamic, isCompactReexports, isColocationPlugin } from './detect-babel-plugins';
 import { ResolvedDep } from '@embroider/core/src/resolver';
+import TemplateCompilerBroccoliPlugin from './template-compiler-broccoli-plugin';
 
 const stockTreeNames = Object.freeze([
   'addon',
@@ -83,7 +84,7 @@ class V1AddonCompatResolver implements Resolver {
       params,
     };
   }
-  astTransformer(_templateCompiler: TemplateCompiler): unknown {
+  astTransformer(_templateCompiler: NodeTemplateCompiler): unknown {
     return;
   }
   dependenciesOf(_moduleName: string): ResolvedDep[] {
@@ -122,15 +123,15 @@ export default class V1Addon {
 
   // this is only defined when there are custom AST transforms that need it
   @Memoize()
-  private get templateCompiler(): TemplateCompiler | undefined {
+  private get templateCompiler(): NodeTemplateCompiler | undefined {
     let htmlbars = this.addonInstance.addons.find((a: any) => a.name === 'ember-cli-htmlbars');
     if (htmlbars) {
       let options = htmlbars.htmlbarsOptions() as HTMLBarsOptions;
       if (options.plugins && options.plugins.ast) {
         // our macros don't run here in stage1
         options.plugins.ast = options.plugins.ast.filter((p: any) => !isEmbroiderMacrosPlugin(p));
         if (options.plugins.ast.length > 0) {
-          return new TemplateCompiler({
+          return new NodeTemplateCompiler({
             compilerPath: options.templateCompilerPath,
             EmberENV: {},
             plugins: options.plugins,
@@ -169,7 +170,7 @@ export default class V1Addon {
       _addon: this,
       toTree(this: { _addon: V1Addon }, tree: Node): Node {
         if (this._addon.templateCompiler) {
-          return this._addon.templateCompiler.applyTransformsToTree(tree);
+          return new TemplateCompilerBroccoliPlugin(tree, this._addon.templateCompiler, 1);
         } else {
           // when there are no custom AST transforms, we don't need to do
           // anything at all.
@@ -268,8 +269,27 @@ export default class V1Addon {
     return this.packageJSON.name;
   }
 
+  // you can override this to change the *input* packageJSON that the rest of
+  // stage1 will see. If you want to see and change the *output* packageJSON see
+  // `newPackageJSON`.
   protected get packageJSON() {
-    return this.addonInstance.pkg;
+    return this.packageCache.get(this.root).packageJSON;
+  }
+
+  protected get newPackageJSON() {
+    // shallow copy only! This is OK as long as we're only changing top-level
+    // keys in this method
+    let pkg = Object.assign({}, this.packageJSON);
+    let meta: AddonMeta = Object.assign({}, pkg.meta, this.packageMeta);
+    pkg['ember-addon'] = meta;
+
+    // classic addons don't get to customize their entrypoints like this. We
+    // always rewrite them so their entrypoint is index.js, so whatever was here
+    // is just misleading to stage3 packagers that might look (rollup does).
+    delete pkg.main;
+    delete pkg.module;
+
+    return pkg;
   }
 
   @Memoize()
@@ -513,7 +533,15 @@ export default class V1Addon {
   // things to the package metadata.
   protected get packageMeta(): Partial<AddonMeta> {
     let built = this.build();
-    return mergeWithAppend(built.staticMeta, ...built.dynamicMeta.map(d => d()));
+    return mergeWithAppend(
+      {
+        version: 2,
+        'auto-upgraded': true,
+        type: 'addon',
+      },
+      built.staticMeta,
+      ...built.dynamicMeta.map(d => d())
+    );
   }
 
   @Memoize()
@@ -758,20 +786,23 @@ export default class V1Addon {
   }
 
   private maybeSetDirectoryMeta(built: IntermediateBuild, tree: Node, localDir: string, key: keyof AddonMeta): Node {
-    // unforunately Funnel doesn't create destDir if its input exists but is
-    // empty. And we want to only put the app-js key in package.json if
-    // there's really a directory for it to point to. So we need to monitor
-    // the output and use dynamicMeta.
-    let dirExists = false;
+    let files: AddonMeta['app-js'];
     built.dynamicMeta.push(() => {
-      if (dirExists) {
-        return { [key]: localDir };
+      if (files) {
+        return { [key]: files };
       } else {
         return {};
       }
     });
     return new ObserveTree(tree, (outputPath: string) => {
-      dirExists = pathExistsSync(join(outputPath, localDir));
+      let dir = join(outputPath, localDir);
+      if (existsSync(dir)) {
+        files = Object.fromEntries(
+          walkSync(dir, { globs: ['**/*.js', '**/*.hbs'] }).map(f => [`./${f}`, `./${localDir}/${f}`])
+        );
+      } else {
+        files = undefined;
+      }
     });
   }
 
@@ -877,7 +908,7 @@ export default class V1Addon {
         publicAssets = {};
         for (let filename of walkSync(join(outputPath, 'public'))) {
           if (!filename.endsWith('/')) {
-            publicAssets[`public/${filename}`] = filename;
+            publicAssets[`./public/${filename}`] = './' + filename;
           }
         }
       });
@@ -938,12 +969,7 @@ export default class V1Addon {
   }
 
   private buildPackageJSON(built: IntermediateBuild) {
-    let packageJSONRewriter = new RewritePackageJSON(
-      this.rootTree,
-      () => this.packageMeta,
-      this.packageCache.get(this.root).packageJSON
-    );
-    built.trees.push(packageJSONRewriter);
+    built.trees.push(new RewritePackageJSON(this.rootTree, () => this.newPackageJSON));
   }
 
   @Memoize()
@@ -1000,7 +1026,7 @@ function babelPluginAllowedInStage1(plugin: PluginItem) {
     return false;
   }
 
-  if (TemplateCompiler.isInlinePrecompilePlugin(plugin)) {
+  if (NodeTemplateCompiler.isInlinePrecompilePlugin(plugin)) {
     // Similarly, the inline precompile plugin must not run in stage1. We
     // want all templates uncompiled. Instead, we will be adding our own
     // plugin that only runs custom AST transforms inside inline

packages/compat/tests/namespaced-app.test.ts

@@ -42,7 +42,7 @@ describe('namespaced app', function () {
       'implicit-modules have correct paths'
     );
     assertFile.matches(
-      /d\(["']@ef4\/namespaced-app\/app['"], function\(\)\{ return i\(['"]\.\.\/\.\.\/app"\);\}\);/,
+      /d\(["']@ef4\/namespaced-app\/app['"], function\(\)\{ return i\(['"]\.\.\/\.\.\/app\.js"\);\}\);/,
       `app's own modules are correct`
     );
   });

packages/compat/tests/resolver.test.ts

@@ -3,7 +3,7 @@ import { join, dirname } from 'path';
 import Options, { optionsWithDefaults } from '../src/options';
 import sortBy from 'lodash/sortBy';
 import { tmpdir } from 'os';
-import { TemplateCompiler, throwOnWarnings } from '@embroider/core';
+import { NodeTemplateCompiler, throwOnWarnings } from '@embroider/core';
 import { emberTemplateCompilerPath } from '@embroider/test-support';
 import { Options as AdjustImportsOptions } from '@embroider/core/src/babel-plugin-adjust-imports';
 import Resolver from '../src/resolver';
@@ -45,12 +45,7 @@ describe('compat-resolver', function () {
         otherOptions.adjustImportsImports
       ),
     });
-    let compiler = new TemplateCompiler({
-      compilerPath,
-      resolver,
-      EmberENV,
-      plugins,
-    });
+    let compiler = new NodeTemplateCompiler({ compilerPath, resolver, EmberENV, plugins });
     return function (relativePath: string, contents: string) {
       let moduleName = givenFile(relativePath);
       let { dependencies } = compiler.precompile(moduleName, contents);

packages/compat/tests/route-split.test.ts

@@ -113,7 +113,7 @@ describe('splitAtRoutes', function () {
     });
 
     it('dynamically imports the route entrypoint from the main entrypoint', function () {
-      expectFile('./assets/my-app.js').matches('import("./_route_/people")');
+      expectFile('./assets/my-app.js').matches('import("./_route_/people.js")');
     });
 
     it('has split controllers in route entrypoint', function () {
@@ -292,7 +292,7 @@ describe('splitAtRoutes', function () {
     });
 
     it('dynamically imports the route entrypoint from the main entrypoint', function () {
-      expectFile('./assets/my-app.js').matches('import("./_route_/people")');
+      expectFile('./assets/my-app.js').matches('import("./_route_/people.js")');
     });
 
     it('has split controllers in route entrypoint', function () {

packages/compat/tests/stage1.test.ts

@@ -116,7 +116,7 @@ describe('stage1 build', function () {
 
     test('addon metadata', function () {
       let assertMeta = expectFile('node_modules/my-addon/package.json').json('ember-addon');
-      assertMeta.get('app-js').equals('_app_'); // should have app-js metadata
+      assertMeta.get('app-js').deepEquals({ './components/hello-world.js': './_app_/components/hello-world.js' }); // should have app-js metadata
       assertMeta
         .get('implicit-modules')
         .includes('./components/hello-world', 'staticAddonTrees is off so we should include the component implicitly');