diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 01e299d315..86b5752f7b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,7 +3,7 @@ name: CI
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 7 * * *' # daily, 7am
+    - cron: "0 7 * * *" # daily, 7am
 
   push:
     branches:
@@ -22,8 +22,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node: ['20', '22', '24']
-        os: [ubuntu-latest, 'macos-latest'] # TODO add windows here
+        node: ["20", "22", "24"]
+        os: [ubuntu-latest, "macos-latest"] # TODO add windows here
 
     steps:
       - uses: actions/checkout@v4
@@ -37,6 +37,29 @@ jobs:
       - run: pnpm install --no-lockfile
       - run: pnpm --filter "@embroider/*" test
 
+  legacy-macros:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node: ["12.22.12", "14.21.3", "16.20.2", "18.20.8", "20.19.5", "22.20.0", "24.9.0"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22 # this is only used for the pnpm install, the user-node-version above will be used in tests
+          cache: pnpm
+      - run: pnpm install --no-lockfile
+      - name: downgrade Node Version in .npmrc
+        run: printf "auto-install-peers=false\nuse-node-version=${{matrix.node}}" > .npmrc
+      - run: pnpm node -v
+      - run: cat .npmrc
+      - run: pnpm --filter "@embroider/macros" test:smoke
+
   preflight:
     runs-on: ubuntu-latest
     timeout-minutes: 5
@@ -55,9 +78,9 @@ jobs:
           echo "matrix=$matrix_json" >> $GITHUB_OUTPUT
 
   test:
-    needs: ['preflight']
+    needs: ["preflight"]
     name: ${{ matrix.name }}
-    runs-on: '${{ matrix.os }}-latest'
+    runs-on: "${{ matrix.os }}-latest"
     timeout-minutes: 60
 
     strategy:
diff --git a/packages/macros/package.json b/packages/macros/package.json
index 5faaebcb61..ed86e19d55 100644
--- a/packages/macros/package.json
+++ b/packages/macros/package.json
@@ -26,7 +26,8 @@
     "src/**/*.js.map"
   ],
   "scripts": {
-    "test": "vitest"
+    "test": "vitest",
+    "test:smoke": "node smoke-test.js"
   },
   "dependencies": {
     "@embroider/shared-internals": "workspace:*",
diff --git a/packages/macros/smoke-test.js b/packages/macros/smoke-test.js
new file mode 100644
index 0000000000..ff7b450628
--- /dev/null
+++ b/packages/macros/smoke-test.js
@@ -0,0 +1,34 @@
+console.log(process.version);
+
+var babel = require("@babel/core");
+
+const { join } = require('path');
+
+const { MacrosConfig } = require('./src/node');
+
+let config = MacrosConfig.for({}, __dirname);
+
+let response = babel.transform(`
+import { macroCondition } from '@embroider/macros';
+export default function() {
+  if (macroCondition(true)) {
+    return 'alpha';
+  } else {
+    return 'beta';
+  }
+}
+`, {
+  filename: join(__dirname, 'sample.js'),
+  configFile: false,
+  plugins: config.babelPluginConfig(),
+});
+
+
+if(response.code.trim() !== `export default function () {
+  {
+    return 'alpha';
+  }
+}`.trim()) {
+  console.error(response.code);
+  throw new Error("smoke test failed");
+}
diff --git a/packages/macros/src/babel/app-ember-satisfies.ts b/packages/macros/src/babel/app-ember-satisfies.ts
index 88700f39cc..7d54eb1d99 100644
--- a/packages/macros/src/babel/app-ember-satisfies.ts
+++ b/packages/macros/src/babel/app-ember-satisfies.ts
@@ -4,9 +4,9 @@ import type State from './state';
 import { satisfies, coerce } from 'semver';
 import error from './error';
 import { assertArray } from './evaluate-json';
-import { pathToFileURL } from 'node:url';
-import { createRequire } from 'node:module';
-import { dirname } from 'node:path';
+import { pathToFileURL } from 'url';
+import { createRequire } from 'module';
+import { dirname } from 'path';
 import findUp from 'find-up';
 
 const packageName = 'ember-source';
diff --git a/packages/macros/tests/babel/import-sync.test.ts b/packages/macros/tests/babel/import-sync.test.ts
index 03546bc517..5a40c58928 100644
--- a/packages/macros/tests/babel/import-sync.test.ts
+++ b/packages/macros/tests/babel/import-sync.test.ts
@@ -66,17 +66,19 @@ describe('importSync', function () {
         import * as _importSync20 from "../../README";
         import * as _importSync40 from "../../node_modules";
         import * as _importSync60 from "../../package";
-        import * as _importSync80 from "../../src";
-        import * as _importSync00 from "../../tests";
-        import * as _importSync100 from "../../vitest.config";
+        import * as _importSync80 from "../../smoke-test";
+        import * as _importSync00 from "../../src";
+        import * as _importSync100 from "../../tests";
+        import * as _importSync120 from "../../vitest.config";
         function getFile(file) {
           return {
             "README": esc(_importSync20),
             "node_modules": esc(_importSync40),
             "package": esc(_importSync60),
-            "src": esc(_importSync80),
-            "tests": esc(_importSync00),
-            "vitest.config": esc(_importSync100)
+            "smoke-test": esc(_importSync80),
+            "src": esc(_importSync00),
+            "tests": esc(_importSync100),
+            "vitest.config": esc(_importSync120)
           }[file].default;
         }"
       `);