diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 01e299d315..da2f4880bc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,6 +37,29 @@ jobs:
       - run: pnpm install --no-lockfile
       - run: pnpm --filter "@embroider/*" test
 
+  legacy-macros:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node: ["12.22.12", "14.21.3", "16.20.2", "18.20.8", "20.19.5", "22.20.0", "24.9.0"]
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22 # this is only used for the pnpm install, the user-node-version above will be used in tests
+          cache: pnpm
+      - run: pnpm install --no-lockfile
+      - name: downgrade Node Version in .npmrc
+        run: printf "auto-install-peers=false\nuse-node-version=${{matrix.node}}" > .npmrc
+      - run: pnpm node -v
+      - run: cat .npmrc
+      - run: pnpm --filter "@embroider/macros" test:smoke
+
   preflight:
     runs-on: ubuntu-latest
     timeout-minutes: 5
diff --git a/packages/macros/package.json b/packages/macros/package.json
index 5faaebcb61..ed86e19d55 100644
--- a/packages/macros/package.json
+++ b/packages/macros/package.json
@@ -26,7 +26,8 @@
     "src/**/*.js.map"
   ],
   "scripts": {
-    "test": "vitest"
+    "test": "vitest",
+    "test:smoke": "node smoke-test.js"
   },
   "dependencies": {
     "@embroider/shared-internals": "workspace:*",
diff --git a/packages/macros/smoke-test.js b/packages/macros/smoke-test.js
new file mode 100644
index 0000000000..ff7b450628
--- /dev/null
+++ b/packages/macros/smoke-test.js
@@ -0,0 +1,34 @@
+console.log(process.version);
+
+var babel = require("@babel/core");
+
+const { join } = require('path');
+
+const { MacrosConfig } = require('./src/node');
+
+let config = MacrosConfig.for({}, __dirname);
+
+let response = babel.transform(`
+import { macroCondition } from '@embroider/macros';
+export default function() {
+  if (macroCondition(true)) {
+    return 'alpha';
+  } else {
+    return 'beta';
+  }
+}
+`, {
+  filename: join(__dirname, 'sample.js'),
+  configFile: false,
+  plugins: config.babelPluginConfig(),
+});
+
+
+if(response.code.trim() !== `export default function () {
+  {
+    return 'alpha';
+  }
+}`.trim()) {
+  console.error(response.code);
+  throw new Error("smoke test failed");
+}
diff --git a/packages/macros/tests/babel/import-sync.test.ts b/packages/macros/tests/babel/import-sync.test.ts
index 03546bc517..5a40c58928 100644
--- a/packages/macros/tests/babel/import-sync.test.ts
+++ b/packages/macros/tests/babel/import-sync.test.ts
@@ -66,17 +66,19 @@ describe('importSync', function () {
         import * as _importSync20 from "../../README";
         import * as _importSync40 from "../../node_modules";
         import * as _importSync60 from "../../package";
-        import * as _importSync80 from "../../src";
-        import * as _importSync00 from "../../tests";
-        import * as _importSync100 from "../../vitest.config";
+        import * as _importSync80 from "../../smoke-test";
+        import * as _importSync00 from "../../src";
+        import * as _importSync100 from "../../tests";
+        import * as _importSync120 from "../../vitest.config";
         function getFile(file) {
           return {
             "README": esc(_importSync20),
             "node_modules": esc(_importSync40),
             "package": esc(_importSync60),
-            "src": esc(_importSync80),
-            "tests": esc(_importSync00),
-            "vitest.config": esc(_importSync100)
+            "smoke-test": esc(_importSync80),
+            "src": esc(_importSync00),
+            "tests": esc(_importSync100),
+            "vitest.config": esc(_importSync120)
           }[file].default;
         }"
       `);