diff --git a/package.json b/package.json
index c43ce6a47..4d0e1cff1 100644
--- a/package.json
+++ b/package.json
@@ -93,7 +93,7 @@
     "build:production": "GATSBY_NEWRELIC_ENV=production gatsby build",
     "build:staging": "GATSBY_NEWRELIC_ENV=staging gatsby build",
     "build:related-content": "BUILD_RELATED_CONTENT=true yarn run build:production",
-    "fetch-observability-packs": "CI=true node ./scripts/actions/fetch-observability-packs.js",
+    "fetch-observability-packs": "node ./scripts/actions/fetch-observability-packs.js",
     "develop": "gatsby develop",
     "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
     "start": "yarn run develop",
diff --git a/scripts/actions/fetch-observability-packs.js b/scripts/actions/fetch-observability-packs.js
index 0767e2695..0065e7d10 100644
--- a/scripts/actions/fetch-observability-packs.js
+++ b/scripts/actions/fetch-observability-packs.js
@@ -112,13 +112,13 @@ const main = async (query, url, token) => {
     console.log(
       'No packs were returned from the api, check the logs for errors.'
     );
-    if (process.env.CI) {
+    if (require.main === module) {
       process.exit(1);
     }
   }
 };
 
-if (process.env.CI) {
+if (require.main === module) {
   validateEnvVars();
   main(packQuery, NR_API_URL, NR_API_TOKEN);
 }