fix: spec files cant find .env variable on CI (#2514)

* bugfix: add env import to spec files correctly * 🔧 (ci.yml): fix indentation for consistency and readability in CI workflow * ci: add inputs to workflow * bugfix: improve flaky test general bugs to run smoothier on CI * ci: add more shards to improve tests on CI
langflow-ai · Jul 4, 2024 · fb21fdd · fb21fdd
1 parent be21f2f
commit fb21fdd
Show file tree

Hide file tree

Showing 21 changed files with 216 additions and 124 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,90 +8,99 @@ on:
         description: "(Optional) Branch to checkout"
         required: false
         type: string
+      openai_api_key:
+        description: "OpenAI API Key"
+        required: false
+        type: string
+      store_api_key:
+        description: "Store API Key"
+        required: false
+        type: string
   pull_request:
 
+
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-    path-filter:
-        name: Filter Paths
-        runs-on: ubuntu-latest
-        outputs:
-            python: ${{ steps.filter.outputs.python }}
-            frontend: ${{ steps.filter.outputs.frontend }}
-            docs: ${{ steps.filter.outputs.docs }}
-            tests: ${{ steps.filter.outputs.tests }}
-        steps:
-        - name: Checkout code
-          uses: actions/checkout@v4
-          with:
-            ref: ${{ inputs.branch || github.ref }}
-        - name: Filter Paths
-          id: filter
-          uses: dorny/paths-filter@v3
-          with:
-            filters: |
-              python:
-                - "src/backend/**"
-                - "src/backend/**.py"
-                - "pyproject.toml"
-                - "poetry.lock"
-                - "**/python_test.yml"
-              tests:
-                - "tests/**"
-                - "src/frontend/tests/**"
-              frontend:
-                - "src/frontend/**"
-                - "**/typescript_test.yml"
-              docs:
-                - "docs/**"
-
-    test-backend:
-        needs: path-filter
-        name: Run Backend Tests
-        if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
-        uses: ./.github/workflows/python_test.yml
-
-
-
-    test-frontend:
-      needs: path-filter
-      name: Run Frontend Tests
-      if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
-      uses: ./.github/workflows/typescript_test.yml
+  path-filter:
+    name: Filter Paths
+    runs-on: ubuntu-latest
+    outputs:
+      python: ${{ steps.filter.outputs.python }}
+      frontend: ${{ steps.filter.outputs.frontend }}
+      docs: ${{ steps.filter.outputs.docs }}
+      tests: ${{ steps.filter.outputs.tests }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch || github.ref }}
+      - name: Filter Paths
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            python:
+              - "src/backend/**"
+              - "src/backend/**.py"
+              - "pyproject.toml"
+              - "poetry.lock"
+              - "**/python_test.yml"
+            tests:
+              - "tests/**"
+              - "src/frontend/tests/**"
+            frontend:
+              - "src/frontend/**"
+              - "**/typescript_test.yml"
+            docs:
+              - "docs/**"
 
+  test-backend:
+    needs: path-filter
+    name: Run Backend Tests
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    uses: ./.github/workflows/python_test.yml
 
-    lint-backend:
-        needs: path-filter
-        if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
-        name: Lint Backend
-        uses: ./.github/workflows/lint-py.yml
-        # Run only if there are python files changed
+  test-frontend:
+    needs: path-filter
+    name: Run Frontend Tests
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.frontend == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    uses: ./.github/workflows/typescript_test.yml
+    secrets:
+      OPENAI_API_KEY: "${{ secrets.OPENAI_API_KEY }}"
+      STORE_API_KEY: "${{ secrets.STORE_API_KEY }}"
 
-    test-docs-build:
-        needs: path-filter
-        if: ${{ needs.path-filter.outputs.docs == 'true' }}
-        name: Test Docs Build
-        uses: ./.github/workflows/docs_test.yml
+  lint-backend:
+    needs: path-filter
+    if: ${{ needs.path-filter.outputs.python == 'true' || needs.path-filter.outputs.tests == 'true' }}
+    name: Lint Backend
+    uses: ./.github/workflows/lint-py.yml
+    # Run only if there are python files changed
 
+  test-docs-build:
+    needs: path-filter
+    if: ${{ needs.path-filter.outputs.docs == 'true' }}
+    name: Test Docs Build
+    uses: ./.github/workflows/docs_test.yml
 
-    # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
-    ci_success:
-        name: "CI Success"
-        needs: [test-backend, test-frontend, lint-backend, test-docs-build]
-        if: |
-            always()
-        runs-on: ubuntu-latest
-        env:
-            JOBS_JSON: ${{ toJSON(needs) }}
-            RESULTS_JSON: ${{ toJSON(needs.*.result) }}
-            EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}}
-        steps:
-        - name: "CI Success"
-          run: |
-            echo $JOBS_JSON
-            echo $RESULTS_JSON
-            echo "Exiting with $EXIT_CODE"
-            exit $EXIT_CODE
+  # https://github.com/langchain-ai/langchain/blob/master/.github/workflows/check_diffs.yml
+  ci_success:
+    name: "CI Success"
+    needs: [test-backend, test-frontend, lint-backend, test-docs-build]
+    if: |
+      always()
+    runs-on: ubuntu-latest
+    env:
+      JOBS_JSON: ${{ toJSON(needs) }}
+      RESULTS_JSON: ${{ toJSON(needs.*.result) }}
+      EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}}
+    steps:
+      - name: "CI Success"
+        run: |
+          echo $JOBS_JSON
+          echo $RESULTS_JSON
+          echo "Exiting with $EXIT_CODE"
+          exit $EXIT_CODE
diff --git a/.github/workflows/typescript_test.yml b/.github/workflows/typescript_test.yml
@@ -2,14 +2,18 @@ name: Run Frontend Tests
 
 on:
   workflow_call:
+    secrets:
+      OPENAI_API_KEY:
+        required: true
+      STORE_API_KEY:
+        required: true
   workflow_dispatch:
     inputs:
       branch:
         description: "(Optional) Branch to checkout"
         required: false
         type: string
 
-
 env:
   POETRY_VERSION: "1.8.3"
   NODE_VERSION: "21"
@@ -25,11 +29,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
-        shardTotal: [10]
+        shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+        shardTotal: [14]
     env:
-      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      STORE_API_KEY: ${{ secrets.STORE_API_KEY }}
+      OPENAI_API_KEY: ${{ inputs.openai_api_key || secrets.OPENAI_API_KEY }}
+      STORE_API_KEY: ${{ inputs.store_api_key || secrets.STORE_API_KEY }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

diff --git a/src/frontend/tests/end-to-end/Basic Prompting.spec.ts b/src/frontend/tests/end-to-end/Basic Prompting.spec.ts
@@ -1,11 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("Basic Prompting (Hello, World)", async ({ page }) => {
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

diff --git a/src/frontend/tests/end-to-end/Blog Writer.spec.ts b/src/frontend/tests/end-to-end/Blog Writer.spec.ts
@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Blog Writer", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Blog Writer", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

diff --git a/src/frontend/tests/end-to-end/Document QA.spec.ts b/src/frontend/tests/end-to-end/Document QA.spec.ts
@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Document QA", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Document QA", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

diff --git a/src/frontend/tests/end-to-end/Memory Chatbot.spec.ts b/src/frontend/tests/end-to-end/Memory Chatbot.spec.ts
@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
 import path from "path";
 
 test("Memory Chatbot", async ({ page }) => {
@@ -7,6 +8,10 @@ test("Memory Chatbot", async ({ page }) => {
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(2000);
 

diff --git a/src/frontend/tests/end-to-end/chatInputOutputUser-shard-0.spec.ts b/src/frontend/tests/end-to-end/chatInputOutputUser-shard-0.spec.ts
@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must be able to send an image on chat", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

diff --git a/src/frontend/tests/end-to-end/chatInputOutputUser-shard-1.spec.ts b/src/frontend/tests/end-to-end/chatInputOutputUser-shard-1.spec.ts
@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must be able to see output inspection", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

diff --git a/src/frontend/tests/end-to-end/chatInputOutputUser-shard-2.spec.ts b/src/frontend/tests/end-to-end/chatInputOutputUser-shard-2.spec.ts
@@ -4,14 +4,15 @@ import { readFileSync } from "fs";
 import path from "path";
 
 test("user must interact with chat with Input/Output", async ({ page }) => {
-  if (!process.env.CI) {
-    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
-  }
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
 
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
 
   await page.waitForTimeout(1000);

diff --git a/src/frontend/tests/end-to-end/decisionFlow.spec.ts b/src/frontend/tests/end-to-end/decisionFlow.spec.ts
@@ -1,10 +1,17 @@
 import { expect, test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should create a flow with decision", async ({ page }) => {
   test.skip(
     !process?.env?.OPENAI_API_KEY,
     "OPENAI_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.locator("span").filter({ hasText: "My Collection" }).isVisible();
   await page.waitForTimeout(2000);

diff --git a/src/frontend/tests/end-to-end/deleteComponents.spec.ts b/src/frontend/tests/end-to-end/deleteComponents.spec.ts
@@ -1,10 +1,17 @@
 import { test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should delete a component", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
+
   await page.goto("/");
   await page.waitForTimeout(1000);
 

diff --git a/src/frontend/tests/end-to-end/deleteFlows.spec.ts b/src/frontend/tests/end-to-end/deleteFlows.spec.ts
@@ -1,10 +1,16 @@
 import { test } from "@playwright/test";
+import * as dotenv from "dotenv";
+import path from "path";
 
 test("should delete a flow", async ({ page }) => {
   test.skip(
     !process?.env?.STORE_API_KEY,
     "STORE_API_KEY required to run this test",
   );
+
+  if (!process.env.CI) {
+    dotenv.config({ path: path.resolve(__dirname, "../../.env") });
+  }
   await page.goto("/");
   await page.waitForTimeout(1000);