[Manual][Backport 2.x] [Tests] Add Github workflow for Test Orchestrator in FT Repo to run cypress tests within Dashboards repo

.github/workflows/dashboards_cypress_workflow.yml

@@ -0,0 +1,143 @@
+name: Orchestrator cypress workflow
+run-name: dashboards_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator.
+# Trigger on dispatch event sent from FT repo orchestrator
+on:
+  workflow_dispatch:
+    inputs:
+      test_repo:
+        description: 'Cypress test repo'
+        default: ''
+        required: false
+        type: string
+      test_branch:
+        description: 'Cypress test branch (default: source branch)'
+        required: false
+        type: string
+      specs:
+        description: 'Test group to run'
+        required: false
+        type: string
+      build_id:
+        description: 'Build Id'
+        required: false
+        type: string
+      OS_URL:
+        description: 'OpenSearch release artifact'
+        required: false
+        type: string
+      OSD_URL:
+        description: 'OpenSearch Dashboards release artifact'
+        required: false
+        type: string
+      UNIQUE_ID:
+        description: 'Unique Id for the workflow execution'
+        required: true
+        type: string
+      SECURITY_ENABLED:
+        required: false
+        type: string
+
+env:
+  TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }}
+  TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}"
+  OSD_PATH: 'osd'
+  CYPRESS_BROWSER: 'chromium'
+  JOB_ID: ${{ inputs.UNIQUE_ID}}
+  OPENSEARCH:  ${{ inputs.OS_URL != '' && inputs.OS_URL  || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }}
+  DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL  || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }}
+  OPENSEARCH_DIR: 'cypress/opensearch'
+  DASHBOARDS_DIR: 'cypress/opensearch-dashboards'
+  SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }}
+  SPEC: 'cypress/integration/core_opensearch_dashboards/*'
+
+jobs:
+  cypress-tests:
+    runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
+    env:
+      # prevents extra Cypress installation progress messages
+      CI: 1
+      # avoid warnings like "tput: No value for $TERM and no -T specified"
+      TERM: xterm
+    name: Run cypress tests ${{ inputs.UNIQUE_ID}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: ./${{ env.OSD_PATH }}
+          repository: ${{ env.TEST_REPO }}
+          ref: '${{ env.TEST_BRANCH }}'
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: './${{ env.OSD_PATH }}/.nvmrc'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g [email protected]
+          yarn config set network-timeout 1000000 -g
+
+      - name: Get Cypress version
+        id: cypress_version
+        run: |
+          echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
+
+      - name: Cache Cypress
+        id: cache-cypress
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/Cypress
+          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
+        env:
+          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
+      - run: npx cypress cache list
+      - run: npx cypress cache path
+
+      - name: Get package version (Linux)
+        run: |
+          cd ${{ env.OSD_PATH }}
+          echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV
+
+      - name: Run bootstrap
+        run: |
+          cd ${{ env.OSD_PATH }}
+          yarn osd bootstrap
+
+      - name: Download and extract Opensearch artifacts
+        run: |
+          CWD=$(pwd)
+          mkdir -p $CWD/${{ env.OPENSEARCH_DIR }}
+          source ${{ env.OSD_PATH }}/scripts/common/utils.sh
+          open_artifact $CWD/${{ env.OPENSEARCH_DIR }} ${{ env.OPENSEARCH }}
+
+      - name: Download and extract Opensearch Dashboards artifacts
+        run: |
+          CWD=$(pwd)
+          mkdir -p $CWD/${{ env.DASHBOARDS_DIR }}
+          source ${{ env.OSD_PATH }}/scripts/common/utils.sh
+          open_artifact $CWD/${{ env.DASHBOARDS_DIR }} ${{ env.DASHBOARDS }}
+
+      - name: Run Cypress tests 
+        run: |
+          source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh
+          run_dashboards_cypress_tests  
+
+      # Screenshots are only captured on failures
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: osd-cypress-screenshots
+          path: ${{ env.OSD_PATH }}/cypress/screenshots
+          retention-days: 1
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: osd-cypress-videos
+          path: ${{ env.OSD_PATH }}/cypress/videos
+          retention-days: 1

CHANGELOG.md

@@ -92,6 +92,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [CI] Enable inputs for manually triggered Cypress test jobs ([#5134](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5134))
 - [CI] Replace usage of deprecated `set-output` in workflows ([#5340](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5340))
 - [Chore] Add `--security` for `opensearch snapshot` and `opensearch_dashboards` to configure local setup with the security plugin ([#5451](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5451))
+- [Tests] Add Github workflow for Test Orchestrator in FT Repo to run cypress tests within Dashboards repo ([#5725](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5725))
 - [Chore] Updates default dev environment security credentials ([#5736](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5736))
 
 ### 📝 Documentation

cypress.config.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+const { defineConfig } = require('cypress');
+
+module.exports = defineConfig({
+  defaultCommandTimeout: 60000,
+  requestTimeout: 60000,
+  responseTimeout: 60000,
+  baseUrl: 'http://localhost:5601',
+  viewportWidth: 2000,
+  viewportHeight: 1320,
+  env: {
+    openSearchUrl: 'http://localhost:9200',
+    SECURITY_ENABLED: false,
+    AGGREGATION_VIEW: false,
+    username: 'admin',
+    password: 'myStrongPassword123!',
+    ENDPOINT_WITH_PROXY: false,
+    MANAGED_SERVICE_ENDPOINT: false,
+    VISBUILDER_ENABLED: true,
+    DATASOURCE_MANAGEMENT_ENABLED: false,
+    ML_COMMONS_DASHBOARDS_ENABLED: true,
+    WAIT_FOR_LOADER_BUFFER_MS: 0,
+  },
+});

cypress/integration/core_opensearch_dashboards/dashboard_sanity_test_spec.js

@@ -0,0 +1,164 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  CommonUI,
+  MiscUtils,
+} from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+
+/**
+ * dashboard_sample_data test suite description:
+ * 1) Visit the home page of opensearchdashboard, check key UI elements display
+ * 2) add sample data of eCommerce, flights, web logs from tutorial page
+ * 3) check each sample data dashboard key UI elements display
+ */
+export function dashboardSanityTests() {
+  const commonUI = new CommonUI(cy);
+  const miscUtils = new MiscUtils(cy);
+  const baseURL = new URL(Cypress.config().baseUrl);
+  // remove trailing slash
+  const path = baseURL.pathname.replace(/\/$/, '');
+
+  describe('dashboard sample data validation', () => {
+    before(() => {});
+
+    after(() => {});
+
+    describe('checking home page', () => {
+      before(() => {
+        // Go to the home page
+        miscUtils.visitPage('app/home#');
+        cy.window().then((win) => win.localStorage.setItem('home:welcome:show', false));
+        cy.reload(true);
+      });
+
+      after(() => {
+        cy.window().then((win) => win.localStorage.removeItem('home:welcome:show'));
+      });
+
+      it('checking opensearch_dashboards_overview display', () => {
+        // Check that opensearch_dashboards_overview is visable
+        commonUI.checkElementExists(`a[href="${path}/app/opensearch_dashboards_overview"]`, 1);
+      });
+
+      it('checking tutorial_directory display', () => {
+        // Check that tutorial_directory is visable
+        commonUI.checkElementExists(`a[href="${path}/app/home#/tutorial_directory"]`, 2);
+      });
+
+      it('checking management display', () => {
+        // Check that management is visable
+        commonUI.checkElementExists(`a[href="${path}/app/management"]`, 1);
+      });
+
+      it('checking dev_tools display', () => {
+        // Check that dev_tools is visable
+        commonUI.checkElementExists(`a[href="${path}/app/dev_tools#/console"]`, 2);
+      });
+
+      it('settings display', () => {
+        // Check that settings is visable
+        commonUI.checkElementExists(
+          `a[href="${path}/app/management/opensearch-dashboards/settings#defaultRoute"]`,
+          1
+        );
+      });
+
+      it('checking feature_directory display', () => {
+        // Check that feature_directory is visable
+        commonUI.checkElementExists(`a[href="${path}/app/home#/feature_directory"]`, 1);
+      });
+
+      it('checking navigation display', () => {
+        // Check that navigation is visable
+        commonUI.checkElementExists('button[data-test-subj="toggleNavButton"]', 1);
+      });
+
+      it('checking Help menu display', () => {
+        // Check that Help menu is visable
+        commonUI.checkElementExists('button[aria-label="Help menu"]', 1);
+      });
+    });
+
+    describe('checking Dev Tools', () => {
+      before(() => {
+        // Go to the Dev Tools page
+        miscUtils.visitPage('app/dev_tools#/console');
+      });
+
+      after(() => {});
+
+      it('checking welcome panel display', () => {
+        commonUI.checkElementExists('div[data-test-subj="welcomePanel"]', 1);
+      });
+
+      it('checking dismiss button display', () => {
+        commonUI.checkElementExists('button[data-test-subj="help-close-button"]', 1);
+      });
+
+      it('checking console input area display', () => {
+        commonUI.checkElementExists('div[data-test-subj="request-editor"]', 1);
+      });
+
+      it('checking console output area display', () => {
+        commonUI.checkElementExists('div[data-test-subj="response-editor"]', 1);
+      });
+    });
+
+    describe('adding sample data', () => {
+      before(() => {
+        miscUtils.addSampleData();
+      });
+
+      after(() => {
+        miscUtils.removeSampleData();
+      });
+
+      it('checking ecommerce dashboards displayed', () => {
+        miscUtils.viewData('ecommerce');
+        commonUI.checkElementContainsValue(
+          'span[title="[eCommerce] Revenue Dashboard"]',
+          1,
+          '\\[eCommerce\\] Revenue Dashboard'
+        );
+        commonUI.checkElementContainsValue(
+          'div[data-test-subj="markdownBody"] > h3',
+          1,
+          'Sample eCommerce Data'
+        );
+      });
+
+      it('checking flights dashboards displayed', () => {
+        miscUtils.viewData('flights');
+        commonUI.checkElementContainsValue(
+          'span[title="[Flights] Global Flight Dashboard"]',
+          1,
+          '\\[Flights\\] Global Flight Dashboard'
+        );
+        commonUI.checkElementContainsValue(
+          'div[data-test-subj="markdownBody"] > h3',
+          1,
+          'Sample Flight data'
+        );
+      });
+
+      it('checking web logs dashboards displayed', () => {
+        miscUtils.viewData('logs');
+        commonUI.checkElementContainsValue(
+          'span[title="[Logs] Web Traffic"]',
+          1,
+          '\\[Logs\\] Web Traffic'
+        );
+        commonUI.checkElementContainsValue(
+          'div[data-test-subj="markdownBody"] > h3',
+          1,
+          'Sample Logs Data'
+        );
+      });
+    });
+  });
+}
+
+dashboardSanityTests();

package.json

@@ -75,7 +75,10 @@
     "docs:acceptApiChanges": "scripts/use_node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept",
     "osd:bootstrap": "scripts/use_node scripts/build_ts_refs && scripts/use_node scripts/register_git_hook",
     "spec_to_console": "scripts/use_node scripts/spec_to_console",
-    "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\""
+    "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"",
+    "cypress:run-without-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=false",
+    "cypress:run-with-security": "env TZ=America/Los_Angeles NO_COLOR=1 cypress run --headless --env SECURITY_ENABLED=true,openSearchUrl=https://localhost:9200,WAIT_FOR_LOADER_BUFFER_MS=500"
+
   },
   "repository": {
     "type": "git",
@@ -229,7 +232,9 @@
     "type-detect": "^4.0.8",
     "uuid": "3.3.2",
     "whatwg-fetch": "^3.0.0",
-    "yauzl": "^2.10.0"
+    "yauzl": "^2.10.0",
+    "@opensearch-dashboards-test/opensearch-dashboards-test-library": "https://github.com/opensearch-project/opensearch-dashboards-test-library/archive/refs/tags/1.0.6.tar.gz"
+
   },
   "devDependencies": {
     "@babel/core": "^7.22.9",
@@ -358,6 +363,7 @@
     "chromedriver": "^107.0.3",
     "classnames": "2.3.1",
     "compare-versions": "3.5.1",
+    "cypress": "9.5.4",
     "d3": "3.5.17",
     "d3-cloud": "1.2.5",
     "dedent": "^0.7.0",

scripts/bwctest_osd.sh

@@ -5,9 +5,9 @@
 
 set -e
 
-. scripts/bwc/utils.sh
-. scripts/bwc/opensearch_service.sh
-. scripts/bwc/opensearch_dashboards_service.sh
+. scripts/common/utils.sh
+. scripts/common/opensearch_service.sh
+. scripts/common/opensearch_dashboards_service.sh
 . scripts/bwc/generate_test_data.sh
 
 # For every release, add sample data and new version below: