diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9dc6871f47b..6b17259af03 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,13 +7,25 @@ @DavidHuber-NOAA # Specific directory owners -/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA +# dev/ +dev/jobs/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul +dev/ush/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul +dev/workflow/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul +dev/test/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul +dev/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA @DavidHuber-NOAA +dev/ctests/ @TerrenceMcGuinness-NOAA @aerorahul /ecf/ @lgannoaa -/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA - # Specific file owners +# dev side scripts +dev/ush/compare_f90nml.py @WalterKolczynski-NOAA @aerorahul + +# Specific workflow scripts +dev/workflow/generate_workflows.sh @DavidHuber-NOAA +dev/workflow/build_compute.py @DavidHuber-NOAA @aerorahul +dev/workflow/build_*opts.yaml @DavidHuber-NOAA @aerorahul + # build scripts sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA @@ -67,7 +79,6 @@ jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA jobs/JGLOBAL_*SNOW* @jiaruidong2017 jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash -jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA # scripts scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA @@ -125,63 +136,62 @@ scripts/exglobal_snow_analysis.py @jiaruidong2017 scripts/exglobal_stage_ic.py @KateFriedman-NOAA # ush -WAM_XML_to_ASCII.pl -atmos_ensstat.sh -atmos_extractvars.sh @EricSinsky-NOAA -bash_utils.sh @WalterKolczynski-NOAA -calcanl_gfs.py @CoryMartin-NOAA -calcinc_gfs.py @CoryMartin-NOAA -compare_f90nml.py @WalterKolczynski-NOAA @aerorahul -detect_machine.sh @WalterKolczynski-NOAA -extractvars_tools.sh @EricSinsky-NOAA -file_utils.sh @WalterKolczynski-NOAA -forecast_det.sh @aerorahul @WalterKolczynski-NOAA -forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA -forecast_predet.sh @aerorahul @WalterKolczynski-NOAA -fv3gfs_remap_weights.sh -gaussian_sfcanl.sh @GeorgeGayno-NOAA -getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA -getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA -getgfsnctime @CoryMartin-NOAA -getncdimlen @CoryMartin-NOAA -gfs_bfr2gpk.sh @BoCui-NOAA -gfs_bufr.sh @BoCui-NOAA -gfs_bufr_netcdf.sh @BoCui-NOAA -gfs_sndp.sh @BoCui-NOAA -gfs_truncate_enkf.sh @CoryMartin-NOAA -global_savefits.sh -gsi_utils.py @CoryMartin-NOAA -interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA -interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA -jjob_header.sh @WalterKolczynski-NOAA -link_crtm_fix.sh @WalterKolczynski-NOAA -load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul -load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA -load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA -merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA -minmon_xtrct_*.pl @EdwardSafford-NOAA -module-setup.sh @WalterKolczynski-NOAA @aerorahul -ocnice_extractvars.sh @EricSinsky-NOAA -ozn_xtrct.sh @EdwardSafford-NOAA -parse-storm-type.pl -parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa -parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen -parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa -parsing_namelists_FV3_nest.sh @guoqing-noaa -parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang -parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash -parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa -preamble.sh @WalterKolczynski-NOAA -product_functions.sh @WalterKolczynski-NOAA @aerorahul -radmon_*.sh @EdwardSafford-NOAA -rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA -run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA -syndat_getjtbul.sh @JiayiPeng-NOAA -syndat_qctropcy.sh @JiayiPeng-NOAA -tropcy_relocate.sh @JiayiPeng-NOAA -tropcy_relocate_extrkr.sh @JiayiPeng-NOAA -wave_*.sh @JessicaMeixner-NOAA @sbanihash -regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA +ush/WAM_XML_to_ASCII.pl +ush/atmos_ensstat.sh +ush/atmos_extractvars.sh @EricSinsky-NOAA +ush/bash_utils.sh @WalterKolczynski-NOAA +ush/calcanl_gfs.py @CoryMartin-NOAA +ush/calcinc_gfs.py @CoryMartin-NOAA +ush/detect_machine.sh @WalterKolczynski-NOAA +ush/extractvars_tools.sh @EricSinsky-NOAA +ush/file_utils.sh @WalterKolczynski-NOAA +ush/forecast_det.sh @aerorahul @WalterKolczynski-NOAA +ush/forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA +ush/forecast_predet.sh @aerorahul @WalterKolczynski-NOAA +ush/fv3gfs_remap_weights.sh +ush/gaussian_sfcanl.sh @GeorgeGayno-NOAA +ush/getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +ush/getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA +ush/getgfsnctime @CoryMartin-NOAA +ush/getncdimlen @CoryMartin-NOAA +ush/gfs_bfr2gpk.sh @BoCui-NOAA +ush/gfs_bufr.sh @BoCui-NOAA +ush/gfs_bufr_netcdf.sh @BoCui-NOAA +ush/gfs_sndp.sh @BoCui-NOAA +ush/gfs_truncate_enkf.sh @CoryMartin-NOAA +ush/global_savefits.sh +ush/gsi_utils.py @CoryMartin-NOAA +ush/interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +ush/interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA +ush/jjob_header.sh @WalterKolczynski-NOAA +ush/link_crtm_fix.sh @WalterKolczynski-NOAA +ush/load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul +ush/load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA +ush/load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA +ush/merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA +ush/minmon_xtrct_*.pl @EdwardSafford-NOAA +ush/module-setup.sh @WalterKolczynski-NOAA @aerorahul +ush/ocnice_extractvars.sh @EricSinsky-NOAA +ush/ozn_xtrct.sh @EdwardSafford-NOAA +ush/parse-storm-type.pl +ush/parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +ush/parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen +ush/parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +ush/parsing_namelists_FV3_nest.sh @guoqing-noaa +ush/parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang +ush/parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash +ush/parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa +ush/preamble.sh @WalterKolczynski-NOAA +ush/product_functions.sh @WalterKolczynski-NOAA @aerorahul +ush/radmon_*.sh @EdwardSafford-NOAA +ush/rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA +ush/run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA +ush/syndat_getjtbul.sh @JiayiPeng-NOAA +ush/syndat_qctropcy.sh @JiayiPeng-NOAA +ush/tropcy_relocate.sh @JiayiPeng-NOAA +ush/tropcy_relocate_extrkr.sh @JiayiPeng-NOAA +ush/wave_*.sh @JessicaMeixner-NOAA @sbanihash +ush/regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA # ush/python ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA @@ -212,7 +222,3 @@ ush/python/pygfs/ufswm/ufs.py @aerorahul ush/python/pygfs/utils/__init__.py @aerorahul ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA -# Specific workflow scripts -workflow/generate_workflows.sh @DavidHuber-NOAA -workflow/build_compute.py @DavidHuber-NOAA @aerorahul -workflow/build_*opts.yaml @DavidHuber-NOAA @aerorahul diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index 43ec49d7798..450d2546b8e 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -1,68 +1,73 @@ name: CI Unit Tests on: [pull_request, push, workflow_dispatch] +env: + ROCOTO_VERSION: "1.3.7" + jobs: ci_pytest: - runs-on: ubuntu-22.04 - name: Run unit tests on CI system - permissions: - checks: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 + runs-on: ubuntu-22.04 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11.8 + steps: + - name: Checkout global-workflow + uses: actions/checkout@v4 + with: + path: global-workflow - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl - python -m pip install --upgrade pip - pip install pytest - pip install wxflow - pip install wget + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11.8 - - name: Cache Rocoto Install - uses: actions/cache@v4 - with: - path: ~/rocoto - key: ${{ runner.os }}-rocoto-${{ hashFiles('**/ci-unit_tests.yaml') }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl + python -m pip install --upgrade pip + python -m pip install -r global-workflow/dev/workflow/requirements.txt + pip install pytest + pip install wget - - name: Install Rocoto - run: | - if [ ! -d "$HOME/rocoto/bin" ]; then - git clone https://github.com/christopherwharrop/rocoto.git $HOME/rocoto - cd $HOME/rocoto - ./INSTALL - fi - echo "$HOME/rocoto/bin" >> $GITHUB_PATH + - name: Cache Rocoto + id: cache-rocoto + uses: actions/cache@v4 + with: + path: | + rocoto-${{ env.ROCOTO_VERSION }} + key: ${{ runner.os }}-rocoto-${{ env.ROCOTO_VERSION }} - - name: Run tests - shell: bash - run: | - sudo mkdir -p /scratch1/NCEPDEV - sudo mkdir -p /scratch1/NCEPDEV/stmp2 - sudo mkdir -p /scratch1/NCEPDEV/stmp4 - sudo mkdir -p /scratch1/NCEPDEV/global - sudo chmod -R a+w /scratch1/NCEPDEV - cd $GITHUB_WORKSPACE/sorc - git submodule update --init -j 9 - ./link_workflow.sh - cd $GITHUB_WORKSPACE/ci/scripts/tests - ln -s ../wxflow + - name: Install Rocoto + id: install-rocoto + if: steps.cache-rocoto.outputs.cache-hit != 'true' + run: | + git clone https://github.com/christopherwharrop/rocoto.git rocoto-${{ env.ROCOTO_VERSION }} + cd rocoto-${{ env.ROCOTO_VERSION }} + git checkout ${{ env.ROCOTO_VERSION }} + ./INSTALL + echo "export PATH=${PWD}/bin:\$PATH" >> rocoto_path.sh + chmod +x rocoto_path.sh - pytest -v --junitxml $GITHUB_WORKSPACE/ci/scripts/tests/test-results.xml + - name: Run tests + shell: bash + run: | + sudo mkdir -p /scratch1/NCEPDEV + sudo mkdir -p /scratch1/NCEPDEV/stmp2 + sudo mkdir -p /scratch1/NCEPDEV/stmp4 + sudo mkdir -p /scratch1/NCEPDEV/global + sudo chmod -R a+w /scratch1/NCEPDEV + source rocoto-${{ env.ROCOTO_VERSION }}/rocoto_path.sh + cd global-workflow/sorc + git submodule update --init -j 2 wxflow ufs_model.fd + ./link_workflow.sh + cd ../dev/ci/scripts/tests + pytest -v --junitxml test-results.xml - - name: Publish Test Results - if: always() - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - files: ci/scripts/tests/test-results.xml - job_summary: true - comment_mode: off + - name: Publish Test Results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: global-workflow/dev/ci/scripts/tests/test-results.xml + job_summary: true + comment_mode: off diff --git a/.github/workflows/pw_aws_ci.yaml b/.github/workflows/pw_aws_ci.yaml index 6225b0503a4..490e52cb0fa 100644 --- a/.github/workflows/pw_aws_ci.yaml +++ b/.github/workflows/pw_aws_ci.yaml @@ -88,7 +88,7 @@ jobs: checkout: needs: fetch-branch - runs-on: + runs-on: - self-hosted - aws - parallelworks @@ -103,9 +103,9 @@ jobs: repository: ${{ needs.fetch-branch.outputs.repo }} ref: ${{ needs.fetch-branch.outputs.branch }} - build-link: + build-link: needs: checkout - runs-on: + runs-on: - self-hosted - aws - parallelworks @@ -123,7 +123,7 @@ jobs: create-experiments: needs: build-link - runs-on: + runs-on: - self-hosted - aws - parallelworks @@ -139,13 +139,13 @@ jobs: run: | mkdir -p ${{ env.RUNTESTS }} cd ${{ env.TEST_DIR }}/HOMEgfs - source workflow/gw_setup.sh - source ci/platforms/config.noaacloud - ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite + source dev/ush/gw_setup.sh + source dev/ci/platforms/config.noaacloud + ./dev/workflow/create_experiment.py --yaml dev/ci/cases/pr/${{ matrix.case }}.yaml --overwrite run-experiments: needs: create-experiments - runs-on: + runs-on: - self-hosted - aws - parallelworks @@ -157,11 +157,11 @@ jobs: - name: Run Experiment ${{ matrix.case }} run: | cd ${{ env.TEST_DIR }}/HOMEgfs - ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs + ./dev/ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs clean-up: needs: run-experiments - runs-on: + runs-on: - self-hosted - aws - parallelworks diff --git a/ci/scripts/tests/test_create_experiment.py b/ci/scripts/tests/test_create_experiment.py deleted file mode 100644 index 03f3a30805d..00000000000 --- a/ci/scripts/tests/test_create_experiment.py +++ /dev/null @@ -1,29 +0,0 @@ -from wxflow import Executable -from shutil import rmtree -import os -import copy - -_here = os.path.dirname(__file__) -HOMEgfs = os.sep.join(_here.split(os.sep)[:-3]) -RUNDIR = os.path.join(_here, 'testdata/RUNDIR') - - -def test_create_experiment(): - - create_experiment_script = Executable(f'{HOMEgfs}/workflow/create_experiment.py') - yaml_dir = yaml_dir = os.path.join(HOMEgfs, 'ci/cases/pr') - env = os.environ.copy() - env['RUNTESTS'] = RUNDIR - - for case in os.listdir(yaml_dir): - if case.endswith('.yaml'): - with open(os.path.join(yaml_dir, case), 'r') as file: - file_contents = file.read() - if 'ICSDIR_ROOT' not in file_contents: - create_experiment = copy.deepcopy(create_experiment_script) - create_experiment.add_default_arg(['-y', f'../../cases/pr/{case}', '--overwrite']) - env['pslot'] = os.path.splitext(case)[0] - create_experiment(env=env) - assert (create_experiment.returncode == 0) - - rmtree(RUNDIR) diff --git a/ci/scripts/utils/ci_utils_wrapper.sh b/ci/scripts/utils/ci_utils_wrapper.sh deleted file mode 100755 index a4fd1776c83..00000000000 --- a/ci/scripts/utils/ci_utils_wrapper.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" -source "${HOMEgfs}/ush/detect_machine.sh" - -utility_function="${1}" - -source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh" -${utility_function} "${@:2}" diff --git a/ci/.gitlab-ci.yml b/dev/ci/.gitlab-ci.yml similarity index 72% rename from ci/.gitlab-ci.yml rename to dev/ci/.gitlab-ci.yml index 1882234bb87..0083096d32a 100644 --- a/ci/.gitlab-ci.yml +++ b/dev/ci/.gitlab-ci.yml @@ -6,9 +6,9 @@ stages: # Global variables variables: BUILD: 'TODAY' - GIT_CLONE_PATH: '$CI_BUILDS_DIR/${BUILD}/global-workflow' - HOMEGFS: $GIT_CLONE_PATH - RUNTESTS_DIR: $CI_BUILDS_DIR/${BUILD}/RUNTESTS + GIT_CLONE_PATH: '${CI_BUILDS_DIR}/${BUILD}/global-workflow' + HOMEGFS: ${GIT_CLONE_PATH} + RUNTESTS_DIR: ${CI_BUILDS_DIR}/${BUILD}/RUNTESTS GIT_DEPTH: 1 RUNNER_SCRIPT_TIMEOUT: 6h RUNNER_AFTER_SCRIPT_TIMEOUT: 6h @@ -22,7 +22,7 @@ build: stage: build script: - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" - - ci/scripts/utils/ci_utils_wrapper.sh build_compute + - dev/ci/scripts/utils/ci_utils_wrapper.sh build_compute - sorc/link_workflow.sh - mkdir -p ${RUNTESTS_DIR} # TODO - Add more machines to the list and make @@ -34,7 +34,7 @@ build: - ${MACHINE} -# Create experiments stage from a fixed list of cases in $HOMEGFS/ci/cases/pr +# Create experiments stage from a fixed list of cases in $HOMEGFS/dev/ci/cases/pr # TODO: Find a way to dynamically generate the list of cases (maybe using Jinja2) setup_experiments: variables: @@ -42,7 +42,7 @@ setup_experiments: stage: create_experiments script: - export RUNTESTS=${RUNTESTS_DIR} - - ${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/ci/cases/pr/${caseName}.yaml + - ${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEGFS}/dev/ci/cases/pr/${caseName}.yaml parallel: matrix: - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] @@ -52,15 +52,15 @@ setup_experiments: - build # Running the list of experiments created in the previous stage -# using the run-check_ci.sh script from $HOMEgfs/ci/scripts directory +# using the run-check_ci.sh script from $HOMEgfs/dev/ci/scripts directory run_tests: variables: GIT_STRATEGY: none stage: run_tests script: - echo "Using build directory ${HOMEGFS} (dated ${BUILD_DATE})" - - pslot=$(${HOMEGFS}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) - - ${HOMEGFS}/ci/scripts/run-check_ci.sh ${CI_BUILDS_DIR}/${BUILD} ${pslot} global-workflow + - pslot=$(${HOMEGFS}/dev/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${RUNTESTS_DIR} ${caseName}) + - ${HOMEGFS}/dev/ci/scripts/run-check_ci.sh ${CI_BUILDS_DIR}/${BUILD} ${pslot} global-workflow parallel: matrix: - caseName: ["C48_ATM", "C48mx500_3DVarAOWCDA", "C48mx500_hybAOWCDA", "C48_S2SWA_gefs", "C48_S2SW", "C96_atm3DVar", "C96C48_hybatmDA", "C96C48_hybatmaerosnowDA"] diff --git a/ci/Jenkinsfile b/dev/ci/Jenkinsfile similarity index 90% rename from ci/Jenkinsfile rename to dev/ci/Jenkinsfile index 7033416f670..3d31079e110 100644 --- a/ci/Jenkinsfile +++ b/dev/ci/Jenkinsfile @@ -2,6 +2,7 @@ def Machine = 'none' def machine = 'none' def CUSTOM_WORKSPACE = 'none' def HOMEgfs = 'none' +def HOMEgfs_dev = 'none' def CI_CASES = '' def GH = 'none' // Map of the machine names (MACHINE_ID) to the Jenkins Node names @@ -90,6 +91,7 @@ pipeline { GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow" + HOMEgfs_dev = "${CUSTOM_WORKSPACE}/global-workflow/dev" sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/global-workflow; mkdir -p ${CUSTOM_WORKSPACE}/global-workflow") sh(script: "rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS; mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS") sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """) @@ -127,7 +129,7 @@ pipeline { def error_logs_message = "" dir("${HOMEgfs}/sorc") { try { - sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables + sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh build_compute") // build the global-workflow executables } catch (Exception error_build) { echo "Failed to build global-workflow: ${error_build.getMessage()}" if ( fileExists("logs/error.logs") ) { @@ -146,12 +148,12 @@ pipeline { } try { sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID} """) gist_url=sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_BUILD_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) } catch (Exception error_comment) { @@ -174,8 +176,8 @@ pipeline { } // Get a list of CI cases to run CI_CASES = sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/get_host_case_list.py ${machine} """, returnStdout: true).trim().split() echo "Cases to run: ${CI_CASES}" } @@ -200,8 +202,8 @@ pipeline { def error_output = "" try { error_output = sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs_dev}/ci/cases/pr/${caseName}.yaml """, returnStdout: true).trim() } catch (Exception error_create) { sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body '${caseName} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n```\n${error_output}```' """) @@ -213,22 +215,22 @@ pipeline { stage("Running ${caseName}") { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { script { - def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() + def pslot = sh(script: "${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") try { sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow' """) sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot} """) } catch (Exception error_experment) { sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot} """) ws(CUSTOM_WORKSPACE) { def error_logs = "" @@ -250,13 +252,13 @@ pipeline { } try { gist_url = sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --multiple --format=github --gist PR_${env.CHANGE_ID} | tail -n 1 """, returnStdout: true).trim() sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body 'Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n```\n${error_logs_message}```\n\nFollow link here to view the contents of the above file(s): ${gist_url}' """) sh(script: """ - source ${HOMEgfs}/workflow/gw_setup.sh - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} + source ${HOMEgfs_dev}/ush/gw_setup.sh + ${HOMEgfs_dev}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID} """) } catch (Exception error_comment) { echo "Failed to comment on PR: ${error_comment.getMessage()}" @@ -281,7 +283,7 @@ pipeline { } parallel parallelStages + [failFast: true] } - } + } } } diff --git a/ci/Jenkinsfile4AWS b/dev/ci/Jenkinsfile4AWS similarity index 100% rename from ci/Jenkinsfile4AWS rename to dev/ci/Jenkinsfile4AWS diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/dev/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml similarity index 90% rename from ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml rename to dev/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index e18d3913f26..49fd7a74894 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/dev/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -18,4 +18,4 @@ arguments: #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2 #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ - yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/gfsv17/marine3dvar.yaml diff --git a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml b/dev/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml similarity index 89% rename from ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml rename to dev/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml index 66f3b1b4060..04322da6420 100644 --- a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml +++ b/dev/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml @@ -18,4 +18,4 @@ arguments: edate: 2021070306 #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ - yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/gfsv17/marinehyb.yaml diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/dev/ci/cases/gfsv17/marine3dvar.yaml similarity index 100% rename from ci/cases/gfsv17/marine3dvar.yaml rename to dev/ci/cases/gfsv17/marine3dvar.yaml diff --git a/ci/cases/gfsv17/marinehyb.yaml b/dev/ci/cases/gfsv17/marinehyb.yaml similarity index 100% rename from ci/cases/gfsv17/marinehyb.yaml rename to dev/ci/cases/gfsv17/marinehyb.yaml diff --git a/ci/cases/hires/C1152_S2SW.yaml b/dev/ci/cases/hires/C1152_S2SW.yaml similarity index 83% rename from ci/cases/hires/C1152_S2SW.yaml rename to dev/ci/cases/hires/C1152_S2SW.yaml index 8c05b129361..2228129f89d 100644 --- a/ci/cases/hires/C1152_S2SW.yaml +++ b/dev/ci/cases/hires/C1152_S2SW.yaml @@ -12,4 +12,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2019120300 edate: 2019120300 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/hires/C768_S2SW.yaml b/dev/ci/cases/hires/C768_S2SW.yaml similarity index 83% rename from ci/cases/hires/C768_S2SW.yaml rename to dev/ci/cases/hires/C768_S2SW.yaml index 6cb17671737..c433e93b37c 100644 --- a/ci/cases/hires/C768_S2SW.yaml +++ b/dev/ci/cases/hires/C768_S2SW.yaml @@ -12,4 +12,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2019120300 edate: 2019120300 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C48_ATM.yaml b/dev/ci/cases/pr/C48_ATM.yaml similarity index 82% rename from ci/cases/pr/C48_ATM.yaml rename to dev/ci/cases/pr/C48_ATM.yaml index 18658ae5188..f1cb063b9e2 100644 --- a/ci/cases/pr/C48_ATM.yaml +++ b/dev/ci/cases/pr/C48_ATM.yaml @@ -11,4 +11,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C48_S2SW.yaml b/dev/ci/cases/pr/C48_S2SW.yaml similarity index 83% rename from ci/cases/pr/C48_S2SW.yaml rename to dev/ci/cases/pr/C48_S2SW.yaml index 972d02ce4fc..8398d47c392 100644 --- a/ci/cases/pr/C48_S2SW.yaml +++ b/dev/ci/cases/pr/C48_S2SW.yaml @@ -12,4 +12,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/dev/ci/cases/pr/C48_S2SWA_gefs.yaml similarity index 86% rename from ci/cases/pr/C48_S2SWA_gefs.yaml rename to dev/ci/cases/pr/C48_S2SWA_gefs.yaml index fe2482cbb3a..405a8115e53 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/dev/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -16,7 +16,7 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gefs_defaults_ci.yaml skip_ci_on_hosts: - None diff --git a/ci/cases/pr/C48_S2SW_extended.yaml b/dev/ci/cases/pr/C48_S2SW_extended.yaml similarity index 88% rename from ci/cases/pr/C48_S2SW_extended.yaml rename to dev/ci/cases/pr/C48_S2SW_extended.yaml index 8dd52a4a720..943aabeb11d 100644 --- a/ci/cases/pr/C48_S2SW_extended.yaml +++ b/dev/ci/cases/pr/C48_S2SW_extended.yaml @@ -12,7 +12,7 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_extended_ci.yaml skip_ci_on_hosts: - hera diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml similarity index 87% rename from ci/cases/pr/C48mx500_3DVarAOWCDA.yaml rename to dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index a1077e3e172..65be1b289ba 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -16,7 +16,7 @@ arguments: nens: 0 interval: 6 start: warm - yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/soca_gfs_defaults_ci.yaml skip_ci_on_hosts: - gaeac5 diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml similarity index 87% rename from ci/cases/pr/C48mx500_hybAOWCDA.yaml rename to dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml index cfef477ec13..0210638a5e4 100644 --- a/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -17,7 +17,7 @@ arguments: nens: 2 interval: 0 start: warm - yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml skip_ci_on_hosts: - gaeac5 diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_hybatmDA.yaml similarity index 89% rename from ci/cases/pr/C96C48_hybatmDA.yaml rename to dev/ci/cases/pr/C96C48_hybatmDA.yaml index 8584a54ec8e..b874bebd80f 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmDA.yaml @@ -18,7 +18,7 @@ arguments: nens: 2 interval: 24 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml skip_ci_on_hosts: - awsepicglobalworkflow diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/dev/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml similarity index 86% rename from ci/cases/pr/C96C48_hybatmaerosnowDA.yaml rename to dev/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index 59c086368f6..d4fe2f6283f 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -16,7 +16,7 @@ arguments: nens: 2 interval: 24 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: - orion diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml similarity index 87% rename from ci/cases/pr/C96C48_ufs_hybatmDA.yaml rename to dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index fd25172c81b..37f4299cdb7 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -16,7 +16,7 @@ arguments: nens: 2 interval: 24 start: warm - yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml skip_ci_on_hosts: - gaeac5 diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/dev/ci/cases/pr/C96_atm3DVar.yaml similarity index 88% rename from ci/cases/pr/C96_atm3DVar.yaml rename to dev/ci/cases/pr/C96_atm3DVar.yaml index 76b99f011b0..e5749a49ec8 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar.yaml @@ -15,7 +15,7 @@ arguments: nens: 0 interval: 24 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_metp_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_metp_ci.yaml skip_ci_on_hosts: - gaeac5 diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml similarity index 88% rename from ci/cases/pr/C96_atm3DVar_extended.yaml rename to dev/ci/cases/pr/C96_atm3DVar_extended.yaml index aec3c52087f..68f8938a5d1 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -15,7 +15,7 @@ arguments: nens: 0 interval: 6 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_extended_ci.yaml skip_ci_on_hosts: - hera diff --git a/ci/cases/pr/C96mx100_S2S.yaml b/dev/ci/cases/pr/C96mx100_S2S.yaml similarity index 88% rename from ci/cases/pr/C96mx100_S2S.yaml rename to dev/ci/cases/pr/C96mx100_S2S.yaml index 0559c04cd81..cf976899df5 100644 --- a/ci/cases/pr/C96mx100_S2S.yaml +++ b/dev/ci/cases/pr/C96mx100_S2S.yaml @@ -16,7 +16,7 @@ arguments: comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 - yaml: {{ HOMEgfs }}/ci/cases/yamls/sfs_defaults.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_defaults.yaml skip_ci_on_hosts: - gaeac6 diff --git a/ci/cases/sfs/C96mx100_S2S.yaml b/dev/ci/cases/sfs/C96mx100_S2S.yaml similarity index 88% rename from ci/cases/sfs/C96mx100_S2S.yaml rename to dev/ci/cases/sfs/C96mx100_S2S.yaml index 224e73a2a6e..f24bbbaea05 100644 --- a/ci/cases/sfs/C96mx100_S2S.yaml +++ b/dev/ci/cases/sfs/C96mx100_S2S.yaml @@ -16,4 +16,4 @@ arguments: comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 - yaml: {{ HOMEgfs }}/ci/cases/yamls/sfs_full.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/sfs_full.yaml diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/dev/ci/cases/weekly/C384C192_hybatmda.yaml similarity index 87% rename from ci/cases/weekly/C384C192_hybatmda.yaml rename to dev/ci/cases/weekly/C384C192_hybatmda.yaml index 9a2d823a58f..397f13b42c0 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/dev/ci/cases/weekly/C384C192_hybatmda.yaml @@ -17,4 +17,4 @@ arguments: nens: 2 interval: 24 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_S2SWA.yaml b/dev/ci/cases/weekly/C384_S2SWA.yaml similarity index 83% rename from ci/cases/weekly/C384_S2SWA.yaml rename to dev/ci/cases/weekly/C384_S2SWA.yaml index 023dd7c2a14..f509e4556de 100644 --- a/ci/cases/weekly/C384_S2SWA.yaml +++ b/dev/ci/cases/weekly/C384_S2SWA.yaml @@ -12,4 +12,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2016070100 edate: 2016070100 - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/dev/ci/cases/weekly/C384_atm3DVar.yaml similarity index 87% rename from ci/cases/weekly/C384_atm3DVar.yaml rename to dev/ci/cases/weekly/C384_atm3DVar.yaml index 8681a5d58ef..f424666e58c 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/dev/ci/cases/weekly/C384_atm3DVar.yaml @@ -17,4 +17,4 @@ arguments: nens: 0 interval: 24 start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml b/dev/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml rename to dev/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/yamls/atmsoilDA_defaults_ci.yaml b/dev/ci/cases/yamls/atmsoilDA_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/atmsoilDA_defaults_ci.yaml rename to dev/ci/cases/yamls/atmsoilDA_defaults_ci.yaml diff --git a/ci/cases/yamls/gefs_defaults_ci.yaml b/dev/ci/cases/yamls/gefs_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/gefs_defaults_ci.yaml rename to dev/ci/cases/yamls/gefs_defaults_ci.yaml diff --git a/ci/cases/yamls/gefs_replay_ci.yaml b/dev/ci/cases/yamls/gefs_replay_ci.yaml similarity index 100% rename from ci/cases/yamls/gefs_replay_ci.yaml rename to dev/ci/cases/yamls/gefs_replay_ci.yaml diff --git a/ci/cases/yamls/gfs_defaults_ci.yaml b/dev/ci/cases/yamls/gfs_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/gfs_defaults_ci.yaml rename to dev/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/yamls/gfs_extended_ci.yaml b/dev/ci/cases/yamls/gfs_extended_ci.yaml similarity index 100% rename from ci/cases/yamls/gfs_extended_ci.yaml rename to dev/ci/cases/yamls/gfs_extended_ci.yaml diff --git a/ci/cases/yamls/gfs_metp_ci.yaml b/dev/ci/cases/yamls/gfs_metp_ci.yaml similarity index 100% rename from ci/cases/yamls/gfs_metp_ci.yaml rename to dev/ci/cases/yamls/gfs_metp_ci.yaml diff --git a/ci/cases/yamls/sfs_defaults.yaml b/dev/ci/cases/yamls/sfs_defaults.yaml similarity index 100% rename from ci/cases/yamls/sfs_defaults.yaml rename to dev/ci/cases/yamls/sfs_defaults.yaml diff --git a/ci/cases/yamls/sfs_full.yaml b/dev/ci/cases/yamls/sfs_full.yaml similarity index 100% rename from ci/cases/yamls/sfs_full.yaml rename to dev/ci/cases/yamls/sfs_full.yaml diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/dev/ci/cases/yamls/soca_gfs_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/soca_gfs_defaults_ci.yaml rename to dev/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml b/dev/ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml similarity index 100% rename from ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml rename to dev/ci/cases/yamls/soca_hyb_gfs_defaults_ci.yaml diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml similarity index 100% rename from ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml rename to dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/platforms/config.gaeac5 b/dev/ci/platforms/config.gaeac5 similarity index 100% rename from ci/platforms/config.gaeac5 rename to dev/ci/platforms/config.gaeac5 diff --git a/ci/platforms/config.gaeac6 b/dev/ci/platforms/config.gaeac6 similarity index 98% rename from ci/platforms/config.gaeac6 rename to dev/ci/platforms/config.gaeac6 index b55b93ce599..f32c7e5627e 100644 --- a/ci/platforms/config.gaeac6 +++ b/dev/ci/platforms/config.gaeac6 @@ -27,9 +27,9 @@ export JENKINS_WORK_DIR="${GFS_CI_ROOT}/Jenkins/workspace" # NOTE: JENKINS custom_workspace directory where CI jobs are run # /gpfs/f6/drsa-precip3/proj-shared/global/CI -# and is defined in $HOMEgfs/ci/Jenkinsfile +# and is defined in $HOMEgfs/dev/ci/Jenkinsfile + - ######################################################################### # GitLab CI configuration # These variables are referenced directly by launch_gitlab_runner.sh @@ -62,4 +62,3 @@ export STAGED_TESTS_DIR="${GFS_CI_ROOT}/STAGED_TESTS_DIR" # HPC account information - For job submission export HPC_ACCOUNT=drsa-precip3 - diff --git a/ci/platforms/config.hera b/dev/ci/platforms/config.hera similarity index 97% rename from ci/platforms/config.hera rename to dev/ci/platforms/config.hera index a8a14647389..1b589cf943d 100644 --- a/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -27,7 +27,7 @@ export JENKINS_AGENT_LAUNCH_DIR=${GFS_CI_ROOT}/Jenkins/agent export JENKINS_WORK_DIR=${GFS_CI_ROOT}/Jenkins/workspace # NOTE: JENKINS custom_workspace directory where CI jobs are run -# and is defined in $HOMEgfs/ci/Jenkinsfile as custom_workspace +# and is defined in $HOMEgfs/dev/ci/Jenkinsfile as custom_workspace # /scratch1/NCEPDEV/global/glopara/CI diff --git a/ci/platforms/config.hercules b/dev/ci/platforms/config.hercules similarity index 100% rename from ci/platforms/config.hercules rename to dev/ci/platforms/config.hercules diff --git a/ci/platforms/config.orion b/dev/ci/platforms/config.orion similarity index 100% rename from ci/platforms/config.orion rename to dev/ci/platforms/config.orion diff --git a/ci/platforms/config.wcoss2 b/dev/ci/platforms/config.wcoss2 similarity index 100% rename from ci/platforms/config.wcoss2 rename to dev/ci/platforms/config.wcoss2 diff --git a/ci/scripts/check_ci.sh b/dev/ci/scripts/check_ci.sh similarity index 98% rename from ci/scripts/check_ci.sh rename to dev/ci/scripts/check_ci.sh index be61690588a..b1a8fa05d9b 100755 --- a/ci/scripts/check_ci.sh +++ b/dev/ci/scripts/check_ci.sh @@ -8,7 +8,7 @@ set -eux # to run from within a cron job in the CI Managers account ##################################################################################### -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' diff --git a/ci/scripts/clone-build_ci.sh b/dev/ci/scripts/clone-build_ci.sh similarity index 100% rename from ci/scripts/clone-build_ci.sh rename to dev/ci/scripts/clone-build_ci.sh diff --git a/ci/scripts/driver.sh b/dev/ci/scripts/driver.sh similarity index 80% rename from ci/scripts/driver.sh rename to dev/ci/scripts/driver.sh index f4cb6cf9ad0..fd1d1a66c7b 100755 --- a/ci/scripts/driver.sh +++ b/dev/ci/scripts/driver.sh @@ -19,7 +19,7 @@ export REPO_URL=${REPO_URL:-"git@github.com:NOAA-EMC/global-workflow.git"} ################################################################ # Setup the reletive paths to scripts and PS4 for better logging ################################################################ -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date +'%D %r')" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' @@ -32,7 +32,7 @@ source "${ROOT_DIR}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" - source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}" + source "${ROOT_DIR}/dev/ci/platforms/config.${MACHINE_ID}" ;; *) echo "Unsupported platform. Exiting with error." @@ -46,7 +46,7 @@ esac HOMEgfs=${ROOT_DIR} export HOMEgfs set +x -source "${ROOT_DIR}/ci/scripts/utils/ci_utils.sh" +source "${ROOT_DIR}/dev/ci/scripts/utils/ci_utils.sh" source "${ROOT_DIR}/ush/module-setup.sh" module use "${ROOT_DIR}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" @@ -70,7 +70,7 @@ export GH pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" if [[ ! -f "${pr_list_dbfile}" ]]; then - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --create --dbfile "${pr_list_dbfile}" fi pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" --state "open" | awk '{print $1}') || true @@ -78,7 +78,7 @@ pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" - for pr in ${pr_list}; do pr_dir="${GFS_BASH_CI_ROOT}/PR/${pr}" [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}" - db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}") + db_list=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}") output_ci_single="${pr_dir}/output_single.log" ############################################################# # Check if a Ready labeled PR has changed back from once set @@ -89,7 +89,7 @@ for pr in ${pr_list}; do if [[ "${db_list}" == *"already is in list"* ]]; then # Get the the PID and HOST of the driver.sh cron job # that is stored int he CI database for this PR - driver_ID=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display "${pr}" | awk '{print $4}') || true + driver_ID=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --display "${pr}" | awk '{print $4}') || true driver_PID=$(echo "${driver_ID}" | cut -d":" -f1) || true driver_HOST=$(echo "${driver_ID}" | cut -d":" -f2) || true host_name=$(hostname -s) @@ -135,14 +135,14 @@ for pr in ${pr_list}; do sed -i "1 i\`\`\`" "${output_ci_single}" fi "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}" fi done pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true + pr_list=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true fi if [[ -z "${pr_list}" ]]; then echo "no PRs open and ready for checkout/build .. exiting" @@ -157,7 +157,7 @@ fi for pr in ${pr_list}; do # Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron - pr_building=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true + pr_building=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true if [[ -n "${pr_building}" ]]; then continue fi @@ -168,7 +168,7 @@ for pr in ${pr_list}; do driver_build_PID=$$ driver_build_HOST=$(hostname -s) "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Ready" --add-label "CI-${MACHINE_ID^}-Building" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building "${driver_build_PID}:${driver_build_HOST}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Building "${driver_build_PID}:${driver_build_HOST}" rm -Rf "${pr_dir}" mkdir -p "${pr_dir}" { @@ -184,7 +184,7 @@ for pr in ${pr_list}; do fi "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci_single}" set +e - "${ROOT_DIR}/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}" + "${ROOT_DIR}/dev/ci/scripts/clone-build_ci.sh" -p "${pr}" -d "${pr_dir}" -o "${output_ci}" ci_status=$? ################################################################## # Checking for special case when Ready label was updated @@ -193,7 +193,7 @@ for pr in ${pr_list}; do # we need to exit this instance of the driver script ################################################################# if [[ ${ci_status} -ne 0 ]]; then - build_PID_check=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true + build_PID_check=$("${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true if [[ "${build_PID_check}" -ne "$$" ]]; then echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting" exit 0 @@ -201,7 +201,7 @@ for pr in ${pr_list}; do fi set -e if [[ ${ci_status} -eq 0 ]]; then - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built "0:0" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Built "0:0" #setup space to put an experiment # export RUNTESTS for yaml case files to pickup export RUNTESTS="${pr_dir}/RUNTESTS" @@ -215,23 +215,23 @@ for pr in ${pr_list}; do cd "${HOMEgfs}" pr_sha=$(git rev-parse --short HEAD) - for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do + for yaml_config in "${HOMEgfs}/dev/ci/cases/pr/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true # export pslot for yaml case files to pickup export pslot="${case}_${pr_sha}" rm -Rf "${STMP}/RUNDIRS/${pslot}" set +e - export LOGFILE_PATH="${HOMEgfs}/ci/scripts/create_experiment.log" + export LOGFILE_PATH="${HOMEgfs}/dev/ci/scripts/create_experiment.log" rm -f "${LOGFILE_PATH}" - yaml_case_file="${HOMEgfs}/ci/cases/pr/${case}.yaml" - skip_hosts=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" --yaml "${yaml_case_file}" --key skip_ci_on_hosts --string) + yaml_case_file="${HOMEgfs}/dev/ci/cases/pr/${case}.yaml" + skip_hosts=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" --yaml "${yaml_case_file}" --key skip_ci_on_hosts --string) if [[ "${skip_hosts}" == *"${MACHINE_ID}"* ]]; then { echo "Case setup: Skipped for experiment ${pslot}" || true } >> "${output_ci}" continue fi - "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml" --overwrite > "${LOGFILE_PATH}" 2>&1 + "${HOMEgfs}/dev/workflow/create_experiment.py" --yaml "${HOMEgfs}/dev/ci/cases/pr/${case}.yaml" --overwrite > "${LOGFILE_PATH}" 2>&1 ci_status=$? set -e if [[ ${ci_status} -eq 0 ]]; then @@ -245,14 +245,14 @@ for pr in ${pr_list}; do cat "${LOGFILE_PATH}" } >> "${output_ci}" "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" exit 1 fi done "${GH}" pr edit --repo "${REPO_URL}" "${pr}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Running" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --update_pr "${pr}" Open Running "0:0" "${GH}" pr comment "${pr}" --repo "${REPO_URL}" --body-file "${output_ci}" else # failed to clone and build @@ -263,10 +263,10 @@ for pr in ${pr_list}; do } >> "${output_ci}" "${GH}" pr edit "${pr}" --repo "${REPO_URL}" --remove-label "CI-${MACHINE_ID^}-Building" --add-label "CI-${MACHINE_ID^}-Failed" - "${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" + "${ROOT_DIR}/dev/ci/scripts/utils/pr_list_database.py" --remove_pr "${pr}" --dbfile "${pr_list_dbfile}" if [[ -f "${HOMEgfs}/sorc/logs/error.logs" ]]; then - gist_URL=$("${ROOT_DIR}/ci/scripts/utils/ci_utils_wrapper.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs") + gist_URL=$("${ROOT_DIR}/dev/ci/scripts/utils/ci_utils_wrapper.sh" publish_logs "PR_${pr}" "${HOMEgfs}/sorc" "${HOMEgfs}/sorc/logs/error.logs") { echo -e "\nError logs from build" echo "Gist URL: ${gist_URL}" diff --git a/ci/scripts/driver_weekly.sh b/dev/ci/scripts/driver_weekly.sh similarity index 98% rename from ci/scripts/driver_weekly.sh rename to dev/ci/scripts/driver_weekly.sh index 6fac3d3e917..cd11ec1bc03 100755 --- a/ci/scripts/driver_weekly.sh +++ b/dev/ci/scripts/driver_weekly.sh @@ -27,7 +27,7 @@ export REPO_URL="ssh://git@ssh.github.com:443/NOAA-EMC/global-workflow.git" ################################################################ # Setup the relative paths to scripts and PS4 for better logging ################################################################ -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]' diff --git a/ci/scripts/run-check_ci.sh b/dev/ci/scripts/run-check_ci.sh similarity index 96% rename from ci/scripts/run-check_ci.sh rename to dev/ci/scripts/run-check_ci.sh index 5c49a21c4ba..fe85d275034 100755 --- a/ci/scripts/run-check_ci.sh +++ b/dev/ci/scripts/run-check_ci.sh @@ -30,7 +30,7 @@ run_check_logfile="${RUNTESTS}/ci-run_check.log" # Source modules and setup logging echo "Source modules." -source "${HOMEgfs}/workflow/gw_setup.sh" +source "${HOMEgfs}/dev/ush/gw_setup.sh" # cd into the experiment directory echo "cd ${RUNTESTS}/EXPDIR/${pslot}" @@ -69,7 +69,7 @@ while true; do # Get job statistics echo "Gather Rocoto statistics" # shellcheck disable=SC2312 # We want to use the exit code of the command - full_state=$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v) + full_state=$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${xml}" -d "${db}" -v) error_stat=$? for state in CYCLES_TOTAL CYCLES_DONE SUCCEEDED FAIL DEAD; do @@ -113,4 +113,3 @@ while true; do done exit "${rc}" - diff --git a/ci/scripts/run_ci.sh b/dev/ci/scripts/run_ci.sh similarity index 91% rename from ci/scripts/run_ci.sh rename to dev/ci/scripts/run_ci.sh index efbd7efcd99..aa77a7dbf6b 100755 --- a/ci/scripts/run_ci.sh +++ b/dev/ci/scripts/run_ci.sh @@ -9,7 +9,7 @@ set -eux # Abstract TODO ##################################################################################### -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" scriptname=$(basename "${BASH_SOURCE[0]}") echo "Begin ${scriptname} at $(date -u)" || true export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]' @@ -22,7 +22,7 @@ source "${HOMEgfs}/ush/detect_machine.sh" case ${MACHINE_ID} in hera | orion | hercules | wcoss2 | gaea) echo "Running Automated Testing on ${MACHINE_ID}" - source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" ;; *) echo "Unsupported platform. Exiting with error." @@ -48,7 +48,7 @@ pr_list_dbfile="${GFS_BASH_CI_ROOT}/open_pr_list.db" pr_list="" if [[ -f "${pr_list_dbfile}" ]]; then - pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true + pr_list=$("${HOMEgfs}/dev/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true pr_list=$(echo "${pr_list}" | tr ' ' '\n' | head -n "${max_concurrent_pr}" | tr '\n' ' ') || true fi if [[ -z "${pr_list}" ]]; then diff --git a/dev/ci/scripts/tests/test_create_experiment.py b/dev/ci/scripts/tests/test_create_experiment.py new file mode 100644 index 00000000000..6f1e1390f29 --- /dev/null +++ b/dev/ci/scripts/tests/test_create_experiment.py @@ -0,0 +1,33 @@ +import os +from shutil import rmtree + +from wxflow import Executable + +_here = os.path.dirname(__file__) + +HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) +RUNDIR_FAKE = os.path.join(_here, 'testdata/RUNDIR') +ICSDIR_FAKE = os.path.join(_here, 'testdata/ICSDIR') + + +def test_create_experiment(): + + create_experiment = Executable(f'{HOMEgfs}/dev/workflow/create_experiment.py') + create_experiment.add_default_arg(['--overwrite']) + yaml_dir = yaml_dir = os.path.join(HOMEgfs, 'dev/ci/cases/pr') + env = os.environ.copy() + env['RUNTESTS'] = RUNDIR_FAKE + env['ICSDIR_ROOT'] = ICSDIR_FAKE + + err = 0 + for case in os.listdir(yaml_dir): + if case.endswith('.yaml'): + env['pslot'] = os.path.splitext(case)[0] + cmd_args = ['-y', f'{yaml_dir}/{case}'] + create_experiment(*cmd_args, env=env) + if create_experiment.returncode: + print(f"FATAL ERROR: Failed to create experiment for {case}") + err = 1 + assert err == 0, f"create_experiment.py failed!" + + rmtree(RUNDIR_FAKE) diff --git a/ci/scripts/tests/test_rocotostat.py b/dev/ci/scripts/tests/test_rocotostat.py similarity index 97% rename from ci/scripts/tests/test_rocotostat.py rename to dev/ci/scripts/tests/test_rocotostat.py index a3c56df9881..08ff4c5fd5d 100755 --- a/ci/scripts/tests/test_rocotostat.py +++ b/dev/ci/scripts/tests/test_rocotostat.py @@ -6,8 +6,8 @@ script_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(os.path.dirname(script_dir), 'utils')) -from rocotostat import rocoto_statcount, rocotostat_summary, is_done, is_stalled, CommandNotFoundError -from wxflow import which +from rocotostat import rocoto_statcount, rocotostat_summary, is_done, is_stalled +from wxflow import which, CommandNotFoundError test_data_url = 'https://noaa-nws-global-pds.s3.amazonaws.com/data/CI/' diff --git a/ci/scripts/tests/test_setup.py b/dev/ci/scripts/tests/test_setup.py similarity index 87% rename from ci/scripts/tests/test_setup.py rename to dev/ci/scripts/tests/test_setup.py index 9561ed04418..38f14e11459 100755 --- a/ci/scripts/tests/test_setup.py +++ b/dev/ci/scripts/tests/test_setup.py @@ -1,10 +1,11 @@ -from wxflow import Executable, Configuration, ProcessError -from shutil import rmtree -import pytest import os +import pytest +from shutil import rmtree + +from wxflow import Executable, Configuration, ProcessError _here = os.path.dirname(__file__) -HOMEgfs = os.sep.join(_here.split(os.sep)[:-3]) +HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) RUNDIR = os.path.join(_here, 'testdata/RUNDIR') pslot = "C48_ATM" account = "fv3-cpu" @@ -20,7 +21,7 @@ def test_setup_expt(): "--idate", "2021032312", "--edate", "2021032312", "--overwrite", "--account", account ] - setup_expt_script = Executable(os.path.join(HOMEgfs, "workflow", "setup_expt.py")) + setup_expt_script = Executable(os.path.join(HOMEgfs, "dev/workflow/setup_expt.py")) setup_expt_script.add_default_arg(arguments) setup_expt_script() assert (setup_expt_script.returncode == 0) @@ -28,7 +29,7 @@ def test_setup_expt(): def test_setup_xml(): - setup_xml_script = Executable(os.path.join(HOMEgfs, "workflow/setup_xml.py")) + setup_xml_script = Executable(os.path.join(HOMEgfs, "dev/workflow/setup_xml.py")) setup_xml_script.add_default_arg(f"{RUNDIR}/{pslot}") setup_xml_script() assert (setup_xml_script.returncode == 0) @@ -58,7 +59,7 @@ def test_setup_xml_fail_config_env_cornercase(): os.chmod('run_setup_xml.sh', 0o755) try: - setup_xml_script = Executable(os.path.join(HOMEgfs, "ci", "scripts", "tests", "run_setup_xml.sh")) + setup_xml_script = Executable(os.path.join(HOMEgfs, "dev/ci/scripts/tests/run_setup_xml.sh")) setup_xml_script.add_default_arg(f"{RUNDIR}/{pslot}") setup_xml_script() assert (setup_xml_script.returncode == 0) diff --git a/dev/ci/scripts/tests/wxflow b/dev/ci/scripts/tests/wxflow new file mode 120000 index 00000000000..553b3a1926c --- /dev/null +++ b/dev/ci/scripts/tests/wxflow @@ -0,0 +1 @@ +../../../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/ci/scripts/utils/ci_utils.sh b/dev/ci/scripts/utils/ci_utils.sh similarity index 79% rename from ci/scripts/utils/ci_utils.sh rename to dev/ci/scripts/utils/ci_utils.sh index 5122ec68fc0..5bc6da43368 100755 --- a/ci/scripts/utils/ci_utils.sh +++ b/dev/ci/scripts/utils/ci_utils.sh @@ -61,7 +61,7 @@ function get_pr_case_list () { # loop over every yaml file in the PR's ci/cases # and create an run directory for each one for this PR loop ############################################################# - for yaml_config in "${HOMEgfs}/ci/cases/pr/"*.yaml; do + for yaml_config in "${HOMEgfs}/dev/ci/cases/pr/"*.yaml; do case=$(basename "${yaml_config}" .yaml) || true echo "${case}" done @@ -70,7 +70,7 @@ function get_pr_case_list () { function get_pslot_list () { local RUNTESTS="${1}" - + ############################################################# # loop over expdir directories in RUNTESTS # and create list of the directory names (pslot) with the hash tag @@ -86,7 +86,7 @@ function get_pslot () { local RUNTESTS="${1}" local case="${2}" - + ############################################################# # loop over expdir directories in RUNTESTS # and return the name of the pslot with its tag that matches the case @@ -118,25 +118,25 @@ function create_experiment () { pr_sha=$(git rev-parse --short HEAD) case=$(basename "${yaml_config}" .yaml) || true export pslot=${case}_${pr_sha} - + if [[ ${MACHINE_ID} == "noaacloud" ]]; then - source "${HOMEgfs}/ci/platforms/config.${PW_CSP}" + source "${HOMEgfs}/dev/ci/platforms/config.${PW_CSP}" else - source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" + source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" fi - source "${HOMEgfs}/workflow/gw_setup.sh" + source "${HOMEgfs}/dev/ush/gw_setup.sh" # Remove RUNDIRS dir incase this is a retry (STMP now in host file) if [[ ${MACHINE_ID} == "noaacloud" ]]; then - STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) + STMP=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/dev/workflow/hosts/${PW_CSP}pw.yaml" -k STMP -s) else - STMP=$("${HOMEgfs}/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) + STMP=$("${HOMEgfs}/dev/ci/scripts/utils/parse_yaml.py" -y "${HOMEgfs}/dev/workflow/hosts/${MACHINE_ID}.yaml" -k STMP -s) fi echo "Removing ${STMP}/RUNDIRS/${pslot} directory incase this is a retry" rm -Rf "${STMP}/RUNDIRS/${pslot}" - "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" + "${HOMEgfs}/${system}/dev/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}" } @@ -160,8 +160,8 @@ function publish_logs() { if [[ -n "${full_paths}" ]]; then # shellcheck disable=SC2027,SC2086 - ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null - URL="$("${HOMEgfs}/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" + ${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py --file ${full_paths} --repo ${PR_header} > /dev/null + URL="$("${HOMEgfs}/dev/ci/scripts/utils/publish_logs.py" --file "${full_paths}" --gist "${PR_header}")" fi echo "${URL}" } @@ -178,7 +178,7 @@ function cleanup_experiment() { pslot=$(basename "${EXPDIR}") # Use the Python utility to get the required variables - read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true + read -r ARCDIR ATARDIR STMP COMROOT < <("${HOMEgfs}/dev/ci/scripts/utils/get_config_var.py" ARCDIR ATARDIR STMP COMROOT "${EXPDIR}") || true rm -Rf "${ARCDIR:?}" rm -Rf "${ATARDIR:?}" @@ -189,7 +189,12 @@ function cleanup_experiment() { function build_compute () { - source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}" - "${HOMEgfs}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all + source "${HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}" + # TODO: when it's safe to build on C6 compute nodes again, do so + if [[ "${MACHINE_ID}" == "gaeac6" ]]; then + "${HOMEgfs}/sorc/build_all.sh" -v -k all + else + "${HOMEgfs}/sorc/build_compute.sh" -A "${HPC_ACCOUNT}" -v all + fi } diff --git a/dev/ci/scripts/utils/ci_utils_wrapper.sh b/dev/ci/scripts/utils/ci_utils_wrapper.sh new file mode 100755 index 00000000000..28e76459704 --- /dev/null +++ b/dev/ci/scripts/utils/ci_utils_wrapper.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" +source "${HOMEgfs}/ush/detect_machine.sh" + +utility_function="${1}" + +source "${HOMEgfs}/dev/ci/scripts/utils/ci_utils.sh" +${utility_function} "${@:2}" diff --git a/ci/scripts/utils/get_config_var.py b/dev/ci/scripts/utils/get_config_var.py similarity index 100% rename from ci/scripts/utils/get_config_var.py rename to dev/ci/scripts/utils/get_config_var.py diff --git a/ci/scripts/utils/get_host_case_list.py b/dev/ci/scripts/utils/get_host_case_list.py similarity index 91% rename from ci/scripts/utils/get_host_case_list.py rename to dev/ci/scripts/utils/get_host_case_list.py index eb10f29f05f..59c62539181 100755 --- a/ci/scripts/utils/get_host_case_list.py +++ b/dev/ci/scripts/utils/get_host_case_list.py @@ -7,7 +7,7 @@ from wxflow import AttrDict _here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) if __name__ == '__main__': @@ -22,7 +22,7 @@ data = AttrDict(HOMEgfs=_top) data.update(os.environ) - case_files = glob.glob(f'{HOMEgfs}/ci/cases/pr/*.yaml') + case_files = glob.glob(f'{HOMEgfs}/dev/ci/cases/pr/*.yaml') for case_yaml in case_files: case_conf = parse_j2yaml(path=case_yaml, data=data) if 'skip_ci_on_hosts' in case_conf: diff --git a/ci/scripts/utils/githubpr.py b/dev/ci/scripts/utils/githubpr.py similarity index 100% rename from ci/scripts/utils/githubpr.py rename to dev/ci/scripts/utils/githubpr.py diff --git a/ci/scripts/utils/launch_gitlab_runner.sh b/dev/ci/scripts/utils/launch_gitlab_runner.sh similarity index 95% rename from ci/scripts/utils/launch_gitlab_runner.sh rename to dev/ci/scripts/utils/launch_gitlab_runner.sh index 028a7f498ed..732c3d26f6f 100755 --- a/ci/scripts/utils/launch_gitlab_runner.sh +++ b/dev/ci/scripts/utils/launch_gitlab_runner.sh @@ -4,7 +4,7 @@ set -e ######################################################################### # launch_gitlab_runner.sh - Script to manage GitLab runners for CI/CD -# +# # This script handles three main operations for GitLab runners: # 1. register - Registers a new GitLab runner with the GitLab server # 2. run - Starts a GitLab runner in the background @@ -14,7 +14,7 @@ set -e ######################################################################### # Set the HOMEGFS_ variable to the root directory of the global workflow -HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" # Get the hostname of the current machine host="$(hostname)" @@ -36,9 +36,9 @@ case "${MACHINE_ID}" in esac # Source the platform-specific configuration file -# This file contains platform-specific variables such as GITLAB_URL, GITLAB_CI_BUILDS_DIR, +# This file contains platform-specific variables such as GITLAB_URL, GITLAB_CI_BUILDS_DIR, # and GITLAB_RUNNER_DIR which are required for runner registration and execution -source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" +source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" # Change to the GitLab runner directory defined in the platform config cd "${GITLAB_RUNNER_DIR}" || exit 1 @@ -59,14 +59,14 @@ rm -f "${GITLAB_LOG}" # 3. A gitlab_token file in the current directory GITLAB_RUNNER_TOKEN="${2:-${GITLAB_RUNNER_TOKEN}}" if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then - if [[ -f gitlab_token ]]; then + if [[ -f gitlab_token ]]; then source gitlab_token - fi + fi fi if [[ -z "${GITLAB_RUNNER_TOKEN}" ]]; then echo "ERROR: GITLAB_RUNNER_TOKEN not set" exit 1 -fi +fi # Download the GitLab runner binary if it does not exist if [[ ! -f gitlab-runner ]]; then @@ -92,7 +92,7 @@ if [[ "${1}" == "register" ]]; then # --custom_build_dir-enabled: Enable custom build directories # --request-concurrency: Number of concurrent requests that can be handled ./gitlab-runner register -n -t "${GITLAB_RUNNER_TOKEN}" --url "${GITLAB_URL}" --executor shell --shell bash --builds-dir "${GITLAB_CI_BUILDS_DIR}" --custom_build_dir-enabled true --request-concurrency 24 - + # Set the concurrent job limit in the GitLab runner config file sed -i 's/concurrent.*/concurrent = 24/' ~/.gitlab-runner/config.toml exit 0 diff --git a/ci/scripts/utils/launch_java_agent.sh b/dev/ci/scripts/utils/launch_java_agent.sh similarity index 97% rename from ci/scripts/utils/launch_java_agent.sh rename to dev/ci/scripts/utils/launch_java_agent.sh index 5b28cc83009..8e348d09b60 100755 --- a/ci/scripts/utils/launch_java_agent.sh +++ b/dev/ci/scripts/utils/launch_java_agent.sh @@ -65,7 +65,7 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov" controller_user=${controller_user:-"terry.mcguinness"} controller_user_auth_token="jenkins_token" -HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )" +HOMEGFS_="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." >/dev/null 2>&1 && pwd )" host=$(hostname) ######################################################################### @@ -91,9 +91,9 @@ module use "${HOMEGFS_}/modulefiles" module load "module_gwsetup.${MACHINE_ID}" if [[ ${MACHINE_ID} == "noaacloud" ]]; then - source "${HOMEGFS_}/ci/platforms/config.${PW_CSP}" + source "${HOMEGFS_}/dev/ci/platforms/config.${PW_CSP}" else - source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}" + source "${HOMEGFS_}/dev/ci/platforms/config.${MACHINE_ID}" fi JAVA_HOME="${JENKINS_AGENT_LAUNCH_DIR}/JAVA/jdk-17.0.10" @@ -171,13 +171,13 @@ check_node_online() { lauch_agent () { echo "Launching Jenkins Agent on ${host} using internal workspace ${JENKINS_WORK_DIR}" - + # Clear the remoting cache if [[ -d "${JENKINS_WORK_DIR}/remoting" ]]; then echo "Clearing remoting cache in ${JENKINS_WORK_DIR}/remoting" rm -rf "${JENKINS_WORK_DIR}/remoting" fi - + command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}" echo -e "Launching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}" ${command} >> "${LOG}" 2>&1 & diff --git a/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt b/dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt similarity index 100% rename from ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt rename to dev/ci/scripts/utils/parallel_works/UserBootstrap_centos7.txt diff --git a/ci/scripts/utils/parallel_works/provision_runner.sh b/dev/ci/scripts/utils/parallel_works/provision_runner.sh similarity index 100% rename from ci/scripts/utils/parallel_works/provision_runner.sh rename to dev/ci/scripts/utils/parallel_works/provision_runner.sh diff --git a/ci/scripts/utils/parse_yaml.py b/dev/ci/scripts/utils/parse_yaml.py similarity index 99% rename from ci/scripts/utils/parse_yaml.py rename to dev/ci/scripts/utils/parse_yaml.py index 6950fead258..c22ad69983e 100755 --- a/ci/scripts/utils/parse_yaml.py +++ b/dev/ci/scripts/utils/parse_yaml.py @@ -11,7 +11,7 @@ from pathlib import Path _here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../..')) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../../../..')) description = """parse yaml file and return value of key""" diff --git a/ci/scripts/utils/pr_list_database.py b/dev/ci/scripts/utils/pr_list_database.py similarity index 100% rename from ci/scripts/utils/pr_list_database.py rename to dev/ci/scripts/utils/pr_list_database.py diff --git a/ci/scripts/utils/publish_logs.py b/dev/ci/scripts/utils/publish_logs.py similarity index 100% rename from ci/scripts/utils/publish_logs.py rename to dev/ci/scripts/utils/publish_logs.py diff --git a/ci/scripts/utils/rocotostat.py b/dev/ci/scripts/utils/rocotostat.py similarity index 100% rename from ci/scripts/utils/rocotostat.py rename to dev/ci/scripts/utils/rocotostat.py diff --git a/dev/ci/scripts/utils/wxflow b/dev/ci/scripts/utils/wxflow new file mode 120000 index 00000000000..553b3a1926c --- /dev/null +++ b/dev/ci/scripts/utils/wxflow @@ -0,0 +1 @@ +../../../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/ci/scripts/utils/wxflow b/dev/ci/scripts/wxflow similarity index 100% rename from ci/scripts/utils/wxflow rename to dev/ci/scripts/wxflow diff --git a/ctests/CMakeLists.txt b/dev/ctests/CMakeLists.txt similarity index 98% rename from ctests/CMakeLists.txt rename to dev/ctests/CMakeLists.txt index 19fcfcda756..3ec286852be 100644 --- a/ctests/CMakeLists.txt +++ b/dev/ctests/CMakeLists.txt @@ -74,24 +74,24 @@ function(AddJJOBTest) ${ARGN}) set(TEST_NAME ${ARG_CASE}_${ARG_JOB}) - set(CASE_PATH ${HOMEgfs}/ci/cases/pr) + set(CASE_PATH ${HOMEgfs}/dev/ci/cases/pr) set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml) add_test(NAME test_${TEST_NAME}_setup COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}") - + add_test(NAME test_${TEST_NAME}_stage COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}") - + add_test(NAME test_${TEST_NAME}_execute COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts) set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}") - + # TODO - This is a stub for the validation step add_test(NAME test_${TEST_NAME}_validate COMMAND ./validate.sh ${TEST_NAME} ${ARG_TEST_DATE} diff --git a/ctests/README.md b/dev/ctests/README.md similarity index 86% rename from ctests/README.md rename to dev/ctests/README.md index d19b7d0196d..386ef34e854 100644 --- a/ctests/README.md +++ b/dev/ctests/README.md @@ -10,7 +10,7 @@ The CTest framework consists of the following scripts: - **execute.sh.in**: Executes the JJOB and monitors its status. - **validate.sh.in**: Validates the results of the JJOB. -**NOTE:** So far only test C48_ATM *gfs_fcst_set0* has `output_files` for the validation step using a basic chksum for testing. Further development using grib and NETCDF comparison tools is pending. +**NOTE:** So far only test C48_ATM *gfs_fcst_set0* has `output_files` for the validation step using a basic chksum for testing. Further development using grib and NETCDF comparison tools is pending. ## Usage @@ -24,14 +24,14 @@ export HPC_ACCOUNT="your_hpc_account" export ICSDIR_ROOT="/path/to/icsdir_root" export STAGED_TESTS_DIR="/path/to/staged_tests_dir" ``` -**NOTE**: The the specific values for these three enviroment variables can be found in `$HOMEgfs/ci/platforms/config.$MACHINE_ID` and may also be added to the `cmake` command line with the `-D` option +**NOTE**: The the specific values for these three enviroment variables can be found in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID` and may also be added to the `cmake` command line with the `-D` option # Run CMake to configure the ctest framework ```shell -cd $HOMEgfs/ctests +cd $HOMEgfs/dev/ctests mkdir build cd build -cmake ../.. +cmake ../.. ``` @@ -59,7 +59,7 @@ Or simply use the '-R' switch to run any individual test: ctest -R test_C48_S2SW_gfs_fcst_seg0_execute -V ``` -To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows: +To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/dev/ctest/CMakeLists.txt` file as follows: ```cmake AddJJOBTest( CASE "C48_ATM" @@ -67,4 +67,4 @@ AddJJOBTest( TEST_DATE "2021032312" ) ``` -Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml` +Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/dev/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml` diff --git a/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml b/dev/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml similarity index 100% rename from ctests/cases/C48_ATM_gfs_fcst_seg0.yaml rename to dev/ctests/cases/C48_ATM_gfs_fcst_seg0.yaml diff --git a/ctests/cases/C48_S2SW_gfs_atmos_prod_f000-f003.yaml b/dev/ctests/cases/C48_S2SW_gfs_atmos_prod_f000-f003.yaml similarity index 100% rename from ctests/cases/C48_S2SW_gfs_atmos_prod_f000-f003.yaml rename to dev/ctests/cases/C48_S2SW_gfs_atmos_prod_f000-f003.yaml diff --git a/ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml b/dev/ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml similarity index 100% rename from ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml rename to dev/ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml diff --git a/ctests/scripts/execute.sh.in b/dev/ctests/scripts/execute.sh.in similarity index 100% rename from ctests/scripts/execute.sh.in rename to dev/ctests/scripts/execute.sh.in diff --git a/ctests/scripts/setup.sh.in b/dev/ctests/scripts/setup.sh.in similarity index 94% rename from ctests/scripts/setup.sh.in rename to dev/ctests/scripts/setup.sh.in index 6c4a772b650..43ac2b7b01b 100755 --- a/ctests/scripts/setup.sh.in +++ b/dev/ctests/scripts/setup.sh.in @@ -12,7 +12,7 @@ ICSDIR_ROOT="@ICSDIR_ROOT@" HPC_ACCOUNT="@HPC_ACCOUNT@" set +x -source "${HOMEgfs}/workflow/gw_setup.sh" +source "${HOMEgfs}/dev/ush/gw_setup.sh" set -x pslot="${TEST_NAME}" \ diff --git a/ctests/scripts/stage.py b/dev/ctests/scripts/stage.py similarity index 99% rename from ctests/scripts/stage.py rename to dev/ctests/scripts/stage.py index ea4009a495c..ae256424584 100755 --- a/ctests/scripts/stage.py +++ b/dev/ctests/scripts/stage.py @@ -20,7 +20,6 @@ """ import os -import datetime from argparse import ArgumentParser from pathlib import Path diff --git a/ctests/scripts/stage.sh.in b/dev/ctests/scripts/stage.sh.in similarity index 79% rename from ctests/scripts/stage.sh.in rename to dev/ctests/scripts/stage.sh.in index acff1672318..afdb9d9c214 100755 --- a/ctests/scripts/stage.sh.in +++ b/dev/ctests/scripts/stage.sh.in @@ -12,7 +12,7 @@ HOMEgfs="@PROJECT_SOURCE_DIR@" # Load the runtime environment for this script (needs wxflow and its dependencies) set +x -source "${HOMEgfs}/workflow/gw_setup.sh" +source "${HOMEgfs}/dev/ush/gw_setup.sh" rc=$? if [[ "${rc}" -ne 0 ]]; then exit "${rc}" @@ -21,12 +21,12 @@ set -x PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH -YAML_FILE="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml" +YAML_FILE="${HOMEgfs}/dev/ctests/cases/${TEST_NAME}.yaml" TEST_NAME="${TEST_NAME}" \ RUNTESTS="${RUNTESTS}" \ STAGED_TESTS_DIR="${STAGED_TESTS_DIR}" \ -"${HOMEgfs}/ctests/scripts/stage.py" --yaml "${YAML_FILE}" --test_date "${TEST_DATE}" +"${HOMEgfs}/dev/ctests/scripts/stage.py" --yaml "${YAML_FILE}" --test_date "${TEST_DATE}" rc=$? if [[ "${rc}" -ne 0 ]]; then set +x diff --git a/ctests/scripts/validate.py b/dev/ctests/scripts/validate.py similarity index 100% rename from ctests/scripts/validate.py rename to dev/ctests/scripts/validate.py diff --git a/ctests/scripts/validate.sh.in b/dev/ctests/scripts/validate.sh.in similarity index 80% rename from ctests/scripts/validate.sh.in rename to dev/ctests/scripts/validate.sh.in index f21f97bd65f..079eca2c41f 100755 --- a/ctests/scripts/validate.sh.in +++ b/dev/ctests/scripts/validate.sh.in @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -ux - + TEST_NAME=${1:?"Name of the test is required"} TEST_DATE=${2:?"idate of the test is required"} - + # CMake to fill these variables STAGED_TESTS_DIR="@STAGED_TESTS_DIR@" RUNTESTS="@RUNTESTS@" @@ -11,7 +11,7 @@ HOMEgfs="@PROJECT_SOURCE_DIR@" # Load the runtime environment for this script (needs wxflow and its dependencies) set +x -source "${HOMEgfs}/workflow/gw_setup.sh" +source "${HOMEgfs}/dev/ush/gw_setup.sh" rc=$? if [[ "${rc}" -ne 0 ]]; then exit "${rc}" @@ -20,13 +20,13 @@ set -x PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src" export PYTHONPATH -YAML_FILE="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml" +YAML_FILE="${HOMEgfs}/dev/ctests/cases/${TEST_NAME}.yaml" echo "validating '${TEST_NAME}' with yaml file '${YAML_FILE}'" TEST_NAME="${TEST_NAME}" \ RUNTESTS="${RUNTESTS}" \ STAGED_TESTS_DIR="${STAGED_TESTS_DIR}" \ -"${HOMEgfs}/ctests/scripts/validate.py" --yaml "${YAML_FILE}" --test_date "${TEST_DATE}" +"${HOMEgfs}/dev/ctests/scripts/validate.py" --yaml "${YAML_FILE}" --test_date "${TEST_DATE}" rc=$? if [[ "${rc}" -ne 0 ]]; then set +x @@ -34,5 +34,5 @@ if [[ "${rc}" -ne 0 ]]; then set -x exit "${rc}" fi - + exit 0 diff --git a/jobs/rocoto/aeroanlfinal.sh b/dev/jobs/aeroanlfinal.sh similarity index 100% rename from jobs/rocoto/aeroanlfinal.sh rename to dev/jobs/aeroanlfinal.sh diff --git a/jobs/rocoto/aeroanlgenb.sh b/dev/jobs/aeroanlgenb.sh similarity index 100% rename from jobs/rocoto/aeroanlgenb.sh rename to dev/jobs/aeroanlgenb.sh diff --git a/jobs/rocoto/aeroanlinit.sh b/dev/jobs/aeroanlinit.sh similarity index 100% rename from jobs/rocoto/aeroanlinit.sh rename to dev/jobs/aeroanlinit.sh diff --git a/jobs/rocoto/aeroanlvar.sh b/dev/jobs/aeroanlvar.sh similarity index 100% rename from jobs/rocoto/aeroanlvar.sh rename to dev/jobs/aeroanlvar.sh diff --git a/jobs/rocoto/aerosol_init.sh b/dev/jobs/aerosol_init.sh similarity index 79% rename from jobs/rocoto/aerosol_init.sh rename to dev/jobs/aerosol_init.sh index 1b53f44c44f..e28261c8ed4 100755 --- a/jobs/rocoto/aerosol_init.sh +++ b/dev/jobs/aerosol_init.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -source $HOMEgfs/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -13,8 +13,8 @@ fi ############################################################### # Source relevant configs configs="base aerosol_init" -for config in $configs; do - source $EXPDIR/config.${config} +for config in ${configs}; do + source "${EXPDIR}/config.${config}" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -23,13 +23,13 @@ done ############################################################### # Source machine runtime environment -source ${BASE_ENV}/${machine}.env aerosol_init +source "${BASE_ENV}/${machine}.env" aerosol_init status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" fi -${HOMEgfs}/scripts/exgfs_aero_init_aerosol.py +"${HOMEgfs}/scripts/exgfs_aero_init_aerosol.py" status=$? if [[ ${status} -ne 0 ]]; then diff --git a/jobs/rocoto/anal.sh b/dev/jobs/anal.sh similarity index 89% rename from jobs/rocoto/anal.sh rename to dev/jobs/anal.sh index 72ee9918e4b..112a3c46670 100755 --- a/jobs/rocoto/anal.sh +++ b/dev/jobs/anal.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/jobs/rocoto/analcalc.sh b/dev/jobs/analcalc.sh similarity index 79% rename from jobs/rocoto/analcalc.sh rename to dev/jobs/analcalc.sh index fad80b2c859..4e3348b8bd3 100755 --- a/jobs/rocoto/analcalc.sh +++ b/dev/jobs/analcalc.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,7 +15,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC" status=$? diff --git a/jobs/rocoto/analcalc_fv3jedi.sh b/dev/jobs/analcalc_fv3jedi.sh similarity index 100% rename from jobs/rocoto/analcalc_fv3jedi.sh rename to dev/jobs/analcalc_fv3jedi.sh diff --git a/jobs/rocoto/analdiag.sh b/dev/jobs/analdiag.sh similarity index 79% rename from jobs/rocoto/analdiag.sh rename to dev/jobs/analdiag.sh index 7b23eba69bc..61346865191 100755 --- a/jobs/rocoto/analdiag.sh +++ b/dev/jobs/analdiag.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,7 +15,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +"${HOMEgfs}/jobs/JGDAS_ATMOS_ANALYSIS_DIAG" status=$? diff --git a/jobs/rocoto/anlstat.sh b/dev/jobs/anlstat.sh similarity index 100% rename from jobs/rocoto/anlstat.sh rename to dev/jobs/anlstat.sh diff --git a/jobs/rocoto/arch_tars.sh b/dev/jobs/arch_tars.sh similarity index 100% rename from jobs/rocoto/arch_tars.sh rename to dev/jobs/arch_tars.sh diff --git a/jobs/rocoto/arch_vrfy.sh b/dev/jobs/arch_vrfy.sh similarity index 100% rename from jobs/rocoto/arch_vrfy.sh rename to dev/jobs/arch_vrfy.sh diff --git a/jobs/rocoto/atmanlfinal.sh b/dev/jobs/atmanlfinal.sh similarity index 100% rename from jobs/rocoto/atmanlfinal.sh rename to dev/jobs/atmanlfinal.sh diff --git a/jobs/rocoto/atmanlfv3inc.sh b/dev/jobs/atmanlfv3inc.sh similarity index 100% rename from jobs/rocoto/atmanlfv3inc.sh rename to dev/jobs/atmanlfv3inc.sh diff --git a/jobs/rocoto/atmanlinit.sh b/dev/jobs/atmanlinit.sh similarity index 100% rename from jobs/rocoto/atmanlinit.sh rename to dev/jobs/atmanlinit.sh diff --git a/jobs/rocoto/atmanlvar.sh b/dev/jobs/atmanlvar.sh similarity index 100% rename from jobs/rocoto/atmanlvar.sh rename to dev/jobs/atmanlvar.sh diff --git a/jobs/rocoto/atmensanlfinal.sh b/dev/jobs/atmensanlfinal.sh similarity index 100% rename from jobs/rocoto/atmensanlfinal.sh rename to dev/jobs/atmensanlfinal.sh diff --git a/jobs/rocoto/atmensanlfv3inc.sh b/dev/jobs/atmensanlfv3inc.sh similarity index 100% rename from jobs/rocoto/atmensanlfv3inc.sh rename to dev/jobs/atmensanlfv3inc.sh diff --git a/jobs/rocoto/atmensanlinit.sh b/dev/jobs/atmensanlinit.sh similarity index 100% rename from jobs/rocoto/atmensanlinit.sh rename to dev/jobs/atmensanlinit.sh diff --git a/jobs/rocoto/atmensanlletkf.sh b/dev/jobs/atmensanlletkf.sh similarity index 100% rename from jobs/rocoto/atmensanlletkf.sh rename to dev/jobs/atmensanlletkf.sh diff --git a/jobs/rocoto/atmensanlobs.sh b/dev/jobs/atmensanlobs.sh similarity index 100% rename from jobs/rocoto/atmensanlobs.sh rename to dev/jobs/atmensanlobs.sh diff --git a/jobs/rocoto/atmensanlsol.sh b/dev/jobs/atmensanlsol.sh similarity index 100% rename from jobs/rocoto/atmensanlsol.sh rename to dev/jobs/atmensanlsol.sh diff --git a/jobs/rocoto/atmos_ensstat.sh b/dev/jobs/atmos_ensstat.sh similarity index 100% rename from jobs/rocoto/atmos_ensstat.sh rename to dev/jobs/atmos_ensstat.sh diff --git a/jobs/rocoto/atmos_products.sh b/dev/jobs/atmos_products.sh similarity index 100% rename from jobs/rocoto/atmos_products.sh rename to dev/jobs/atmos_products.sh diff --git a/jobs/rocoto/awips_20km_1p0deg.sh b/dev/jobs/awips_20km_1p0deg.sh similarity index 100% rename from jobs/rocoto/awips_20km_1p0deg.sh rename to dev/jobs/awips_20km_1p0deg.sh diff --git a/jobs/rocoto/cleanup.sh b/dev/jobs/cleanup.sh similarity index 100% rename from jobs/rocoto/cleanup.sh rename to dev/jobs/cleanup.sh diff --git a/jobs/rocoto/earc_tars.sh b/dev/jobs/earc_tars.sh similarity index 100% rename from jobs/rocoto/earc_tars.sh rename to dev/jobs/earc_tars.sh diff --git a/jobs/rocoto/earc_vrfy.sh b/dev/jobs/earc_vrfy.sh similarity index 100% rename from jobs/rocoto/earc_vrfy.sh rename to dev/jobs/earc_vrfy.sh diff --git a/jobs/rocoto/ecen.sh b/dev/jobs/ecen.sh similarity index 75% rename from jobs/rocoto/ecen.sh rename to dev/jobs/ecen.sh index dcac5860687..4e320bbf7e6 100755 --- a/jobs/rocoto/ecen.sh +++ b/dev/jobs/ecen.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -12,7 +12,8 @@ fi ############################################################### # Loop over groups to Execute the JJOB -fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +#shellcheck disable=SC2153 # disable spelling error +fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') for fhr in ${fhrlst}; do export FHMIN_ECEN=${fhr} @@ -21,7 +22,7 @@ for fhr in ${fhrlst}; do export job=ecen export jobid="${job}.$$" - ${HOMEgfs}/jobs/JGDAS_ENKF_ECEN + "${HOMEgfs}/jobs/JGDAS_ENKF_ECEN" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/jobs/rocoto/ecen_fv3jedi.sh b/dev/jobs/ecen_fv3jedi.sh similarity index 100% rename from jobs/rocoto/ecen_fv3jedi.sh rename to dev/jobs/ecen_fv3jedi.sh diff --git a/jobs/rocoto/echgres.sh b/dev/jobs/echgres.sh similarity index 79% rename from jobs/rocoto/echgres.sh rename to dev/jobs/echgres.sh index 38b7587a73d..51043c379f8 100755 --- a/jobs/rocoto/echgres.sh +++ b/dev/jobs/echgres.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,7 +15,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ATMOS_CHGRES_FORENKF +"${HOMEgfs}/jobs/JGDAS_ATMOS_CHGRES_FORENKF" status=$? diff --git a/jobs/rocoto/ediag.sh b/dev/jobs/ediag.sh similarity index 81% rename from jobs/rocoto/ediag.sh rename to dev/jobs/ediag.sh index 3f2ada7c0b1..301b0862847 100755 --- a/jobs/rocoto/ediag.sh +++ b/dev/jobs/ediag.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,7 +15,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_DIAG +"${HOMEgfs}/jobs/JGDAS_ENKF_DIAG" status=$? diff --git a/jobs/rocoto/eobs.sh b/dev/jobs/eobs.sh similarity index 89% rename from jobs/rocoto/eobs.sh rename to dev/jobs/eobs.sh index d0d997f28e1..3685d83d7c4 100755 --- a/jobs/rocoto/eobs.sh +++ b/dev/jobs/eobs.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/jobs/rocoto/epos.sh b/dev/jobs/epos.sh similarity index 75% rename from jobs/rocoto/epos.sh rename to dev/jobs/epos.sh index 5c7a679ed4e..7885267d034 100755 --- a/jobs/rocoto/epos.sh +++ b/dev/jobs/epos.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -12,17 +12,18 @@ fi export job="epos" export jobid="${job}.$$" - + ############################################################### # Loop over groups to Execute the JJOB -fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') +#shellcheck disable=SC2153 # disable spelling error +fhrlst=$(echo "${FHRLST}" | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') for fhr in ${fhrlst}; do export FHMIN_EPOS=${fhr} export FHMAX_EPOS=${fhr} export FHOUT_EPOS=${fhr} - ${HOMEgfs}/jobs/JGDAS_ENKF_POST + "${HOMEgfs}/jobs/JGDAS_ENKF_POST" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/jobs/rocoto/esfc.sh b/dev/jobs/esfc.sh similarity index 67% rename from jobs/rocoto/esfc.sh rename to dev/jobs/esfc.sh index 9ae7e4cfa47..a7180b080c5 100755 --- a/jobs/rocoto/esfc.sh +++ b/dev/jobs/esfc.sh @@ -4,10 +4,10 @@ set -x ############################################################### # Source FV3GFS workflow modules -# For DO_GSISOILDA=YES need to switch to ufsda modules +# For DO_GSISOILDA=YES need to switch to ufsda modules # until g-w issue 3390 is resolved. -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#. ${HOMEgfs}/ush/load_ufsda_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +#source "${HOMEgfs}/ush/load_ufsda_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -18,7 +18,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_SFC +"${HOMEgfs}/jobs/JGDAS_ENKF_SFC" status=$? diff --git a/jobs/rocoto/esnowanl.sh b/dev/jobs/esnowanl.sh similarity index 100% rename from jobs/rocoto/esnowanl.sh rename to dev/jobs/esnowanl.sh diff --git a/jobs/rocoto/eupd.sh b/dev/jobs/eupd.sh similarity index 89% rename from jobs/rocoto/eupd.sh rename to dev/jobs/eupd.sh index df3ef3037ac..2e3fca2056b 100755 --- a/jobs/rocoto/eupd.sh +++ b/dev/jobs/eupd.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/jobs/rocoto/extractvars.sh b/dev/jobs/extractvars.sh similarity index 100% rename from jobs/rocoto/extractvars.sh rename to dev/jobs/extractvars.sh diff --git a/jobs/rocoto/fbwind.sh b/dev/jobs/fbwind.sh similarity index 100% rename from jobs/rocoto/fbwind.sh rename to dev/jobs/fbwind.sh diff --git a/jobs/rocoto/fcst.sh b/dev/jobs/fcst.sh similarity index 79% rename from jobs/rocoto/fcst.sh rename to dev/jobs/fcst.sh index 61ca6ab5ec7..268aa50c59c 100755 --- a/jobs/rocoto/fcst.sh +++ b/dev/jobs/fcst.sh @@ -7,13 +7,13 @@ set -x # TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2 source "${HOMEgfs}/ush/detect_machine.sh" if [[ "${MACHINE_ID}" == "wcoss2" ]]; then - . ${HOMEgfs}/ush/load_ufswm_modules.sh + source "${HOMEgfs}/ush/load_ufswm_modules.sh" else - . ${HOMEgfs}/ush/load_fv3gfs_modules.sh + source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" fi status=$? if [[ ${status} -ne 0 ]]; then - exit "${status}" + exit "${status}" fi export job="fcst" diff --git a/jobs/rocoto/fetch.sh b/dev/jobs/fetch.sh similarity index 100% rename from jobs/rocoto/fetch.sh rename to dev/jobs/fetch.sh diff --git a/jobs/rocoto/fit2obs.sh b/dev/jobs/fit2obs.sh similarity index 100% rename from jobs/rocoto/fit2obs.sh rename to dev/jobs/fit2obs.sh diff --git a/jobs/rocoto/gempak.sh b/dev/jobs/gempak.sh similarity index 100% rename from jobs/rocoto/gempak.sh rename to dev/jobs/gempak.sh diff --git a/jobs/rocoto/gempakgrb2spec.sh b/dev/jobs/gempakgrb2spec.sh similarity index 100% rename from jobs/rocoto/gempakgrb2spec.sh rename to dev/jobs/gempakgrb2spec.sh diff --git a/jobs/rocoto/gempakmeta.sh b/dev/jobs/gempakmeta.sh similarity index 100% rename from jobs/rocoto/gempakmeta.sh rename to dev/jobs/gempakmeta.sh diff --git a/jobs/rocoto/gempakmetancdc.sh b/dev/jobs/gempakmetancdc.sh similarity index 100% rename from jobs/rocoto/gempakmetancdc.sh rename to dev/jobs/gempakmetancdc.sh diff --git a/jobs/rocoto/gempakncdcupapgif.sh b/dev/jobs/gempakncdcupapgif.sh similarity index 100% rename from jobs/rocoto/gempakncdcupapgif.sh rename to dev/jobs/gempakncdcupapgif.sh diff --git a/jobs/rocoto/genesis.sh b/dev/jobs/genesis.sh similarity index 100% rename from jobs/rocoto/genesis.sh rename to dev/jobs/genesis.sh diff --git a/jobs/rocoto/genesis_fsu.sh b/dev/jobs/genesis_fsu.sh similarity index 100% rename from jobs/rocoto/genesis_fsu.sh rename to dev/jobs/genesis_fsu.sh diff --git a/jobs/rocoto/globus_arch.sh b/dev/jobs/globus_arch.sh similarity index 100% rename from jobs/rocoto/globus_arch.sh rename to dev/jobs/globus_arch.sh diff --git a/jobs/rocoto/globus_earc.sh b/dev/jobs/globus_earc.sh similarity index 100% rename from jobs/rocoto/globus_earc.sh rename to dev/jobs/globus_earc.sh diff --git a/jobs/rocoto/marineanlchkpt.sh b/dev/jobs/marineanlchkpt.sh similarity index 100% rename from jobs/rocoto/marineanlchkpt.sh rename to dev/jobs/marineanlchkpt.sh diff --git a/jobs/rocoto/marineanlfinal.sh b/dev/jobs/marineanlfinal.sh similarity index 100% rename from jobs/rocoto/marineanlfinal.sh rename to dev/jobs/marineanlfinal.sh diff --git a/jobs/rocoto/marineanlinit.sh b/dev/jobs/marineanlinit.sh similarity index 100% rename from jobs/rocoto/marineanlinit.sh rename to dev/jobs/marineanlinit.sh diff --git a/jobs/rocoto/marineanlletkf.sh b/dev/jobs/marineanlletkf.sh similarity index 100% rename from jobs/rocoto/marineanlletkf.sh rename to dev/jobs/marineanlletkf.sh diff --git a/jobs/rocoto/marineanlvar.sh b/dev/jobs/marineanlvar.sh similarity index 100% rename from jobs/rocoto/marineanlvar.sh rename to dev/jobs/marineanlvar.sh diff --git a/jobs/rocoto/marinebmat.sh b/dev/jobs/marinebmat.sh similarity index 100% rename from jobs/rocoto/marinebmat.sh rename to dev/jobs/marinebmat.sh diff --git a/jobs/rocoto/metp.sh b/dev/jobs/metp.sh similarity index 100% rename from jobs/rocoto/metp.sh rename to dev/jobs/metp.sh diff --git a/jobs/rocoto/mos_ext_grd_fcst.sh b/dev/jobs/mos_ext_grd_fcst.sh similarity index 100% rename from jobs/rocoto/mos_ext_grd_fcst.sh rename to dev/jobs/mos_ext_grd_fcst.sh diff --git a/jobs/rocoto/mos_ext_grd_prdgen.sh b/dev/jobs/mos_ext_grd_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_ext_grd_prdgen.sh rename to dev/jobs/mos_ext_grd_prdgen.sh diff --git a/jobs/rocoto/mos_ext_grd_prep.sh b/dev/jobs/mos_ext_grd_prep.sh similarity index 100% rename from jobs/rocoto/mos_ext_grd_prep.sh rename to dev/jobs/mos_ext_grd_prep.sh diff --git a/jobs/rocoto/mos_ext_stn_fcst.sh b/dev/jobs/mos_ext_stn_fcst.sh similarity index 100% rename from jobs/rocoto/mos_ext_stn_fcst.sh rename to dev/jobs/mos_ext_stn_fcst.sh diff --git a/jobs/rocoto/mos_ext_stn_prdgen.sh b/dev/jobs/mos_ext_stn_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_ext_stn_prdgen.sh rename to dev/jobs/mos_ext_stn_prdgen.sh diff --git a/jobs/rocoto/mos_ext_stn_prep.sh b/dev/jobs/mos_ext_stn_prep.sh similarity index 100% rename from jobs/rocoto/mos_ext_stn_prep.sh rename to dev/jobs/mos_ext_stn_prep.sh diff --git a/jobs/rocoto/mos_grd_fcst.sh b/dev/jobs/mos_grd_fcst.sh similarity index 100% rename from jobs/rocoto/mos_grd_fcst.sh rename to dev/jobs/mos_grd_fcst.sh diff --git a/jobs/rocoto/mos_grd_prdgen.sh b/dev/jobs/mos_grd_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_grd_prdgen.sh rename to dev/jobs/mos_grd_prdgen.sh diff --git a/jobs/rocoto/mos_grd_prep.sh b/dev/jobs/mos_grd_prep.sh similarity index 100% rename from jobs/rocoto/mos_grd_prep.sh rename to dev/jobs/mos_grd_prep.sh diff --git a/jobs/rocoto/mos_stn_fcst.sh b/dev/jobs/mos_stn_fcst.sh similarity index 100% rename from jobs/rocoto/mos_stn_fcst.sh rename to dev/jobs/mos_stn_fcst.sh diff --git a/jobs/rocoto/mos_stn_prdgen.sh b/dev/jobs/mos_stn_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_stn_prdgen.sh rename to dev/jobs/mos_stn_prdgen.sh diff --git a/jobs/rocoto/mos_stn_prep.sh b/dev/jobs/mos_stn_prep.sh similarity index 100% rename from jobs/rocoto/mos_stn_prep.sh rename to dev/jobs/mos_stn_prep.sh diff --git a/jobs/rocoto/mos_wx_ext_prdgen.sh b/dev/jobs/mos_wx_ext_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_wx_ext_prdgen.sh rename to dev/jobs/mos_wx_ext_prdgen.sh diff --git a/jobs/rocoto/mos_wx_prdgen.sh b/dev/jobs/mos_wx_prdgen.sh similarity index 100% rename from jobs/rocoto/mos_wx_prdgen.sh rename to dev/jobs/mos_wx_prdgen.sh diff --git a/jobs/rocoto/npoess.sh b/dev/jobs/npoess.sh similarity index 100% rename from jobs/rocoto/npoess.sh rename to dev/jobs/npoess.sh diff --git a/jobs/rocoto/oceanice_products.sh b/dev/jobs/oceanice_products.sh similarity index 100% rename from jobs/rocoto/oceanice_products.sh rename to dev/jobs/oceanice_products.sh diff --git a/jobs/rocoto/ocnanalecen.sh b/dev/jobs/ocnanalecen.sh similarity index 100% rename from jobs/rocoto/ocnanalecen.sh rename to dev/jobs/ocnanalecen.sh diff --git a/jobs/rocoto/postsnd.sh b/dev/jobs/postsnd.sh similarity index 88% rename from jobs/rocoto/postsnd.sh rename to dev/jobs/postsnd.sh index a426119d374..e10d9a8c768 100755 --- a/jobs/rocoto/postsnd.sh +++ b/dev/jobs/postsnd.sh @@ -13,6 +13,6 @@ export jobid="${job}.$$" ################################################################ # Execute the JJOB -${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND +"${HOMEgfs}/jobs/JGFS_ATMOS_POSTSND" err=$? exit "${err}" diff --git a/jobs/rocoto/prep.sh b/dev/jobs/prep.sh similarity index 91% rename from jobs/rocoto/prep.sh rename to dev/jobs/prep.sh index 21679211b83..e385228757b 100755 --- a/jobs/rocoto/prep.sh +++ b/dev/jobs/prep.sh @@ -2,7 +2,7 @@ ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -20,7 +20,7 @@ RUN_local="${RUN/enkf}" # Set script and dependency variables # Ignore possible spelling error (nothing is misspelled) # shellcheck disable=SC2153 -GDATE=$(${NDATE} -"${assim_freq}" "${PDY}${cyc}") +GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H) # shellcheck disable= gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} @@ -55,10 +55,12 @@ if [[ ${ROTDIR_DUMP} = "YES" ]]; then fi fi # exception handling to ensure no dead link - if [[ $(find ${COM_OBS} -xtype l | wc -l) -ge 1 ]]; then + # shellcheck disable=SC2312 + if [[ $(find "${COM_OBS}" -xtype l | wc -l) -ge 1 ]]; then exit 9 fi - if [[ $(find ${COM_OBS_PREV} -xtype l | wc -l) -ge 1 ]]; then + # shellcheck disable=SC2312 + if [[ $(find "${COM_OBS_PREV}" -xtype l | wc -l) -ge 1 ]]; then exit 9 fi fi @@ -75,10 +77,10 @@ if [[ ${PROCESS_TROPCY} = "YES" ]]; then export COMINsyn=${COMINsyn:-$(compath.py gfs/prod/syndat)} export ARCHSYND=${ROTDIR}/syndat - if [[ ! -d ${ARCHSYND} ]]; then mkdir -p ${ARCHSYND}; fi + if [[ ! -d ${ARCHSYND} ]]; then mkdir -p "${ARCHSYND}"; fi if [[ ! -s ${ARCHSYND}/syndat_akavit ]]; then for file in syndat_akavit syndat_dateck syndat_stmcat.scr syndat_stmcat syndat_sthisto syndat_sthista ; do - cp ${COMINsyn}/${file} ${ARCHSYND}/. + cp "${COMINsyn}/${file}" "${ARCHSYND}"/. done fi diff --git a/jobs/rocoto/prep_emissions.sh b/dev/jobs/prep_emissions.sh similarity index 100% rename from jobs/rocoto/prep_emissions.sh rename to dev/jobs/prep_emissions.sh diff --git a/jobs/rocoto/prepatmiodaobs.sh b/dev/jobs/prepatmiodaobs.sh similarity index 100% rename from jobs/rocoto/prepatmiodaobs.sh rename to dev/jobs/prepatmiodaobs.sh diff --git a/jobs/rocoto/prepobsaero.sh b/dev/jobs/prepobsaero.sh similarity index 100% rename from jobs/rocoto/prepobsaero.sh rename to dev/jobs/prepobsaero.sh diff --git a/jobs/rocoto/prepoceanobs.sh b/dev/jobs/prepoceanobs.sh similarity index 100% rename from jobs/rocoto/prepoceanobs.sh rename to dev/jobs/prepoceanobs.sh diff --git a/jobs/rocoto/sfcanl.sh b/dev/jobs/sfcanl.sh similarity index 67% rename from jobs/rocoto/sfcanl.sh rename to dev/jobs/sfcanl.sh index 7421b189cd3..434a4f0c3fa 100755 --- a/jobs/rocoto/sfcanl.sh +++ b/dev/jobs/sfcanl.sh @@ -4,10 +4,10 @@ set -x ############################################################### # Source FV3GFS workflow modules -# For DO_GSISOILDA=YES need to switch to ufsda modules +# For DO_GSISOILDA=YES need to switch to ufsda modules # until g-w issue 3390 is resolved. -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#. ${HOMEgfs}/ush/load_ufsda_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +#source "${HOMEgfs}/ush/load_ufsda_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -18,7 +18,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_ATMOS_SFCANL +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_SFCANL" status=$? diff --git a/jobs/rocoto/snowanl.sh b/dev/jobs/snowanl.sh similarity index 100% rename from jobs/rocoto/snowanl.sh rename to dev/jobs/snowanl.sh diff --git a/jobs/rocoto/stage_ic.sh b/dev/jobs/stage_ic.sh similarity index 100% rename from jobs/rocoto/stage_ic.sh rename to dev/jobs/stage_ic.sh diff --git a/jobs/rocoto/tracker.sh b/dev/jobs/tracker.sh similarity index 100% rename from jobs/rocoto/tracker.sh rename to dev/jobs/tracker.sh diff --git a/jobs/rocoto/upp.sh b/dev/jobs/upp.sh similarity index 100% rename from jobs/rocoto/upp.sh rename to dev/jobs/upp.sh diff --git a/jobs/rocoto/verfozn.sh b/dev/jobs/verfozn.sh similarity index 100% rename from jobs/rocoto/verfozn.sh rename to dev/jobs/verfozn.sh diff --git a/jobs/rocoto/verfrad.sh b/dev/jobs/verfrad.sh similarity index 100% rename from jobs/rocoto/verfrad.sh rename to dev/jobs/verfrad.sh diff --git a/jobs/rocoto/vminmon.sh b/dev/jobs/vminmon.sh similarity index 100% rename from jobs/rocoto/vminmon.sh rename to dev/jobs/vminmon.sh diff --git a/jobs/rocoto/waveawipsbulls.sh b/dev/jobs/waveawipsbulls.sh similarity index 76% rename from jobs/rocoto/waveawipsbulls.sh rename to dev/jobs/waveawipsbulls.sh index 0d1e8584003..c936524b745 100755 --- a/jobs/rocoto/waveawipsbulls.sh +++ b/dev/jobs/waveawipsbulls.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -source ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,8 +15,8 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_WAVE_PRDGEN_BULLS +"${HOMEgfs}/jobs/JGLOBAL_WAVE_PRDGEN_BULLS" status=$? -exit ${status} +exit "${status}" diff --git a/jobs/rocoto/waveawipsgridded.sh b/dev/jobs/waveawipsgridded.sh similarity index 79% rename from jobs/rocoto/waveawipsgridded.sh rename to dev/jobs/waveawipsgridded.sh index e163328e9db..397c0fe2e12 100755 --- a/jobs/rocoto/waveawipsgridded.sh +++ b/dev/jobs/waveawipsgridded.sh @@ -4,7 +4,7 @@ set -x ############################################################### # Source FV3GFS workflow modules -source ${HOMEgfs}/ush/load_fv3gfs_modules.sh +source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -15,7 +15,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_WAVE_PRDGEN_GRIDDED +"${HOMEgfs}/jobs/JGLOBAL_WAVE_PRDGEN_GRIDDED" status=$? diff --git a/jobs/rocoto/wavegempak.sh b/dev/jobs/wavegempak.sh similarity index 100% rename from jobs/rocoto/wavegempak.sh rename to dev/jobs/wavegempak.sh diff --git a/jobs/rocoto/waveinit.sh b/dev/jobs/waveinit.sh similarity index 54% rename from jobs/rocoto/waveinit.sh rename to dev/jobs/waveinit.sh index c3afef56327..258179fcae7 100755 --- a/jobs/rocoto/waveinit.sh +++ b/dev/jobs/waveinit.sh @@ -3,10 +3,8 @@ set -x ############################################################### -echo -echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -. ${HOMEgfs}/ush/load_ufswm_modules.sh +#source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +source "${HOMEgfs}/ush/load_ufswm_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -16,10 +14,8 @@ export job="waveinit" export jobid="${job}.$$" ############################################################### -echo -echo "=============== START TO RUN WAVE INIT ===============" # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_WAVE_INIT +"${HOMEgfs}/jobs/JGLOBAL_WAVE_INIT" status=$? exit "${status}" diff --git a/jobs/rocoto/wavepostbndpnt.sh b/dev/jobs/wavepostbndpnt.sh similarity index 100% rename from jobs/rocoto/wavepostbndpnt.sh rename to dev/jobs/wavepostbndpnt.sh diff --git a/jobs/rocoto/wavepostbndpntbll.sh b/dev/jobs/wavepostbndpntbll.sh similarity index 100% rename from jobs/rocoto/wavepostbndpntbll.sh rename to dev/jobs/wavepostbndpntbll.sh diff --git a/jobs/rocoto/wavepostpnt.sh b/dev/jobs/wavepostpnt.sh similarity index 100% rename from jobs/rocoto/wavepostpnt.sh rename to dev/jobs/wavepostpnt.sh diff --git a/jobs/rocoto/wavepostsbs.sh b/dev/jobs/wavepostsbs.sh similarity index 100% rename from jobs/rocoto/wavepostsbs.sh rename to dev/jobs/wavepostsbs.sh diff --git a/jobs/rocoto/waveprep.sh b/dev/jobs/waveprep.sh similarity index 57% rename from jobs/rocoto/waveprep.sh rename to dev/jobs/waveprep.sh index b4816726a49..24e75ed0c24 100755 --- a/jobs/rocoto/waveprep.sh +++ b/dev/jobs/waveprep.sh @@ -3,10 +3,8 @@ set -x ############################################################### -echo -echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ===============" -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -. ${HOMEgfs}/ush/load_ufswm_modules.sh +#source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" +source "${HOMEgfs}/ush/load_ufswm_modules.sh" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" @@ -16,10 +14,8 @@ export job="waveprep" export jobid="${job}.$$" ############################################################### -echo -echo "=============== START TO RUN WAVE PREP ===============" # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_WAVE_PREP +"${HOMEgfs}/jobs/JGLOBAL_WAVE_PREP" status=$? if [[ ${status} -ne 0 ]]; then exit "${status}" diff --git a/test/README.md b/dev/test/README.md similarity index 100% rename from test/README.md rename to dev/test/README.md diff --git a/test/coordinates.lst b/dev/test/coordinates.lst similarity index 100% rename from test/coordinates.lst rename to dev/test/coordinates.lst diff --git a/test/diff_ROTDIR.sh b/dev/test/diff_ROTDIR.sh similarity index 61% rename from test/diff_ROTDIR.sh rename to dev/test/diff_ROTDIR.sh index 3a864f5f091..a42af8283f5 100755 --- a/test/diff_ROTDIR.sh +++ b/dev/test/diff_ROTDIR.sh @@ -2,10 +2,10 @@ # # Differences relevant output files in two different experiment ROTDIRs. -# Text files are compared via posix diff. GRiB files are compared via +# Text files are compared via posix diff. GRiB files are compared via # correlation reported by wgrib2. NetCDF files are compared by using # NetCDF operators to calculate a diff then make sure all non-coordinate -# variable differences are zero. File lists are created by globbing key +# variable differences are zero. File lists are created by globbing key # directories under the first experiment given. # # Syntax: @@ -19,8 +19,8 @@ # rotdir: root rotdir where ROTDIRS are held # cdate: experiment date/cycle in YYYYMMDDHH format # expA, expB: experiment ids (PSLOT) to compare -# -# dirA, dirB: full paths to the cycle directories to be compared +# +# dirA, dirB: full paths to the cycle directories to be compared # (${rotdir}/${exp}/gfs.${YYYYMMDD}/${cyc}) # # Options: @@ -30,33 +30,36 @@ set -eu +# shellcheck disable=SC2311 # TODO: #FIXME: remove this when this script is corrected + usage() { # # Print usage statement # - echo <<- 'EOF' + # shellcheck disable=SC2217 + echo <<-'EOF' Differences relevant output files in two different experiment ROTDIRs. - Text files are compared via posix diff. GRiB files are compared via + Text files are compared via posix diff. GRiB files are compared via correlation reported by wgrib2. NetCDF files are compared by using NetCDF operators to calculate a diff then make sure all non-coordinate - variable differences are zero. File lists are created by globbing key + variable differences are zero. File lists are created by globbing key directories under the first experiment given. Syntax: diff_ROTDIR.sh [-c coord_file][-h] rotdir cdate expA expB - + OR - + diff_ROTDIR.sh [-c coord_file][-h] dirA dirB - + Arguments: rotdir: root rotdir where ROTDIRS are held cdate: experiment date/cycle in YYYYMMDDHH format expA, expB: experiment ids (PSLOT) to compare - - dirA, dirB: full paths to the cycle directories to be compared + + dirA, dirB: full paths to the cycle directories to be compared (${rotdir}/${exp}/gfs.${YYYYMMDD}/${cyc}) - + Options: -c coord_file: file containing a list of coordinate variables -h: print usage message and exit @@ -65,38 +68,42 @@ usage() { while getopts ":c:h" option; do case "${option}" in - c) coord_file=${OPTARG} ;; - h) usage; exit 0 ;; - *) echo "Unknown option ${option}"; exit 1 ;; + c) coord_file=${OPTARG} ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option ${option}" + exit 1 + ;; esac done num_args=$# -case $num_args in - 2) # Direct directory paths - dirA=$1 - dirB=$2 - ;; - 4) # Derive directory paths - rotdir=$1 - date=$2 - expA=$3 - expB=$4 - - YYYYMMDD=$(echo $date | cut -c1-8) - cyc=$(echo $date | cut -c9-10) - dirA="$rotdir/$expA/gfs.${YYYYMMDD}/${cyc}" - dirB="$rotdir/$expB/gfs.${YYYYMMDD}/${cyc}" - ;; - *) # Unknown option - echo "${num_args} is not a valid number of arguments, use 2 or 4" - usage - exit 1 - ;; +case ${num_args} in +2) # Direct directory paths + dirA=$1 + dirB=$2 + ;; +4) # Derive directory paths + rotdir=$1 + date=$2 + expA=$3 + expB=$4 + + YYYYMMDD=$(echo "${date}" | cut -c1-8) + cyc=$(echo "${date}" | cut -c9-10) + dirA="${rotdir}/${expA}/gfs.${YYYYMMDD}/${cyc}" + dirB="${rotdir}/${expB}/gfs.${YYYYMMDD}/${cyc}" + ;; +*) # Unknown option + echo "${num_args} is not a valid number of arguments, use 2 or 4" + usage + exit 1 + ;; esac -temp_file=".diff.nc" - # Contains a bunch of NetCDF Operator shortcuts (will load nco module) source ./netcdf_op_functions.sh source ./test_utils.sh @@ -106,22 +113,23 @@ coord_file="${coord_file:-./coordinates.lst}" ## Text files files="" files="${files} atmos/input.nml" # This file will be different because of the fix paths -files="${files} $(basename_list 'atmos/' "$dirA/atmos/storms.*" "$dirA/atmos/trak.*")" -if [[ -d $dirA/ice ]]; then +#shellcheck disable=SC2086 +files="${files} $(basename_list 'atmos/' ${dirA}/atmos/storms.* ${dirA}/atmos/trak.*)" +if [[ -d "${dirA}/ice" ]]; then files="${files} ice/ice_in" fi -if [[ -d $dirA/ocean ]]; then +if [[ -d "${dirA}/ocean" ]]; then files="${files} ocean/MOM_input" fi # if [[ -d $dirA/wave ]]; then # files="${files} $(basename_list 'wave/station/' "$dirA/wave/station/*bull_tar")" # fi -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - diff $fileA $fileB || : + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + diff "${fileA}" "${fileB}" || : done ## GRiB files @@ -129,34 +137,34 @@ done module load wgrib2/2.0.8 files="" -files="${files} $(basename_list 'atmos/' $dirA/atmos/*grb2* $dirA/atmos/*.flux.*)" -if [[ -d $dirA/wave ]]; then - files="${files} $(basename_list 'wave/gridded/' $dirA/wave/gridded/*.grib2)" +files="${files} $(basename_list 'atmos/' "${dirA}"/atmos/*grb2* "${dirA}"/atmos/*.flux.*)" +if [[ -d ${dirA}/wave ]]; then + files="${files} $(basename_list 'wave/gridded/' "${dirA}"/wave/gridded/*.grib2)" fi -if [[ -d $dirA/ocean ]]; then - files="${files} $(basename_list 'ocean/' $dirA/ocean/*grb2)" +if [[ -d ${dirA}/ocean ]]; then + files="${files} $(basename_list 'ocean/' "${dirA}"/ocean/*grb2)" fi -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - ./diff_grib_files.py $fileA $fileB + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + ./diff_grib_files.py "${fileA}" "${fileB}" done ## NetCDF Files files="" -files="${files} $(basename_list 'atmos/' $dirA/atmos/*.nc)" -if [[ -d $dirA/ice ]]; then - files="${files} $(basename_list 'ice/' $dirA/ice/*.nc)" +files="${files} $(basename_list 'atmos/' "${dirA}"/atmos/*.nc)" +if [[ -d ${dirA}/ice ]]; then + files="${files} $(basename_list 'ice/' "${dirA}"/ice/*.nc)" fi -if [[ -d $dirA/ocean ]]; then - files="${files} $(basename_list 'ocean/' $dirA/ocean/*.nc)" +if [[ -d ${dirA}/ocean ]]; then + files="${files} $(basename_list 'ocean/' "${dirA}"/ocean/*.nc)" fi -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - nccmp -q $fileA $fileB $coord_file + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + nccmp -q "${fileA}" "${fileB}" "${coord_file}" done diff --git a/test/diff_UFS_rundir.sh b/dev/test/diff_UFS_rundir.sh similarity index 62% rename from test/diff_UFS_rundir.sh rename to dev/test/diff_UFS_rundir.sh index a305497ef38..f9b5125c71c 100755 --- a/test/diff_UFS_rundir.sh +++ b/dev/test/diff_UFS_rundir.sh @@ -1,7 +1,7 @@ #! /bin/env bash # -# Differences relevant output files in two UFS model directories. GRiB files +# Differences relevant output files in two UFS model directories. GRiB files # are compared via correlation reported by wgrib2. NetCDF files are compared # by using NetCDF operators to calculate a diff then make sure all non- # coordinate variable differences are zero. @@ -19,12 +19,15 @@ set -eu +# shellcheck disable=SC2311 # TODO: #FIXME: remove this when this script is corrected + usage() { # # Print usage statement # - echo <<- 'EOF' - Differences relevant output files in two UFS model directories. GRiB files + #shellcheck disable=SC2217 + echo <<-'EOF' + Differences relevant output files in two UFS model directories. GRiB files are compared via correlation reported by wgrib2. NetCDF files are compared by using NetCDF operators to calculate a diff then make sure all non- coordinate variable differences are zero. @@ -43,68 +46,74 @@ usage() { while getopts ":c:h" option; do case "${option}" in - c) coord_file=${OPTARG} ;; - h) usage; exit 0 ;; - *) echo "Unknown option ${option}"; exit 1 ;; + c) coord_file=${OPTARG} ;; + h) + usage + exit 0 + ;; + *) + echo "Unknown option ${option}" + exit 1 + ;; esac done num_args=$# -case $num_args in - 2) # Direct directory paths - dirA=$1 - dirB=$2 - ;; - *) # Unknown option - echo "${num_args} is not a valid number of arguments, use 2" - usage - exit 1 - ;; +case ${num_args} in +2) # Direct directory paths + dirA=$1 + dirB=$2 + ;; +*) # Unknown option + echo "${num_args} is not a valid number of arguments, use 2" + usage + exit 1 + ;; esac source ./netcdf_op_functions.sh source ./test_utils.sh -temp_file=".diff.nc" coord_file="${coord_file:-./coordinates.lst}" # Input files files="data_table diag_table fd_ufs.yaml field_table ice_in input.nml med_modelio.nml \ model_configure ufs.configure pio_in ww3_multi.inp ww3_shel.inp" -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - if [[ -f "$fileA" ]]; then - diff $fileA $fileB || : + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + if [[ -f "${fileA}" ]]; then + diff "${fileA}" "${fileB}" || : else - echo ; echo; + echo + echo + fi done # GRiB files -files="$(basename_list '' $dirA/GFSFLX.Grb*)" +files="$(basename_list '' "${dirA}"/GFSFLX.Grb*)" module load wgrib2/2.0.8 -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - ./diff_grib_files.py $fileA $fileB + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + ./diff_grib_files.py "${fileA}" "${fileB}" done # NetCDF Files files="" -files="${files} $(basename_list '' $dirA/atmf*.nc $dirA/sfcf*.nc)" -if [[ -d "$dirA/history" ]]; then - files="$(basename_list 'history/' $dirA/history/*.nc)" +files="${files} $(basename_list '' "${dirA}"/atmf*.nc "${dirA}"/sfcf*.nc)" +if [[ -d "${dirA}/history" ]]; then + files="$(basename_list 'history/' "${dirA}"/history/*.nc)" fi -for file in $files; do +for file in ${files}; do echo "=== ${file} ===" - fileA="$dirA/$file" - fileB="$dirB/$file" - nccmp -q $fileA $fileB $coord_file + fileA="${dirA}/${file}" + fileB="${dirB}/${file}" + nccmp -q "${fileA}" "${fileB}" "${coord_file}" done - diff --git a/test/diff_grib_files.py b/dev/test/diff_grib_files.py similarity index 98% rename from test/diff_grib_files.py rename to dev/test/diff_grib_files.py index 9c01afbb18e..ef046383b13 100755 --- a/test/diff_grib_files.py +++ b/dev/test/diff_grib_files.py @@ -1,4 +1,4 @@ -#! /bin/env python3 +#! /usr/bin/env python3 ''' Compares two grib2 files and print any variables that have a non-identity correlation. diff --git a/test/f90nmlcmp.sh b/dev/test/f90nmlcmp.sh similarity index 68% rename from test/f90nmlcmp.sh rename to dev/test/f90nmlcmp.sh index 0acb2b711c5..1b4ad42749b 100755 --- a/test/f90nmlcmp.sh +++ b/dev/test/f90nmlcmp.sh @@ -5,7 +5,7 @@ set -eu # shellcheck disable=SC2155,SC2312 -HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) +HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../.." && pwd -P) declare -rx HOMEgfs source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 @@ -13,7 +13,7 @@ source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 file1=${1:?} file2=${2:?} -"${HOMEgfs}/ush/compare_f90nml.py" "${file1}" "${file2}" +"${HOMEgfs}/dev/ush/compare_f90nml.py" "${file1}" "${file2}" echo " " -"${HOMEgfs}/ush/compare_f90nml.py" "${file2}" "${file1}" +"${HOMEgfs}/dev/ush/compare_f90nml.py" "${file2}" "${file1}" echo " " diff --git a/test/g2cmp.sh b/dev/test/g2cmp.sh similarity index 96% rename from test/g2cmp.sh rename to dev/test/g2cmp.sh index f4a9e50c238..b7c21eb6b68 100755 --- a/test/g2cmp.sh +++ b/dev/test/g2cmp.sh @@ -6,7 +6,7 @@ set -eu # shellcheck disable=SC2155,SC2312 -HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) +HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../.." && pwd -P) declare -rx HOMEgfs file1=${1:?} diff --git a/test/nccmp.sh b/dev/test/nccmp.sh similarity index 94% rename from test/nccmp.sh rename to dev/test/nccmp.sh index b412cdc388c..6ebb073121c 100755 --- a/test/nccmp.sh +++ b/dev/test/nccmp.sh @@ -3,7 +3,7 @@ set -eu # shellcheck disable=SC2155,SC2312 -HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/.." && pwd -P) +HOMEgfs=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )/../.." && pwd -P) declare -rx HOMEgfs source "${HOMEgfs}/ush/load_fv3gfs_modules.sh" 1>/dev/null 2>&1 diff --git a/test/netcdf_op_functions.sh b/dev/test/netcdf_op_functions.sh similarity index 55% rename from test/netcdf_op_functions.sh rename to dev/test/netcdf_op_functions.sh index 0085855ea3f..4d6e90c9185 100644 --- a/test/netcdf_op_functions.sh +++ b/dev/test/netcdf_op_functions.sh @@ -1,96 +1,99 @@ #! /bin/env bash -if [ -t 0 ]; then +# shellcheck disable=SC2312 +# shellcheck disable=SC2317 # TODO: remove this when the script is corrected of errors + +if [[ -t 0 ]]; then module load nco/4.9.3 fi ## NetCDF operator shortcuts # From nco.sourceforge.net/nco.html#Filters-for-ncks # ncattget $att_nm $var_nm $fl_nm : What attributes does variable have? -function ncattget { ncks --trd -M -m ${3} | grep -E -i "^${2} attribute [0-9]+: ${1}" | cut -f 11- -d ' ' | sort ; } +function ncattget { ncks --trd -M -m "${3}" | grep -E -i "^${2} attribute [0-9]+: ${1}" | cut -f 11- -d ' ' | sort ; } # ncunits $att_val $fl_nm : Which variables have given units? -function ncunits { ncks --trd -m ${2} | grep -E -i " attribute [0-9]+: units.+ ${1}" | cut -f 1 -d ' ' | sort ; } +function ncunits { ncks --trd -m "${2}" | grep -E -i " attribute [0-9]+: units.+ ${1}" | cut -f 1 -d ' ' | sort ; } # ncavg $var_nm $fl_nm : What is mean of variable? function ncavg { - temp_file=${PTMP:-$HOME}/foo.nc - ncwa -y avg -O -C -v ${1} ${2} ${temp_file} - ncks --trd -H -C -v ${1} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncwa -y avg -O -C -v "${1}" "${2}" "${temp_file}" + ncks --trd -H -C -v "${1}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncavg $var_nm $fl_nm : What is mean of variable? function ncavg { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo=${1}.avg();print(foo)" ${2} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo=${1}.avg();print(foo)" "${2}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncdmnlst $fl_nm : What dimensions are in file? -function ncdmnlst { ncks --cdl -m ${1} | cut -d ':' -f 1 | cut -d '=' -s -f 1 ; } +function ncdmnlst { ncks --cdl -m "${1}" | cut -d ':' -f 1 | cut -d '=' -s -f 1 ; } # ncvardmnlst $var_nm $fl_nm : What dimensions are in a variable? -function ncvardmnlst { ncks --trd -m -v ${1} ${2} | grep -E -i "^${1} dimension [0-9]+: " | cut -f 4 -d ' ' | sed 's/,//' ; } +function ncvardmnlst { ncks --trd -m -v "${1}" "${2}" | grep -E -i "^${1} dimension [0-9]+: " | cut -f 4 -d ' ' | sed 's/,//' ; } # ncvardmnlatlon $var_nm $fl_nm : Does variable contain both lat and lon dimensions? # function ncvardmnlatlon { flg=$(ncks -C -v ${1} -m ${2} | grep -E -i "${1}\(" | grep -E "lat.*lon|lon.*lat") ; [[ ! -z "$flg" ]] && echo "Yes, ${1} has both lat and lon dimensions" || echo "No, ${1} does not have both lat and lon dimensions" } # ncdmnsz $dmn_nm $fl_nm : What is dimension size? -function ncdmnsz { ncks --trd -m -M ${2} | grep -E -i ": ${1}, size =" | cut -f 7 -d ' ' | uniq ; } +function ncdmnsz { ncks --trd -m -M "${2}" | grep -E -i ": ${1}, size =" | cut -f 7 -d ' ' | uniq ; } # ncgrplst $fl_nm : What groups are in file? -function ncgrplst { ncks -m ${1} | grep 'group:' | cut -d ':' -f 2 | cut -d ' ' -f 2 | sort ; } +function ncgrplst { ncks -m "${1}" | grep 'group:' | cut -d ':' -f 2 | cut -d ' ' -f 2 | sort ; } # ncvarlst $fl_nm : What variables are in file? -function ncvarlst { ncks --trd -m ${1} | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort ; } +function ncvarlst { ncks --trd -m "${1}" | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort ; } # ncmax $var_nm $fl_nm : What is maximum of variable? function ncmax { - temp_file=${PTMP:-$HOME}/foo.nc - ncwa -y max -O -C -v ${1} ${2} ${temp_file} - ncks --trd -H -C -v ${1} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncwa -y max -O -C -v "${1}" "${2}" "${temp_file}" + ncks --trd -H -C -v "${1}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncmax $var_nm $fl_nm : What is maximum of variable? function ncmax { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo=${1}.max();print(foo)" ${2} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo=${1}.max();print(foo)" "${2}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncmdn $var_nm $fl_nm : What is median of variable? function ncmdn { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo=gsl_stats_median_from_sorted_data(${1}.sort());print(foo)" ${2} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo=gsl_stats_median_from_sorted_data(${1}.sort());print(foo)" "${2}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncmin $var_nm $fl_nm : What is minimum of variable? function ncmin { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo=${1}.min();print(foo)" ${2} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo=${1}.min();print(foo)" "${2}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncrng $var_nm $fl_nm : What is range of variable? function ncrng { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo_min=${1}.min();foo_max=${1}.max();print(foo_min,\"%f\");print(\" to \");print(foo_max,\"%f\")" ${2} ${temp_file} - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo_min=${1}.min();foo_max=${1}.max();print(foo_min,\"%f\");print(\" to \");print(foo_max,\"%f\")" "${2}" "${temp_file}" + rm "${temp_file}" } # ncmode $var_nm $fl_nm : What is mode of variable? function ncmode { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo=gsl_stats_median_from_sorted_data(${1}.sort());print(foo)" ${2} ${temp_file} | cut -f 3- -d ' ' - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo=gsl_stats_median_from_sorted_data(${1}.sort());print(foo)" "${2}" "${temp_file}" | cut -f 3- -d ' ' + rm "${temp_file}" } # ncrecsz $fl_nm : What is record dimension size? -function ncrecsz { ncks --trd -M ${1} | grep -E -i "^Root record dimension 0:" | cut -f 10- -d ' ' ; } +function ncrecsz { ncks --trd -M "${1}" | grep -E -i "^Root record dimension 0:" | cut -f 10- -d ' ' ; } # nctypget $var_nm $fl_nm : What type is variable? -function nctypget { ncks --trd -m -v ${1} ${2} | grep -E -i "^${1}: type" | cut -f 3 -d ' ' | cut -f 1 -d ',' ; } +function nctypget { ncks --trd -m -v "${1}" "${2}" | grep -E -i "^${1}: type" | cut -f 3 -d ' ' | cut -f 1 -d ',' ; } function nccorr() { - temp_file=${PTMP:-$HOME}/foo.nc - ncap2 -O -C -v -s "foo_min=${1}.min();foo_max=${1}.max();print(foo_min,\"%f\");print(\" to \");print(foo_max,\"%f\")" ${2} ${temp_file} - rm ${temp_file} + temp_file=${PTMP:-${HOME}}/foo.nc + ncap2 -O -C -v -s "foo_min=${1}.min();foo_max=${1}.max();print(foo_min,\"%f\");print(\" to \");print(foo_max,\"%f\")" "${2}" "${temp_file}" + rm "${temp_file}" } # Heavily modified from original function nccmp() { - # + # # Compare two netcdf files # - # Uses ncdiff to create a difference of two NetCDFs, then checks to + # Uses ncdiff to create a difference of two NetCDFs, then checks to # make sure all non-coordinate fields of the diff are zero. - # + # # Syntax: # nccmp [-q][-z] fileA fileB coord_file # @@ -103,8 +106,8 @@ function nccmp() { # -z: suppress displaying fields with zero difference # # Notes: - # Will create a temporary file .diff in the $PTMP directory - # if PTMP is defined, otherwise .diff is created in the + # Will create a temporary file .diff in the $PTMP directory + # if PTMP is defined, otherwise .diff is created in the # current directory. # @@ -132,46 +135,44 @@ function nccmp() { echo "Comparing ${fileA} and ${fileB}" fi # Create diff of the files - ncdiff ${fileA} ${fileB} ${temp_file} --overwrite + ncdiff "${fileA}" "${fileB}" "${temp_file}" --overwrite if [[ ${quiet} == "NO" ]]; then echo "Difference report:${suppress_msg}" echo "(Coordinate variables will always be non-zero)" fi count=0 # Check each variable - for var in $(ncvarlst ${temp_file}); do - if [[ $(egrep -o "^${var}\$" ${coord_file} | wc -l) == 0 ]]; then + for var in $(ncvarlst "${temp_file}"); do + if [[ $(grep -E -o "^${var}\$" "${coord_file}" | wc -l) == 0 ]]; then # Variable is not in coordinate list - max=$(ncmax $var $temp_file 2> /dev/null) - if [[ -z $max ]]; then + max=$(ncmax "${var}" "${temp_file}" 2> /dev/null) + if [[ -z ${max} ]]; then echo "Error reading max of ${var}" count=$((count + 1)) continue fi - min=$(ncmin $var $temp_file 2> /dev/null) - if [[ -z $min ]]; then + min=$(ncmin "${var}" "${temp_file}" 2> /dev/null) + if [[ -z ${min} ]]; then echo "Error reading min of ${var}" count=$((count + 1)) continue fi - if [[ ${hide_zeros} == "NO" ]] || (( $(echo "$max != 0 || $min != 0" | bc) )); then + if [[ ${hide_zeros} == "NO" ]] || (( $(echo "${max} != 0 || ${min} != 0" | bc) )); then # Min/max is not zero or we are not hiding zeros echo "${var}: ${min}..${max}" count=$((count + 1)) fi else - # + # # ncdiff doesn't difference coordinate variables. Instead coordinates - # are just placed in the diff file. While this is generally what we + # are just placed in the diff file. While this is generally what we # want, when checking for equivilence we need to ignore them. # if [[ ${quiet} == "NO" ]]; then - echo "$Coordinate ${var} ignored" + echo "${Coordinate} ${var} ignored" fi fi done - rm $temp_file + rm "${temp_file}" echo "${count} differences found" } - - diff --git a/test/test_utils.sh b/dev/test/test_utils.sh similarity index 62% rename from test/test_utils.sh rename to dev/test/test_utils.sh index b00e1d49cfe..914886dd5cd 100644 --- a/test/test_utils.sh +++ b/dev/test/test_utils.sh @@ -2,7 +2,7 @@ basename_list() { # - # Take a list of paths, determines the base name, then + # Take a list of paths, determines the base name, then # prepends it to a base path. # # Syntax: @@ -13,14 +13,15 @@ basename_list() { # file_in: List of paths relative to $base/ # # Returns: - # List of paths constructed by prepending $base to each + # List of paths constructed by prepending $base to each # item in $file_in # base="${1}" list="" for file_in in "${@:2}"; do - list="$list ${base}$(basename $file_in)" + # shellcheck disable=SC2086 + list="${list} ${base}$(basename ${file_in})" done - echo $list + echo "${list}" } diff --git a/ush/compare_f90nml.py b/dev/ush/compare_f90nml.py similarity index 100% rename from ush/compare_f90nml.py rename to dev/ush/compare_f90nml.py diff --git a/ush/fetch-fix-data.py b/dev/ush/fetch-fix-data.py similarity index 100% rename from ush/fetch-fix-data.py rename to dev/ush/fetch-fix-data.py diff --git a/workflow/gw_setup.sh b/dev/ush/gw_setup.sh similarity index 81% rename from workflow/gw_setup.sh rename to dev/ush/gw_setup.sh index 930b1aab776..12378379f44 100755 --- a/workflow/gw_setup.sh +++ b/dev/ush/gw_setup.sh @@ -2,13 +2,13 @@ # # Resets the lmod environment and loads the modules necessary to run all the -# scripts necessary to prepare the workflow for use (checkout, experiment +# scripts necessary to prepare the workflow for use (checkout, experiment # setup, etc.). # # This script should be SOURCED to properly setup the environment. # -HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null 2>&1 && pwd )" +HOMEgfs="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )" source "${HOMEgfs}/ush/detect_machine.sh" source "${HOMEgfs}/ush/module-setup.sh" module use "${HOMEgfs}/modulefiles" diff --git a/workflow/README_ecflow.md b/dev/workflow/README_ecflow.md similarity index 100% rename from workflow/README_ecflow.md rename to dev/workflow/README_ecflow.md diff --git a/workflow/__init.py__ b/dev/workflow/__init.py__ similarity index 100% rename from workflow/__init.py__ rename to dev/workflow/__init.py__ diff --git a/workflow/applications/__init__.py b/dev/workflow/applications/__init__.py similarity index 100% rename from workflow/applications/__init__.py rename to dev/workflow/applications/__init__.py diff --git a/workflow/applications/application_factory.py b/dev/workflow/applications/application_factory.py similarity index 100% rename from workflow/applications/application_factory.py rename to dev/workflow/applications/application_factory.py diff --git a/workflow/applications/applications.py b/dev/workflow/applications/applications.py similarity index 100% rename from workflow/applications/applications.py rename to dev/workflow/applications/applications.py diff --git a/workflow/applications/gefs.py b/dev/workflow/applications/gefs.py similarity index 100% rename from workflow/applications/gefs.py rename to dev/workflow/applications/gefs.py diff --git a/workflow/applications/gfs_cycled.py b/dev/workflow/applications/gfs_cycled.py similarity index 100% rename from workflow/applications/gfs_cycled.py rename to dev/workflow/applications/gfs_cycled.py diff --git a/workflow/applications/gfs_forecast_only.py b/dev/workflow/applications/gfs_forecast_only.py similarity index 100% rename from workflow/applications/gfs_forecast_only.py rename to dev/workflow/applications/gfs_forecast_only.py diff --git a/workflow/applications/sfs.py b/dev/workflow/applications/sfs.py similarity index 100% rename from workflow/applications/sfs.py rename to dev/workflow/applications/sfs.py diff --git a/workflow/build_compute.py b/dev/workflow/build_compute.py similarity index 99% rename from workflow/build_compute.py rename to dev/workflow/build_compute.py index 089cb740121..e87e1943b25 100755 --- a/workflow/build_compute.py +++ b/dev/workflow/build_compute.py @@ -15,7 +15,7 @@ _here = os.path.dirname(__file__) -HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '..')) +HOMEgfs = os.path.abspath(os.path.join(os.path.abspath(_here), '../..')) def input_args(*argv): diff --git a/workflow/build_opts.yaml b/dev/workflow/build_opts.yaml similarity index 100% rename from workflow/build_opts.yaml rename to dev/workflow/build_opts.yaml diff --git a/workflow/create_experiment.py b/dev/workflow/create_experiment.py similarity index 95% rename from workflow/create_experiment.py rename to dev/workflow/create_experiment.py index 8f935ab8a5c..88578b62cda 100755 --- a/workflow/create_experiment.py +++ b/dev/workflow/create_experiment.py @@ -2,11 +2,11 @@ """ Basic python script to create an experiment directory on the fly from a given -yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow +yaml file for the arguments to the two scripts below in ${HOMEgfs}/dev/workflow where ${HOMEgfs} is determined from the location of this script. - ${HOMEgfs}/workflow/setup_expt.py - ${HOMEgfs}/workflow/setup_xml.py + ${HOMEgfs}/dev/workflow/setup_expt.py + ${HOMEgfs}/dev/workflow/setup_xml.py The yaml file are simply the arguments for these two scripts. After this scripts runs the experiment is ready for launch. @@ -37,7 +37,7 @@ _here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '..')) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..')) # Setup the logger logger = Logger(logfile_path=os.environ.get("LOGFILE_PATH"), diff --git a/workflow/ecFlow/__init.py__ b/dev/workflow/ecFlow/__init.py__ similarity index 100% rename from workflow/ecFlow/__init.py__ rename to dev/workflow/ecFlow/__init.py__ diff --git a/workflow/ecFlow/ecflow_definitions.py b/dev/workflow/ecFlow/ecflow_definitions.py similarity index 100% rename from workflow/ecFlow/ecflow_definitions.py rename to dev/workflow/ecFlow/ecflow_definitions.py diff --git a/workflow/ecFlow/ecflow_setup.py b/dev/workflow/ecFlow/ecflow_setup.py similarity index 100% rename from workflow/ecFlow/ecflow_setup.py rename to dev/workflow/ecFlow/ecflow_setup.py diff --git a/workflow/ecflow_build.yml b/dev/workflow/ecflow_build.yml similarity index 100% rename from workflow/ecflow_build.yml rename to dev/workflow/ecflow_build.yml diff --git a/workflow/generate_workflows.sh b/dev/workflow/generate_workflows.sh similarity index 96% rename from workflow/generate_workflows.sh rename to dev/workflow/generate_workflows.sh index 3b71ffcc24f..5669b75676d 100755 --- a/workflow/generate_workflows.sh +++ b/dev/workflow/generate_workflows.sh @@ -29,7 +29,7 @@ function _usage() { -D Delete the RUNTESTS and DATAROOT directories if they already exist -Y /path/to/directory/with/YAMLs - If this option is not specified, then the \${HOMEgfs}/ci/cases/pr + If this option is not specified, then the \${HOMEgfs}/dev/ci/cases/pr directory is used. -G Run all valid GFS cases in the specified YAML directory. @@ -49,7 +49,7 @@ function _usage() { -A "HPC account name" Set the HPC account name. If this is not set, the default in - \$HOMEgfs/ci/platform/config.\$machine + \$HOMEgfs/dev/ci/platform/config.\$machine will be used. -c Append the chosen set of tests to your existing crontab @@ -260,15 +260,15 @@ fi # Set HOMEgfs if it wasn't set by the user if [[ "${_specified_home}" == "false" ]]; then script_relpath="$(dirname "${BASH_SOURCE[0]}")" - HOMEgfs="$(cd "${script_relpath}/.." && pwd)" + HOMEgfs="$(cd "${script_relpath}/../.." && pwd)" if [[ "${_verbose}" == "true" ]]; then printf "Setting HOMEgfs to %s\n\n" "${HOMEgfs}" fi fi -# Set the _yaml_dir to HOMEgfs/ci/cases/pr if not explicitly set +# Set the _yaml_dir to HOMEgfs/dev/ci/cases/pr if not explicitly set if [[ "${_specified_yaml_dir}" == false ]]; then - _yaml_dir="${HOMEgfs}/ci/cases/pr" + _yaml_dir="${HOMEgfs}/dev/ci/cases/pr" fi function select_all_yamls() @@ -325,7 +325,7 @@ EOM _yaml="${_nameref_yaml_list[${i}]}" _found=$(grep -l "system: *${system}" "${_yaml_dir}/${_yaml}.yaml") if [[ -z "${_found}" ]]; then - echo "WARNING the yaml file ${_yaml_dir}/${_yaml}.yaml is not designed for the ${_SYSTEM} system" + echo "WARNING: the yaml file ${_yaml_dir}/${_yaml}.yaml is not designed for the ${_SYSTEM} system" echo "Removing this yaml from the set of cases to run" unset '_nameref_yaml_list[${i}]' # Sleep 2 seconds to give the user a moment to react @@ -371,9 +371,9 @@ fi if [[ "${_debug}" == "true" ]]; then set +x fi -if ! source "${HOMEgfs}/workflow/gw_setup.sh" >& stdout; then +if ! source "${HOMEgfs}/dev/ush/gw_setup.sh" >& stdout; then cat stdout - echo "Failed to source ${HOMEgfs}/workflow/gw_setup.sh!" + echo "Failed to source ${HOMEgfs}/dev/ush/gw_setup.sh!" exit 7 fi if [[ "${_verbose}" == "true" ]]; then @@ -385,9 +385,9 @@ if [[ "${_debug}" == "true" ]]; then fi set -u machine=${MACHINE_ID} -platform_config="${HOMEgfs}/ci/platforms/config.${machine}" +platform_config="${HOMEgfs}/dev/ci/platforms/config.${machine}" if [[ -f "${platform_config}" ]]; then - . "${HOMEgfs}/ci/platforms/config.${machine}" + source "${HOMEgfs}/dev/ci/platforms/config.${machine}" else if [[ "${_set_account}" == "false" ]] ; then echo "ERROR Unknown HPC account! Please use the -A option to specify." @@ -395,9 +395,9 @@ else fi fi -# If _yaml_dir is not set, set it to $HOMEgfs/ci/cases/pr +# If _yaml_dir is not set, set it to $HOMEgfs/dev/ci/cases/pr if [[ -z ${_yaml_dir} ]]; then - _yaml_dir="${HOMEgfs}/ci/cases/pr" + _yaml_dir="${HOMEgfs}/dev/ci/cases/pr" fi # Update submodules if requested diff --git a/workflow/hosts.py b/dev/workflow/hosts.py similarity index 100% rename from workflow/hosts.py rename to dev/workflow/hosts.py diff --git a/workflow/hosts/awspw.yaml b/dev/workflow/hosts/awspw.yaml similarity index 100% rename from workflow/hosts/awspw.yaml rename to dev/workflow/hosts/awspw.yaml diff --git a/workflow/hosts/azurepw.yaml b/dev/workflow/hosts/azurepw.yaml similarity index 100% rename from workflow/hosts/azurepw.yaml rename to dev/workflow/hosts/azurepw.yaml diff --git a/workflow/hosts/container.yaml b/dev/workflow/hosts/container.yaml similarity index 100% rename from workflow/hosts/container.yaml rename to dev/workflow/hosts/container.yaml diff --git a/workflow/hosts/gaeac5.yaml b/dev/workflow/hosts/gaeac5.yaml similarity index 100% rename from workflow/hosts/gaeac5.yaml rename to dev/workflow/hosts/gaeac5.yaml diff --git a/workflow/hosts/gaeac6.yaml b/dev/workflow/hosts/gaeac6.yaml similarity index 100% rename from workflow/hosts/gaeac6.yaml rename to dev/workflow/hosts/gaeac6.yaml diff --git a/workflow/hosts/googlepw.yaml b/dev/workflow/hosts/googlepw.yaml similarity index 100% rename from workflow/hosts/googlepw.yaml rename to dev/workflow/hosts/googlepw.yaml diff --git a/workflow/hosts/hera.yaml b/dev/workflow/hosts/hera.yaml similarity index 100% rename from workflow/hosts/hera.yaml rename to dev/workflow/hosts/hera.yaml diff --git a/workflow/hosts/hercules.yaml b/dev/workflow/hosts/hercules.yaml similarity index 100% rename from workflow/hosts/hercules.yaml rename to dev/workflow/hosts/hercules.yaml diff --git a/workflow/hosts/jet.yaml b/dev/workflow/hosts/jet.yaml similarity index 100% rename from workflow/hosts/jet.yaml rename to dev/workflow/hosts/jet.yaml diff --git a/workflow/hosts/orion.yaml b/dev/workflow/hosts/orion.yaml similarity index 100% rename from workflow/hosts/orion.yaml rename to dev/workflow/hosts/orion.yaml diff --git a/workflow/hosts/s4.yaml b/dev/workflow/hosts/s4.yaml similarity index 100% rename from workflow/hosts/s4.yaml rename to dev/workflow/hosts/s4.yaml diff --git a/workflow/hosts/wcoss2.yaml b/dev/workflow/hosts/wcoss2.yaml similarity index 100% rename from workflow/hosts/wcoss2.yaml rename to dev/workflow/hosts/wcoss2.yaml diff --git a/workflow/prod.yml b/dev/workflow/prod.yml similarity index 100% rename from workflow/prod.yml rename to dev/workflow/prod.yml diff --git a/workflow/requirements.txt b/dev/workflow/requirements.txt similarity index 100% rename from workflow/requirements.txt rename to dev/workflow/requirements.txt diff --git a/workflow/rocoto/__init__.py b/dev/workflow/rocoto/__init__.py similarity index 100% rename from workflow/rocoto/__init__.py rename to dev/workflow/rocoto/__init__.py diff --git a/workflow/rocoto/gefs_tasks.py b/dev/workflow/rocoto/gefs_tasks.py similarity index 96% rename from workflow/rocoto/gefs_tasks.py rename to dev/workflow/rocoto/gefs_tasks.py index 9b20b95d32d..591657eb056 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/dev/workflow/rocoto/gefs_tasks.py @@ -16,7 +16,7 @@ def stage_ic(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -33,7 +33,7 @@ def waveinit(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -50,7 +50,7 @@ def prep_emissions(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/prep_emissions.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prep_emissions.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -88,7 +88,7 @@ def fcst(self): 'dependency': dependencies, 'envars': fcst_vars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -145,7 +145,7 @@ def efcs(self): 'dependency': dependencies, 'envars': efcsenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -246,7 +246,7 @@ def _atmosoceaniceprod(self, component: str): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/{config}.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/{config}.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} @@ -288,7 +288,7 @@ def postsnd(self): 'dependency': dependencies, 'envars': postsnd_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/postsnd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -343,7 +343,7 @@ def gempak(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempak.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -400,7 +400,7 @@ def atmos_ensstat(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmos_ensstat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} @@ -456,7 +456,7 @@ def wavepostsbs(self): 'dependency': dependencies, 'envars': wave_post_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostsbs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -495,7 +495,7 @@ def wavepostbndpnt(self): 'dependency': dependencies, 'envars': wave_post_bndpnt_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -540,7 +540,7 @@ def wavepostbndpntbll(self): 'dependency': dependencies, 'envars': wave_post_bndpnt_bull_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpntbll.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -579,7 +579,7 @@ def wavepostpnt(self): 'dependency': dependencies, 'envars': wave_post_pnt_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -624,7 +624,7 @@ def extractvars(self): 'dependency': dependencies, 'envars': extractvars_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/extractvars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -674,7 +674,7 @@ def arch_vrfy(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_vrfy.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -718,7 +718,7 @@ def arch_tars(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_tars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_tars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -741,7 +741,7 @@ def globus(self): 'envars': self.envars, 'cycledef': 'gefs', 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/globus_arch.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/globus_arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -772,7 +772,7 @@ def cleanup(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/cleanup.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' diff --git a/workflow/rocoto/gefs_xml.py b/dev/workflow/rocoto/gefs_xml.py similarity index 100% rename from workflow/rocoto/gefs_xml.py rename to dev/workflow/rocoto/gefs_xml.py diff --git a/workflow/rocoto/gfs_cycled_xml.py b/dev/workflow/rocoto/gfs_cycled_xml.py similarity index 100% rename from workflow/rocoto/gfs_cycled_xml.py rename to dev/workflow/rocoto/gfs_cycled_xml.py diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/dev/workflow/rocoto/gfs_forecast_only_xml.py similarity index 100% rename from workflow/rocoto/gfs_forecast_only_xml.py rename to dev/workflow/rocoto/gfs_forecast_only_xml.py diff --git a/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py similarity index 94% rename from workflow/rocoto/gfs_tasks.py rename to dev/workflow/rocoto/gfs_tasks.py index 71bc2162c2d..0dbd3b329bf 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -26,7 +26,7 @@ def fetch(self): 'resources': resources, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fetch.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fetch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -55,7 +55,7 @@ def stage_ic(self): 'resources': resources, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;', @@ -98,7 +98,7 @@ def prep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/prep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -128,7 +128,7 @@ def waveinit(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -152,7 +152,7 @@ def waveprep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveprep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveprep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -204,7 +204,7 @@ def aerosol_init(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/aerosol_init.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/aerosol_init.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -232,7 +232,7 @@ def anal(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/anal.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/anal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -268,7 +268,7 @@ def sfcanl(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/sfcanl.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/sfcanl.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -297,7 +297,7 @@ def analcalc(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/analcalc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/analcalc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -321,7 +321,7 @@ def analdiag(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/analdiag.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/analdiag.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -345,7 +345,7 @@ def prepatmiodaobs(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/prepatmiodaobs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prepatmiodaobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -381,7 +381,7 @@ def atmanlinit(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -405,7 +405,7 @@ def atmanlvar(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlvar.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -429,7 +429,7 @@ def atmanlfv3inc(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlfv3inc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -453,7 +453,7 @@ def atmanlfinal(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlfinal.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -476,7 +476,7 @@ def prepobsaero(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/prepobsaero.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prepobsaero.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -500,7 +500,7 @@ def aeroanlgenb(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': 'gdas_half,gdas', - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/aeroanlgenb.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -530,7 +530,7 @@ def aeroanlinit(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/aeroanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -561,7 +561,7 @@ def aeroanlvar(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -585,7 +585,7 @@ def aeroanlfinal(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlfinal.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/aeroanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -609,7 +609,7 @@ def snowanl(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/snowanl.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/snowanl.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -634,7 +634,7 @@ def esnowanl(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/esnowanl.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/esnowanl.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -660,7 +660,7 @@ def prepoceanobs(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/prepoceanobs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prepoceanobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -686,7 +686,7 @@ def marineanlletkf(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlletkf.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -716,7 +716,7 @@ def marinebmat(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marinebmat.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marinebmat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -744,7 +744,7 @@ def marineanlinit(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -768,7 +768,7 @@ def marineanlvar(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlvar.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -792,7 +792,7 @@ def ocnanalecen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalecen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/ocnanalecen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -823,7 +823,7 @@ def marineanlchkpt(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlchkpt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlchkpt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -847,7 +847,7 @@ def marineanlfinal(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlfinal.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -916,7 +916,7 @@ def _fcst_forecast_only(self): 'dependency': dependencies, 'envars': fcst_vars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -982,7 +982,7 @@ def _fcst_cycled(self): 'dependency': dependencies, 'envars': fcst_vars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1025,7 +1025,7 @@ def atmanlupp(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/upp.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/upp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1054,7 +1054,7 @@ def atmanlprod(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_products.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmos_products.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1103,7 +1103,7 @@ def _upptask(self, upp_run="forecast", task_id="atmupp"): 'dependency': dependencies, 'envars': postenvars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/upp.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/upp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1184,7 +1184,7 @@ def _atmosoceaniceprod(self, component: str): 'dependency': dependencies, 'envars': postenvars, 'cycledef': cycledef, - 'command': f"{self.HOMEgfs}/jobs/rocoto/{config}.sh", + 'command': f"{self.HOMEgfs}/dev/jobs/{config}.sh", 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1231,7 +1231,7 @@ def wavepostsbs(self): 'dependency': dependencies, 'envars': wave_post_envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostsbs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1258,7 +1258,7 @@ def wavepostbndpnt(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1285,7 +1285,7 @@ def wavepostbndpntbll(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpntbll.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1311,7 +1311,7 @@ def wavepostpnt(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1348,7 +1348,7 @@ def wavegempak(self): 'dependency': dependencies, 'envars': wave_post_envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavegempak.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavegempak.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1377,7 +1377,7 @@ def waveawipsbulls(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveawipsbulls.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveawipsbulls.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1400,7 +1400,7 @@ def waveawipsgridded(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveawipsgridded.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveawipsgridded.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1423,7 +1423,7 @@ def postsnd(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/postsnd.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/postsnd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1460,7 +1460,7 @@ def fbwind(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/fbwind.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fbwind.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1532,7 +1532,7 @@ def awips_20km_1p0deg(self): 'dependency': dependencies, 'envars': awipsenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/awips_20km_1p0deg.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/awips_20km_1p0deg.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1575,7 +1575,7 @@ def gempak(self): 'dependency': dependencies, 'envars': gempak_vars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempak.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempak.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1602,7 +1602,7 @@ def gempakmeta(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakmeta.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempakmeta.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1625,7 +1625,7 @@ def gempakmetancdc(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakmetancdc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempakmetancdc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1648,7 +1648,7 @@ def gempakncdcupapgif(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakncdcupapgif.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempakncdcupapgif.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1676,7 +1676,7 @@ def gempakpgrb2spec(self): 'dependency': dependencies, 'envars': gempak_vars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/gempakgrb2spec.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/gempakgrb2spec.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1718,7 +1718,7 @@ def npoess_pgrb2_0p5deg(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/npoess.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/npoess.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1741,7 +1741,7 @@ def verfozn(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/verfozn.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/verfozn.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1764,7 +1764,7 @@ def verfrad(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/verfrad.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/verfrad.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1787,7 +1787,7 @@ def vminmon(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/vminmon.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/vminmon.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1821,7 +1821,7 @@ def anlstat(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/anlstat.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/anlstat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1844,7 +1844,7 @@ def tracker(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/tracker.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/tracker.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1867,7 +1867,7 @@ def genesis(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/genesis.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/genesis.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1890,7 +1890,7 @@ def genesis_fsu(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/genesis_fsu.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/genesis_fsu.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1913,7 +1913,7 @@ def fit2obs(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/fit2obs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fit2obs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1964,7 +1964,7 @@ def metp(self): 'dependency': dependencies, 'envars': metpenvars, 'cycledef': 'metp,last_gfs', - 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -1993,7 +1993,7 @@ def mos_stn_prep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_prep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_stn_prep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2016,7 +2016,7 @@ def mos_grd_prep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_prep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_grd_prep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2039,7 +2039,7 @@ def mos_ext_stn_prep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_prep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_stn_prep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2062,7 +2062,7 @@ def mos_ext_grd_prep(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_prep.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_grd_prep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2085,7 +2085,7 @@ def mos_stn_fcst(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_stn_fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2111,7 +2111,7 @@ def mos_grd_fcst(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_grd_fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2137,7 +2137,7 @@ def mos_ext_stn_fcst(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_stn_fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2166,7 +2166,7 @@ def mos_ext_grd_fcst(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_grd_fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2189,7 +2189,7 @@ def mos_stn_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_stn_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_stn_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2215,7 +2215,7 @@ def mos_grd_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_grd_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_grd_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2241,7 +2241,7 @@ def mos_ext_stn_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_stn_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_stn_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2270,7 +2270,7 @@ def mos_ext_grd_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_ext_grd_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_ext_grd_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2293,7 +2293,7 @@ def mos_wx_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_wx_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_wx_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2319,7 +2319,7 @@ def mos_wx_ext_prdgen(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/mos_wx_ext_prdgen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/mos_wx_ext_prdgen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2379,7 +2379,7 @@ def arch_vrfy(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_vrfy.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2494,7 +2494,7 @@ def arch_tars(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_tars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_tars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2518,7 +2518,7 @@ def globus_arch(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/globus_arch.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/globus_arch.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2551,7 +2551,7 @@ def globus_earc(self): 'dependency': dependencies, 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/globus_earc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/globus_earc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2700,7 +2700,7 @@ def cleanup(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/cleanup.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2726,7 +2726,7 @@ def eobs(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/eobs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/eobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2749,7 +2749,7 @@ def ediag(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ediag.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/ediag.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2772,7 +2772,7 @@ def eupd(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/eupd.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/eupd.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2798,7 +2798,7 @@ def atmensanlinit(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2824,7 +2824,7 @@ def atmensanlobs(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlobs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2850,7 +2850,7 @@ def atmensanlsol(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlsol.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlsol.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2876,7 +2876,7 @@ def atmensanlletkf(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlletkf.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2905,7 +2905,7 @@ def atmensanlfv3inc(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlfv3inc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2929,7 +2929,7 @@ def atmensanlfinal(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmensanlfinal.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -2989,7 +2989,7 @@ def _get_ecengroups(): 'dependency': dependencies, 'envars': ecenenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ecen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/ecen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3019,7 +3019,7 @@ def ecen_fv3jedi(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ecen_fv3jedi.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/ecen_fv3jedi.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3049,7 +3049,7 @@ def analcalc_fv3jedi(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/analcalc_fv3jedi.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/analcalc_fv3jedi.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3082,7 +3082,7 @@ def esfc(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/esfc.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/esfc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3127,7 +3127,7 @@ def efcs(self): 'dependency': dependencies, 'envars': efcsenvars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3163,7 +3163,7 @@ def echgres(self): 'dependency': dependencies, 'envars': self.envars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/echgres.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/echgres.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3222,7 +3222,7 @@ def _get_eposgroups(epos): 'dependency': dependencies, 'envars': eposenvars, 'cycledef': cycledef, - 'command': f'{self.HOMEgfs}/jobs/rocoto/epos.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/epos.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3262,7 +3262,7 @@ def earc_vrfy(self): 'dependency': dependencies, 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_vrfy.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/earc_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -3304,7 +3304,7 @@ def earc_tars(self): 'dependency': dependencies, 'envars': earcenvars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/earc_tars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/earc_tars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' diff --git a/workflow/rocoto/rocoto.py b/dev/workflow/rocoto/rocoto.py similarity index 100% rename from workflow/rocoto/rocoto.py rename to dev/workflow/rocoto/rocoto.py diff --git a/workflow/rocoto/rocoto_xml_factory.py b/dev/workflow/rocoto/rocoto_xml_factory.py similarity index 100% rename from workflow/rocoto/rocoto_xml_factory.py rename to dev/workflow/rocoto/rocoto_xml_factory.py diff --git a/workflow/rocoto/sfs_tasks.py b/dev/workflow/rocoto/sfs_tasks.py similarity index 96% rename from workflow/rocoto/sfs_tasks.py rename to dev/workflow/rocoto/sfs_tasks.py index 713018eda32..a8bea05a3ec 100644 --- a/workflow/rocoto/sfs_tasks.py +++ b/dev/workflow/rocoto/sfs_tasks.py @@ -16,7 +16,7 @@ def stage_ic(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -33,7 +33,7 @@ def waveinit(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/waveinit.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/waveinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -50,7 +50,7 @@ def prep_emissions(self): 'resources': resources, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/prep_emissions.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/prep_emissions.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -88,7 +88,7 @@ def fcst(self): 'dependency': dependencies, 'envars': fcst_vars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -145,7 +145,7 @@ def efcs(self): 'dependency': dependencies, 'envars': efcsenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/fcst.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/fcst.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -246,7 +246,7 @@ def _atmosoceaniceprod(self, component: str): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/{config}.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/{config}.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} @@ -301,7 +301,7 @@ def atmos_ensstat(self): 'dependency': dependencies, 'envars': postenvars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmos_ensstat.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/atmos_ensstat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;'} @@ -357,7 +357,7 @@ def wavepostsbs(self): 'dependency': dependencies, 'envars': wave_post_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostsbs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -396,7 +396,7 @@ def wavepostbndpnt(self): 'dependency': dependencies, 'envars': wave_post_bndpnt_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -441,7 +441,7 @@ def wavepostbndpntbll(self): 'dependency': dependencies, 'envars': wave_post_bndpnt_bull_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostbndpntbll.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -480,7 +480,7 @@ def wavepostpnt(self): 'dependency': dependencies, 'envars': wave_post_pnt_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/wavepostpnt.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -525,7 +525,7 @@ def extractvars(self): 'dependency': dependencies, 'envars': extractvars_envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/jobs/rocoto/extractvars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/extractvars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -575,7 +575,7 @@ def arch_vrfy(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_vrfy.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_vrfy.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -619,7 +619,7 @@ def arch_tars(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/arch_tars.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/arch_tars.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -662,7 +662,7 @@ def cleanup(self): 'envars': self.envars, 'cycledef': self.run, 'dependency': dependencies, - 'command': f'{self.HOMEgfs}/jobs/rocoto/cleanup.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/cleanup.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' diff --git a/workflow/rocoto/sfs_xml.py b/dev/workflow/rocoto/sfs_xml.py similarity index 100% rename from workflow/rocoto/sfs_xml.py rename to dev/workflow/rocoto/sfs_xml.py diff --git a/workflow/rocoto/tasks.py b/dev/workflow/rocoto/tasks.py similarity index 100% rename from workflow/rocoto/tasks.py rename to dev/workflow/rocoto/tasks.py diff --git a/workflow/rocoto/tasks_factory.py b/dev/workflow/rocoto/tasks_factory.py similarity index 100% rename from workflow/rocoto/tasks_factory.py rename to dev/workflow/rocoto/tasks_factory.py diff --git a/workflow/rocoto/workflow_tasks.py b/dev/workflow/rocoto/workflow_tasks.py similarity index 100% rename from workflow/rocoto/workflow_tasks.py rename to dev/workflow/rocoto/workflow_tasks.py diff --git a/workflow/rocoto/workflow_xml.py b/dev/workflow/rocoto/workflow_xml.py similarity index 98% rename from workflow/rocoto/workflow_xml.py rename to dev/workflow/rocoto/workflow_xml.py index 18170862859..7dc1f11ac70 100644 --- a/workflow/rocoto/workflow_xml.py +++ b/dev/workflow/rocoto/workflow_xml.py @@ -82,10 +82,7 @@ def _get_definitions(self) -> str: entity = OrderedDict() entity['PSLOT'] = self.pslot - entity['ROTDIR'] = self._base['ROTDIR'] - entity['JOBS_DIR'] = self._base['BASE_JOB'] - entity['MAXTRIES'] = self.rocoto_config['maxtries'] # Put them all in an XML key-value syntax @@ -199,7 +196,7 @@ def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None: script_fh.write( "#!/usr/bin/env bash\n" + "set -x\n" + - f"source {self.HOMEgfs}/workflow/gw_setup.sh" + "\n" + + f"source {self.HOMEgfs}/dev/ush/gw_setup.sh" + "\n" + rocotorunstr + "\n" ) diff --git a/workflow/rocoto_viewer.py b/dev/workflow/rocoto_viewer.py similarity index 100% rename from workflow/rocoto_viewer.py rename to dev/workflow/rocoto_viewer.py diff --git a/workflow/setup_ecf.py b/dev/workflow/setup_ecf.py similarity index 100% rename from workflow/setup_ecf.py rename to dev/workflow/setup_ecf.py diff --git a/workflow/setup_expt.py b/dev/workflow/setup_expt.py similarity index 99% rename from workflow/setup_expt.py rename to dev/workflow/setup_expt.py index 7dbbea6e40a..36180e9056f 100755 --- a/workflow/setup_expt.py +++ b/dev/workflow/setup_expt.py @@ -15,7 +15,7 @@ _here = os.path.dirname(__file__) -_top = os.path.abspath(os.path.join(os.path.abspath(_here), '..')) +_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..')) # Setup the logger logger = getLogger(__name__) diff --git a/workflow/setup_xml.py b/dev/workflow/setup_xml.py similarity index 100% rename from workflow/setup_xml.py rename to dev/workflow/setup_xml.py diff --git a/workflow/tests/__init__.py b/dev/workflow/tests/__init__.py similarity index 100% rename from workflow/tests/__init__.py rename to dev/workflow/tests/__init__.py diff --git a/workflow/tests/test_configuration.py b/dev/workflow/tests/test_configuration.py similarity index 100% rename from workflow/tests/test_configuration.py rename to dev/workflow/tests/test_configuration.py diff --git a/workflow/tests/test_hosts.py b/dev/workflow/tests/test_hosts.py similarity index 100% rename from workflow/tests/test_hosts.py rename to dev/workflow/tests/test_hosts.py diff --git a/workflow/tests/test_tasks.py b/dev/workflow/tests/test_tasks.py similarity index 100% rename from workflow/tests/test_tasks.py rename to dev/workflow/tests/test_tasks.py diff --git a/ci/scripts/wxflow b/dev/workflow/wxflow similarity index 100% rename from ci/scripts/wxflow rename to dev/workflow/wxflow diff --git a/docs/source/noaa_csp.rst b/docs/source/noaa_csp.rst index 98754335e25..b6ca03bb5c1 100644 --- a/docs/source/noaa_csp.rst +++ b/docs/source/noaa_csp.rst @@ -106,7 +106,7 @@ Next, click "Fork latest" as shown in the red-circle in :numref:`Figure %s`). Best practices suggest one that is clear, concise, and relevant to the application. @@ -186,7 +186,7 @@ This will create a ``/lustre`` filesystem template after clicking on the red squ :align: center Define Lustre Attributes - + After creating the template, we need to fill in information for this ``/lustre`` filesystem. To do so, go to the NOAA PW website, and click *Lustre* on the left side panel, as indicated by red arrow 1 in :numref:`Figure %s `. Then select the filesystem defined by *Resource name* in :numref:`Figure %s above `, @@ -342,7 +342,7 @@ because Rocoto uses `crontab`, which needs the cluster active all the time, or t After finishing your work on the AWS cluster, you should terminate/stop the cluster, unless you have reasons to keep it active. To stop/terminate the cluster, go to the cluster session, and click the green power button as show in :numref:`Figure %s `. -A window will pop up; click the red *Turn Off* button to switch off the cluster. +A window will pop up; click the red *Turn Off* button to switch off the cluster. .. _stop-cluster: @@ -395,7 +395,7 @@ or accessing the cluster from your web terminal, you can start to clone, compile gwhome=/contrib/Wei.Huang/src/global-workflow-cloud # Change this to your own "global-workflow" source directory cd ${gwhome}/sorc - source ${gwhome}/workflow/gw_setup.sh + source ${gwhome}/dev/ush/gw_setup.sh #build_all.sh build_all.sh -w link_workflow.sh @@ -417,4 +417,3 @@ EPIC has copied the C48 and C96 ATM, GEFS, and some other data to AWS, and the c If users want to run their own case, they need to make changes to the IC path and others to make it work. The execution of the global-workflow should now follow the same steps as those for the RDHPCS on-premises hosts. - diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 4b0c8411062..56caf359644 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -8,7 +8,7 @@ Experiment Setup :: - source workflow/gw_setup.sh + source dev/ush/gw_setup.sh .. warning:: Sourcing gw_setup.sh will wipe your existing lmod environment @@ -22,8 +22,8 @@ Forecast-only experiment Scripts that will be used: - * ``workflow/setup_expt.py`` - * ``workflow/setup_xml.py`` + * ``dev/workflow/setup_expt.py`` + * ``dev/workflow/setup_xml.py`` *************************************** Step 1: Run experiment generator script @@ -33,7 +33,7 @@ The following command examples include variables for reference but users should :: - cd workflow + cd dev/workflow ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--account $ACCOUNT] @@ -69,21 +69,21 @@ Atm-only: :: - cd workflow + cd dev/workflow ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --interval 6 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled: :: - cd workflow + cd dev/workflow ./setup_expt.py gfs forecast-only --app S2SW --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdetatmos 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled with aerosols: :: - cd workflow + cd dev/workflow ./setup_expt.py gfs forecast-only --app S2SWA --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdetatmos 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir **************************************** @@ -135,8 +135,8 @@ Cycled experiment Scripts that will be used: - * ``workflow/setup_expt.py`` - * ``workflow/setup_xml.py`` + * ``dev/workflow/setup_expt.py`` + * ``dev/workflow/setup_xml.py`` *************************************** Step 1) Run experiment generator script @@ -146,7 +146,7 @@ The following command examples include variables for reference but users should :: - cd workflow + cd dev/workflow ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--sdate_gfs $SDATE_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN] @@ -186,7 +186,7 @@ Example: :: - cd workflow + cd dev/workflow ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --interval 6 Example ``setup_expt.py`` on Orion: @@ -235,7 +235,7 @@ Go to your EXPDIR and check/change the following variables within your config.ba * HPSS_PROJECT (project on HPSS if archiving) * ATARDIR (location on HPSS or locally if archiving) -`NOTE`: If you selected ``ARHCOM_TO='globus_hpss``, then you will need to activate your globus connections between Niagara and MSU. See :doc: globus_arch.rst for more details. +`NOTE`: If you selected ``ARCHCOM_TO='globus_hpss``, then you will need to activate your globus connections between Niagara and MSU. See :doc: globus_arch.rst for more details. Now is also the time to change any other variables/settings you wish to change in config.base or other configs. `Do that now.` Once you are done making changes to the configs in your EXPDIR, go back to your clone to run the second setup script. See :doc: configure.rst for more information on configuring your run. diff --git a/parm/config/gefs/config.base.j2 b/parm/config/gefs/config.base.j2 index ab5f9df0c20..fd432b87fc1 100644 --- a/parm/config/gefs/config.base.j2 +++ b/parm/config/gefs/config.base.j2 @@ -94,7 +94,6 @@ export NCLEN="${HOMEgfs}/ush/getncdimlen" # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" -export BASE_JOB="${HOMEgfs}/jobs/rocoto" # EXPERIMENT specific environment parameters export SDATE="{{ SDATE }}" diff --git a/parm/config/gfs/config.base.j2 b/parm/config/gfs/config.base.j2 index 75dac7f6185..8fd6c61caef 100644 --- a/parm/config/gfs/config.base.j2 +++ b/parm/config/gfs/config.base.j2 @@ -120,7 +120,6 @@ export NCLEN="${HOMEgfs}/ush/getncdimlen" # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" -export BASE_JOB="${HOMEgfs}/jobs/rocoto" # EXPERIMENT specific environment parameters export SDATE="{{ SDATE }}" diff --git a/parm/config/sfs/config.base.j2 b/parm/config/sfs/config.base.j2 index c7151efc02e..f3e3311439a 100644 --- a/parm/config/sfs/config.base.j2 +++ b/parm/config/sfs/config.base.j2 @@ -94,7 +94,6 @@ export NCLEN="${HOMEgfs}/ush/getncdimlen" # Machine environment, jobs, and other utility scripts export BASE_ENV="${HOMEgfs}/env" -export BASE_JOB="${HOMEgfs}/jobs/rocoto" # EXPERIMENT specific environment parameters export SDATE="{{ SDATE }}" diff --git a/sorc/build_compute.sh b/sorc/build_compute.sh index 0fb24aa1c4f..9bd53c9c497 100755 --- a/sorc/build_compute.sh +++ b/sorc/build_compute.sh @@ -75,13 +75,13 @@ cd "${HOMEgfs}/sorc" || exit 1 rm -f "${build_xml}" "${build_db}" "${build_lock_db}" echo "Sourcing global-workflow modules ..." -source "${HOMEgfs}/workflow/gw_setup.sh" +source "${HOMEgfs}/dev/ush/gw_setup.sh" yaml="${HOMEgfs}/workflow/build_opts.yaml" echo "Generating build.xml for building global-workflow programs on compute nodes ..." # Catch errors manually from here out set +e -"${HOMEgfs}/workflow/build_compute.py" --account "${HPC_ACCOUNT}" --yaml "${yaml}" --systems "${systems}" +"${HOMEgfs}/dev/workflow/build_compute.py" --account "${HPC_ACCOUNT}" --yaml "${yaml}" --systems "${systems}" rc=$? if [[ "${rc}" -ne 0 ]]; then msg="FATAL ERROR: ${BASH_SOURCE[0]} failed to create 'build.xml' with error code ${rc}" @@ -99,7 +99,7 @@ echo "Running builds on compute nodes" while [[ "${finished}" == "false" ]]; do sleep 1m ${runcmd} - state="$("${HOMEgfs}/ci/scripts/utils/rocotostat.py" -w "${build_xml}" -d "${build_db}")" + state="$("${HOMEgfs}/dev/ci/scripts/utils/rocotostat.py" -w "${build_xml}" -d "${build_db}")" if [[ "${verbose_opt}" == "true" ]]; then echo "Rocoto is in state ${state}" else diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index 43c503ab4d1..21e70aeb256 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -24,6 +24,7 @@ shift $((OPTIND-1)) # shellcheck disable=SC2086 BUILD_JOBS="${BUILD_JOBS:-8}" \ WORKFLOW_BUILD="ON" \ +WORKFLOW_TESTS="OFF" \ ./gdas.cd/build.sh ${_opts} -f exit diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c3d0cab926c..492aa9967df 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c3d0cab926c97ab1f63b850f991ff53209b378b3 +Subproject commit 492aa9967dfe0a3d8bc9e729ed4aa58eb7fc599c diff --git a/workflow/wxflow b/workflow/wxflow deleted file mode 120000 index 7ea96a12bfc..00000000000 --- a/workflow/wxflow +++ /dev/null @@ -1 +0,0 @@ -../sorc/wxflow/src/wxflow \ No newline at end of file