Prepare for ops readiness - part 1

.github/CODEOWNERS

@@ -7,13 +7,25 @@
 @DavidHuber-NOAA
 
 # Specific directory owners
-/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA
+# dev/
+dev/jobs/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul
+dev/ush/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul
+dev/workflow/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul
+dev/test/ @WalterKolczynski-NOAA @DavidHuber-NOAA @KateFriedman-NOAA @aerorahul
+dev/ci/ @TerrenceMcGuinness-NOAA @WalterKolczynski-NOAA @DavidHuber-NOAA
+dev/ctests/ @TerrenceMcGuinness-NOAA @aerorahul
 
 /ecf/ @lgannoaa
 
-/workflow/ @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA
-
 # Specific file owners
+# dev side scripts
+dev/ush/compare_f90nml.py @WalterKolczynski-NOAA @aerorahul
+
+# Specific workflow scripts
+dev/workflow/generate_workflows.sh @DavidHuber-NOAA
+dev/workflow/build_compute.py @DavidHuber-NOAA @aerorahul
+dev/workflow/build_*opts.yaml @DavidHuber-NOAA @aerorahul
+
 # build scripts
 sorc/build_*.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA
 sorc/link_workflow.sh @WalterKolczynski-NOAA @DavidHuber-NOAA @aerorahul @KateFriedman-NOAA
@@ -67,7 +79,6 @@ jobs/JGLOBAL_PREP_OCEAN_OBS @guillaumevernieres @AndrewEichmann-NOAA
 jobs/JGLOBAL_*SNOW* @jiaruidong2017
 jobs/JGLOBAL_STAGE_IC @KateFriedman-NOAA
 jobs/JGLOBAL_WAVE_* @JessicaMeixner-NOAA @sbanihash
-jobs/rocoto/* @WalterKolczynski-NOAA @KateFriedman-NOAA @DavidHuber-NOAA
 
 # scripts
 scripts/exgdas_aero_analysis_generate_bmatrix.py @CoryMartin-NOAA
@@ -125,63 +136,62 @@ scripts/exglobal_snow_analysis.py @jiaruidong2017
 scripts/exglobal_stage_ic.py @KateFriedman-NOAA
 
 # ush
-WAM_XML_to_ASCII.pl
-atmos_ensstat.sh
-atmos_extractvars.sh @EricSinsky-NOAA
-bash_utils.sh @WalterKolczynski-NOAA
-calcanl_gfs.py @CoryMartin-NOAA
-calcinc_gfs.py @CoryMartin-NOAA
-compare_f90nml.py @WalterKolczynski-NOAA @aerorahul
-detect_machine.sh @WalterKolczynski-NOAA
-extractvars_tools.sh @EricSinsky-NOAA
-file_utils.sh @WalterKolczynski-NOAA
-forecast_det.sh @aerorahul @WalterKolczynski-NOAA
-forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA
-forecast_predet.sh @aerorahul @WalterKolczynski-NOAA
-fv3gfs_remap_weights.sh
-gaussian_sfcanl.sh @GeorgeGayno-NOAA
-getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
-getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
-getgfsnctime @CoryMartin-NOAA
-getncdimlen @CoryMartin-NOAA
-gfs_bfr2gpk.sh @BoCui-NOAA
-gfs_bufr.sh @BoCui-NOAA
-gfs_bufr_netcdf.sh @BoCui-NOAA
-gfs_sndp.sh @BoCui-NOAA
-gfs_truncate_enkf.sh @CoryMartin-NOAA
-global_savefits.sh
-gsi_utils.py @CoryMartin-NOAA
-interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
-interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
-jjob_header.sh @WalterKolczynski-NOAA
-link_crtm_fix.sh @WalterKolczynski-NOAA
-load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul
-load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA
-load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA
-merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA
-minmon_xtrct_*.pl @EdwardSafford-NOAA
-module-setup.sh @WalterKolczynski-NOAA @aerorahul
-ocnice_extractvars.sh @EricSinsky-NOAA
-ozn_xtrct.sh @EdwardSafford-NOAA
-parse-storm-type.pl
-parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
-parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen
-parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
-parsing_namelists_FV3_nest.sh @guoqing-noaa
-parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang
-parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash
-parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
-preamble.sh @WalterKolczynski-NOAA
-product_functions.sh @WalterKolczynski-NOAA @aerorahul
-radmon_*.sh @EdwardSafford-NOAA
-rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA
-run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA
-syndat_getjtbul.sh @JiayiPeng-NOAA
-syndat_qctropcy.sh @JiayiPeng-NOAA
-tropcy_relocate.sh @JiayiPeng-NOAA
-tropcy_relocate_extrkr.sh @JiayiPeng-NOAA
-wave_*.sh @JessicaMeixner-NOAA @sbanihash
-regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA
+ush/WAM_XML_to_ASCII.pl
+ush/atmos_ensstat.sh
+ush/atmos_extractvars.sh @EricSinsky-NOAA
+ush/bash_utils.sh @WalterKolczynski-NOAA
+ush/calcanl_gfs.py @CoryMartin-NOAA
+ush/calcinc_gfs.py @CoryMartin-NOAA
+ush/detect_machine.sh @WalterKolczynski-NOAA
+ush/extractvars_tools.sh @EricSinsky-NOAA
+ush/file_utils.sh @WalterKolczynski-NOAA
+ush/forecast_det.sh @aerorahul @WalterKolczynski-NOAA
+ush/forecast_postdet.sh @aerorahul @WalterKolczynski-NOAA
+ush/forecast_predet.sh @aerorahul @WalterKolczynski-NOAA
+ush/fv3gfs_remap_weights.sh
+ush/gaussian_sfcanl.sh @GeorgeGayno-NOAA
+ush/getdump.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
+ush/getges.sh @WalterKolczynski-NOAA @KateFriedman-NOAA
+ush/getgfsnctime @CoryMartin-NOAA
+ush/getncdimlen @CoryMartin-NOAA
+ush/gfs_bfr2gpk.sh @BoCui-NOAA
+ush/gfs_bufr.sh @BoCui-NOAA
+ush/gfs_bufr_netcdf.sh @BoCui-NOAA
+ush/gfs_sndp.sh @BoCui-NOAA
+ush/gfs_truncate_enkf.sh @CoryMartin-NOAA
+ush/global_savefits.sh
+ush/gsi_utils.py @CoryMartin-NOAA
+ush/interp_atmos_master.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
+ush/interp_atmos_sflux.sh @aerorahul @WenMeng-NOAA @WalterKolczynski-NOAA
+ush/jjob_header.sh @WalterKolczynski-NOAA
+ush/link_crtm_fix.sh @WalterKolczynski-NOAA
+ush/load_fv3gfs_modules.sh @WalterKolczynski-NOAA @aerorahul
+ush/load_ufsda_modules.sh @WalterKolczynski-NOAA @aerorahul @CoryMartin-NOAA
+ush/load_ufswm_modules.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA
+ush/merge_fv3_aerosol_tile.py @WalterKolczynski-NOAA
+ush/minmon_xtrct_*.pl @EdwardSafford-NOAA
+ush/module-setup.sh @WalterKolczynski-NOAA @aerorahul
+ush/ocnice_extractvars.sh @EricSinsky-NOAA
+ush/ozn_xtrct.sh @EdwardSafford-NOAA
+ush/parse-storm-type.pl
+ush/parsing_model_configure_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+ush/parsing_namelists_CICE.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @DeniseWorthen
+ush/parsing_namelists_FV3.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+ush/parsing_namelists_FV3_nest.sh @guoqing-noaa
+ush/parsing_namelists_MOM6.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa @jiandewang
+ush/parsing_namelists_WW3.sh @WalterKolczynski-NOAA @aerorahul @JessicaMeixner-NOAA @sbanihash
+ush/parsing_ufs_configure.sh @WalterKolczynski-NOAA @aerorahul @junwang-noaa
+ush/preamble.sh @WalterKolczynski-NOAA
+ush/product_functions.sh @WalterKolczynski-NOAA @aerorahul
+ush/radmon_*.sh @EdwardSafford-NOAA
+ush/rstprod.sh @WalterKolczynski-NOAA @DavidHuber-NOAA
+ush/run_mpmd.sh @WalterKolczynski-NOAA @aerorahul @DavidHuber-NOAA
+ush/syndat_getjtbul.sh @JiayiPeng-NOAA
+ush/syndat_qctropcy.sh @JiayiPeng-NOAA
+ush/tropcy_relocate.sh @JiayiPeng-NOAA
+ush/tropcy_relocate_extrkr.sh @JiayiPeng-NOAA
+ush/wave_*.sh @JessicaMeixner-NOAA @sbanihash
+ush/regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA
 
 # ush/python
 ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA
@@ -212,7 +222,3 @@ ush/python/pygfs/ufswm/ufs.py @aerorahul
 ush/python/pygfs/utils/__init__.py @aerorahul
 ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA
 
-# Specific workflow scripts
-workflow/generate_workflows.sh @DavidHuber-NOAA
-workflow/build_compute.py @DavidHuber-NOAA @aerorahul
-workflow/build_*opts.yaml @DavidHuber-NOAA @aerorahul

.github/workflows/ci_unit_tests.yaml

@@ -1,68 +1,73 @@
 name: CI Unit Tests
 on: [pull_request, push, workflow_dispatch]
 
+env:
+  ROCOTO_VERSION: "1.3.7"
+
 jobs:
 
   ci_pytest:
-      runs-on: ubuntu-22.04
-      name: Run unit tests on CI system
-      permissions:
-        checks: write
-
-      steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+    runs-on: ubuntu-22.04
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11.8
+    steps:
+    - name: Checkout global-workflow
+      uses: actions/checkout@v4
+      with:
+        path: global-workflow
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl
-          python -m pip install --upgrade pip
-          pip install pytest
-          pip install wxflow
-          pip install wget
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11.8
 
-      - name: Cache Rocoto Install
-        uses: actions/cache@v4
-        with:
-          path: ~/rocoto
-          key: ${{ runner.os }}-rocoto-${{ hashFiles('**/ci-unit_tests.yaml') }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y perl libxml-libxml-perl libxml-libxslt-perl libdatetime-perl
+        python -m pip install --upgrade pip
+        python -m pip install -r global-workflow/dev/workflow/requirements.txt
+        pip install pytest
+        pip install wget
 
-      - name: Install Rocoto
-        run: |
-          if [ ! -d "$HOME/rocoto/bin" ]; then
-            git clone https://github.com/christopherwharrop/rocoto.git $HOME/rocoto
-            cd $HOME/rocoto
-            ./INSTALL
-          fi
-          echo "$HOME/rocoto/bin" >> $GITHUB_PATH
+    - name: Cache Rocoto
+      id: cache-rocoto
+      uses: actions/cache@v4
+      with:
+        path: |
+          rocoto-${{ env.ROCOTO_VERSION }}
+        key: ${{ runner.os }}-rocoto-${{ env.ROCOTO_VERSION }}
 
-      - name: Run tests
-        shell: bash
-        run: |
-          sudo mkdir -p /scratch1/NCEPDEV
-          sudo mkdir -p /scratch1/NCEPDEV/stmp2
-          sudo mkdir -p /scratch1/NCEPDEV/stmp4
-          sudo mkdir -p /scratch1/NCEPDEV/global
-          sudo chmod -R a+w /scratch1/NCEPDEV
-          cd $GITHUB_WORKSPACE/sorc
-          git submodule update --init -j 9
-          ./link_workflow.sh
-          cd $GITHUB_WORKSPACE/ci/scripts/tests
-          ln -s ../wxflow
+    - name: Install Rocoto
+      id: install-rocoto
+      if: steps.cache-rocoto.outputs.cache-hit != 'true'
+      run: |
+        git clone https://github.com/christopherwharrop/rocoto.git rocoto-${{ env.ROCOTO_VERSION }}
+        cd rocoto-${{ env.ROCOTO_VERSION }}
+        git checkout ${{ env.ROCOTO_VERSION }}
+        ./INSTALL
+        echo "export PATH=${PWD}/bin:\$PATH" >> rocoto_path.sh
+        chmod +x rocoto_path.sh
 
-          pytest -v --junitxml $GITHUB_WORKSPACE/ci/scripts/tests/test-results.xml
+    - name: Run tests
+      shell: bash
+      run: |
+        sudo mkdir -p /scratch1/NCEPDEV
+        sudo mkdir -p /scratch1/NCEPDEV/stmp2
+        sudo mkdir -p /scratch1/NCEPDEV/stmp4
+        sudo mkdir -p /scratch1/NCEPDEV/global
+        sudo chmod -R a+w /scratch1/NCEPDEV
+        source rocoto-${{ env.ROCOTO_VERSION }}/rocoto_path.sh
+        cd global-workflow/sorc
+        git submodule update --init -j 2 wxflow ufs_model.fd
+        ./link_workflow.sh
+        cd ../dev/ci/scripts/tests
+        pytest -v --junitxml test-results.xml
 
 
-      - name: Publish Test Results
-        if: always()
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        with:
-          files: ci/scripts/tests/test-results.xml
-          job_summary: true
-          comment_mode: off
+    - name: Publish Test Results
+      if: always()
+      uses: EnricoMi/publish-unit-test-result-action@v2
+      with:
+        files: global-workflow/dev/ci/scripts/tests/test-results.xml
+        job_summary: true
+        comment_mode: off

.github/workflows/pw_aws_ci.yaml

@@ -88,7 +88,7 @@ jobs:
 
   checkout:
     needs: fetch-branch
-    runs-on: 
+    runs-on:
       - self-hosted
       - aws
       - parallelworks
@@ -103,9 +103,9 @@ jobs:
         repository: ${{ needs.fetch-branch.outputs.repo }}
         ref: ${{ needs.fetch-branch.outputs.branch }}
 
-  build-link:      
+  build-link:
     needs: checkout
-    runs-on: 
+    runs-on:
       - self-hosted
       - aws
       - parallelworks
@@ -123,7 +123,7 @@ jobs:
 
   create-experiments:
     needs: build-link
-    runs-on: 
+    runs-on:
       - self-hosted
       - aws
       - parallelworks
@@ -139,13 +139,13 @@ jobs:
         run: |
           mkdir -p ${{ env.RUNTESTS }}
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          source workflow/gw_setup.sh
-          source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
+          source dev/ush/gw_setup.sh
+          source dev/ci/platforms/config.noaacloud
+          ./dev/workflow/create_experiment.py --yaml dev/ci/cases/pr/${{ matrix.case }}.yaml --overwrite
 
   run-experiments:
     needs: create-experiments
-    runs-on: 
+    runs-on:
       - self-hosted
       - aws
       - parallelworks
@@ -157,11 +157,11 @@ jobs:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
+          ./dev/ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
 
   clean-up:
     needs: run-experiments
-    runs-on: 
+    runs-on:
       - self-hosted
       - aws
       - parallelworks