run GW using container with spack-stack 1.9.2

.gitignore

@@ -175,6 +175,7 @@ ush/bufr_snocvr_snomad.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
 ush/bufr2ioda_insitu*
+ush/container
 ush/python/ufsda
 ush/python/soca
 ush/python/gen_bufr2ioda_json.py

dev/container/gen-run-cases.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -x
+
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+source "${HOMEgfs}/ush/detect_machine.sh"
+
+run_with_container="YES"
+#run_with_container="NO"
+
+ casetype="pr"
+#yamllist="C48_ATM"
+ yamllist="C48_S2SW"
+#yamllist="C48_S2SWA_gefs"
+#yamllist="C96mx100_S2S"
+
+#casetype=hires
+#yamllist="C768_S2SW"
+
+HOMEDIR=${HOMEgfs}
+img=ubuntu22.04-intel-ufs-env-v1.9.2.img
+if [[ ${MACHINE_ID} = ursa* ]] ; then
+   container=/scratch3/NCEPDEV/nems/role.epic/containers/${img}
+   rundir=/scratch3/NAGAPE/epic/${USER}/run
+   bindings="-B /scratch3 -B /scratch4"
+   HPC_ACCOUNT=epic
+
+   module load rocoto/1.3.7
+   rocotocmd=$(command -v rocotorun)
+elif [[ ${MACHINE_ID} = gaea* ]] ; then
+   container=/gpfs/f6/scratch/Wei.Huang/container/${img}
+   rundir=/gpfs/f6/scratch/${USER}/run
+   bindings="-B /gpfs/f6/scratch -B /ncrc/home1/${USER}"
+   HPC_ACCOUNT=bil-fire8
+
+   rocotocmd=/autofs/ncrc-svm1_home2/Christopher.W.Harrop/rocoto-1.3.7/bin/rocotorun
+elif [[ ${MACHINE_ID} = noaacloud* ]] ; then
+   TOPICDIR=/bucket/global-workflow-shared-data/ICSDIR
+   container=/contrib/containers/${img}
+   rundir=/lustre/${USER}/run
+   bindings="--env \"I_MPI_FABRICS=shm:ofi,I_MPI_DEBUG=6\" -B /apps/slurm/default/lib/libpmi2.so -B /contrib -B /lustre -B /bucket"
+  #bindings="-B /apps/slurm/default/lib/libpmi2.so -B /contrib -B /lustre -B /bucket"
+   HPC_ACCOUNT=${USER}
+
+   module load rocoto/1.3.7
+   rocotocmd=$(command -v rocotorun)
+fi
+
+set -x
+
+mkdir -p "${rundir}"
+mkdir -p "${HOMEDIR}"/exec
+mkdir -p "${HOMEDIR}"/ush/container
+
+cd "${HOMEDIR}/dev/workflow" || exit 1
+
+if [[ "${run_with_container}" == "YES" ]]; then
+   "${HOMEDIR}/dev/container/utils/gen-wrapper.sh" -H "${HOMEDIR}" -c "${container}" -b "${bindings}" -v
+
+   TOPICDIR=${TOPICDIR} \
+   RUNTESTS=${rundir} \
+   RUNDIRS=${rundir} \
+	./generate_workflows.sh \
+	-H "${HOMEDIR}" \
+	-y "${yamllist}" \
+	-Y "${HOMEDIR}/dev/ci/cases/${casetype}" \
+	-A "${HPC_ACCOUNT}" \
+	-e "${USER}@noaa.gov" \
+	-r "${rocotocmd}" \
+    	-v -R
+
+   "${HOMEDIR}/dev/container/utils/create-atmos-products.sh" -H "${HOMEDIR}" -c "${container}" -b "${bindings}"
+   "${HOMEDIR}/dev/container/utils/create-container-links.sh" -H "${HOMEDIR}" -c "${container}" -b "${bindings}" -M "${MACHINE_ID}"
+else
+   TOPICDIR=${TOPICDIR} \
+   RUNTESTS=${rundir} \
+   RUNDIRS=${rundir} \
+        ./generate_workflows.sh \
+        -H "${HOMEDIR}" \
+        -y "${yamllist}" \
+        -Y "${HOMEDIR}/dev/ci/cases/${casetype}" \
+        -A "${HPC_ACCOUNT}" \
+        -e "${USER}@noaa.gov" \
+        -v
+fi
+

dev/container/shell-in-container.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+source "${HOMEgfs}/ush/detect_machine.sh"
+sif=ubuntu22.04-intel-ufs-env-v1.9.2.img
+
+if [[ ${MACHINE_ID} = ursa* ]] ; then
+   img=/scratch3/NCEPDEV/nems/role.epic/containers/${sif}
+   bindings="-B /scratch3 -B /scratch4"
+elif [[ ${MACHINE_ID} = gaea* ]] ; then
+   img=/gpfs/f6/scratch/${USER}/container/${sif}
+   bindings="-B /gpfs/f6/scratch -B /ncrc/home1/${USER}"
+elif [[ ${MACHINE_ID} = noaacloud* ]] ; then
+   img=/contrib/containers/${sif}
+   bindings="-B /contrib -B /lustre -B /bucket"
+fi
+
+singularity shell -e ${bindings} "${img}"
+

dev/container/utils/create-atmos-products.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+verbose=false
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    -H|--HOMEgfs)
+      HOMEgfs="$2"
+      shift 2
+      ;;
+    -c|--container)
+      container="$2"
+      shift 2
+      ;;
+    -b|--bindings)
+      bindings="$2"
+      shift 2
+      ;;
+    -v|--verbose)
+      verbose=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [[ ! -v HOMEgfs || ! -v container || ! -v bindings ]]; then
+   echo "Usage: create-container-links.sh -H/--HOMEgfs gw-home-dir -c/--container container-fullpath -b/--bindings list-of-binding-dirs [-v]"
+   exit 11
+fi
+
+if [[ "${verbose}" == "true" ]]; then
+   echo "Verbose: ${verbose}"
+   echo "HOMEgfs: ${HOMEgfs}"
+   echo "container: ${container}"
+   echo "bindings: ${bindings}"
+fi
+
+eap_script="${HOMEgfs}"/exec/exglobal_atmos_products.sh
+cat > "${eap_script}" << EOF_ATMOS_PRODUCTS
+#!/bin/bash
+ LD_LIBRARY_PATH=\$(dirname ${HOMEgfs})
+ export LD_LIBRARY_PATH
+
+ singularity exec \\
+        ${bindings} \\
+        ${container} \\
+        ${HOMEgfs}/scripts/exglobal_atmos_products.sh "\$@"
+EOF_ATMOS_PRODUCTS
+
+chmod +x "${eap_script}"
+

dev/container/utils/create-container-links.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+verbose=false
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    -H|--HOMEgfs)
+      HOMEgfs="$2"
+      shift 2
+      ;;
+    -c|--container)
+      container="$2"
+      shift 2
+      ;;
+    -b|--bindings)
+      bindings="$2"
+      shift 2
+      ;;
+    -v|--verbose)
+      verbose=true
+      shift
+      ;;
+    -M|--MACHINE_ID)
+      machineid="$2"
+      shift 2
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [[ ! -v HOMEgfs || ! -v container || ! -v bindings || ! -v MACHINE_ID ]]; then
+   echo "Usage: create-container-links.sh -H/--HOMEgfs gw-home-dir -c/--container container-fullpath -b/--bindings -M|--MACHINE_ID list-of-binding-dirs [-v]"
+   exit 11
+fi
+
+if [[ "${verbose}" == "true" ]]; then
+   echo "HOMEgfs: ${HOMEgfs}"
+   echo "container: ${container}"
+   echo "bindings: ${bindings}"
+   echo "Verbose: ${verbose}"
+fi
+
+"${HOMEgfs}/dev/container/utils/link_ww3.sh" -H "${HOMEgfs}" -c "${container}" -b "${bindings}" -t gfs
+"${HOMEgfs}/dev/container/utils/link_ww3.sh" -H "${HOMEgfs}" -c "${container}" -b "${bindings}" -t sfs
+"${HOMEgfs}/dev/container/utils/link_ww3.sh" -H "${HOMEgfs}" -c "${container}" -b "${bindings}" -t gefs
+
+"${HOMEgfs}/dev/container/utils/link_model.sh" -H "${HOMEgfs}" -c "${container}" -m gfs_model -b "${bindings}" -M "${machineid}"
+"${HOMEgfs}/dev/container/utils/link_model.sh" -H "${HOMEgfs}" -c "${container}" -m sfs_model -b "${bindings}" -M "${machineid}"
+"${HOMEgfs}/dev/container/utils/link_model.sh" -H "${HOMEgfs}" -c "${container}" -m gefs_model -b "${bindings}" -M "${machineid}"
+
+"${HOMEgfs}/dev/container/utils/link_gfs_utils.sh" -H "${HOMEgfs}" -c "${container}" -b "${bindings}"
+"${HOMEgfs}/dev/container/utils/link_ufs_utils.sh" -H "${HOMEgfs}" -c "${container}" -b "${bindings}"
+

dev/container/utils/gen-wrapper.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+verbose=false
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+    -H|--HOMEgfs)
+      HOMEgfs="$2"
+      shift 2
+      ;;
+    -b|--bindings)
+      bindings="$2"
+      shift 2
+      ;;
+    -c|--container)
+      container="$2"
+      shift 2
+      ;;
+    -v|--verbose)
+      verbose=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+if [[ ! -v HOMEgfs || ! -v container ]]; then
+   echo "Usage: link_model.sh -H/-HOMEgfs gw-home-dir -c/--container full-path-container-image -b/--bindings -B dirname [-B dirname1 [...]] [-v]"
+   exit 11
+fi
+
+if [[ "${verbose}" == "true" ]]; then
+   set -x
+fi
+
+exec_python_script="${HOMEgfs}"/exec/run_python.sh 
+
+cat > "${exec_python_script}" << EOF_EXEC_PYTHON
+#!/bin/bash
+ LD_LIBRARY_PATH=\$(dirname "${container}")
+ export LD_LIBRARY_PATH
+
+ singularity exec \\
+        ${bindings} \\
+        ${container} \\
+        ${HOMEgfs}/ush/container/run_python.sh "\$@"
+EOF_EXEC_PYTHON
+
+run_python_script="${HOMEgfs}"/ush/container/run_python.sh
+
+cat > "${run_python_script}" << EOF_RUN_PYTHON
+#!/bin/bash
+
+source /usr/lmod/lmod/init/bash
+module purge
+module use "${HOMEgfs}"/sorc/gfs_utils.fd/modulefiles
+module load gfsutils_container.intel
+module load python
+module load py-netcdf4
+module load py-xarray
+module load py-f90nml
+module load py-numpy
+module load py-jinja2
+module load py-pyyaml
+
+wxflowPATH=${HOMEgfs}/ush/python:${HOMEgfs}/sorc/wxflow/src
+export PYTHONPATH=\${PYTHONPATH:+\${PYTHONPATH}:}${HOMEgfs}/ush:\${wxflowPATH}
+
+python "\$@"
+EOF_RUN_PYTHON
+
+sed -i 's/RUN_WITH_CONTAINER=NO/RUN_WITH_CONTAINER=YES/g' "${HOMEgfs}/ush/preamble.sh"
+chmod +x "${exec_python_script}"
+chmod +x "${run_python_script}"
+
+for item in JGLOBAL_WAVE_INIT
+do
+  exec_script="${HOMEgfs}"/exec/"${item}"
+
+cat > "${exec_script}" << EOF_SCRIPT
+#!/bin/bash
+#Need these lines on AWS to run more than one node.
+#export I_MPI_DEBUG=10
+#export I_MPI_FABRICS=shm:ofi
+#export I_MPI_OFI_PROVIDER=tcp
+#export FI_PROVIDER=tcp
+#export FI_TCP_IFACE=eth0
+
+ LD_LIBRARY_PATH=\$(dirname "${container}")
+ export LD_LIBRARY_PATH
+
+ singularity exec \\
+        ${bindings} \\
+        ${container} \\
+        ${HOMEgfs}/jobs/${item}
+EOF_SCRIPT
+
+  chmod +x "${exec_script}"
+done
+