From a2ed32d90616c2d955d6a1274d9e97cf515b0adc Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 12 Aug 2022 15:23:33 +0000 Subject: [PATCH] Remove scritps that have been moved to global workflow Global workflow is moving all scripts that are only used by global workflow currently held by other repos into global workflow. Related workflow issue NOAA-EMC/global-workflow/pull/967 Related workflow PR NOAA-EMC/global-workflow/pull/969 --- .../nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON | 96 --- .../gdas/scripts/exgdas_atmos_vminmon.sh | 131 ---- .../nwprod/gfs/jobs/JGFS_ATMOS_VMINMON | 94 --- .../nwprod/gfs/scripts/exgfs_atmos_vminmon.sh | 127 ---- .../minmon_shared/ush/minmon_xtrct_costs.pl | 231 ------- .../minmon_shared/ush/minmon_xtrct_gnorms.pl | 444 -------------- .../minmon_shared/ush/minmon_xtrct_reduct.pl | 89 --- .../gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN | 113 ---- .../scripts/exgdas_atmos_verfozn.sh | 100 --- .../nwprod/oznmon_shared/ush/ozn_xtrct.sh | 271 --------- .../gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD | 136 ----- .../scripts/exgdas_atmos_verfrad.sh | 228 ------- .../radmon_shared/ush/radmon_err_rpt.sh | 207 ------- .../radmon_shared/ush/radmon_verf_angle.sh | 257 -------- .../radmon_shared/ush/radmon_verf_bcoef.sh | 241 -------- .../radmon_shared/ush/radmon_verf_bcor.sh | 234 ------- .../radmon_shared/ush/radmon_verf_time.sh | 575 ------------------ .../nwprod/radmon_shared/ush/rstprod.sh | 18 - 18 files changed, 3592 deletions(-) delete mode 100755 src/Minimization_Monitor/nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON delete mode 100755 src/Minimization_Monitor/nwprod/gdas/scripts/exgdas_atmos_vminmon.sh delete mode 100755 src/Minimization_Monitor/nwprod/gfs/jobs/JGFS_ATMOS_VMINMON delete mode 100755 src/Minimization_Monitor/nwprod/gfs/scripts/exgfs_atmos_vminmon.sh delete mode 100755 src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_costs.pl delete mode 100755 src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_gnorms.pl delete mode 100755 src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_reduct.pl delete mode 100755 src/Ozone_Monitor/nwprod/gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN delete mode 100755 src/Ozone_Monitor/nwprod/gdas_oznmon/scripts/exgdas_atmos_verfozn.sh delete mode 100755 src/Ozone_Monitor/nwprod/oznmon_shared/ush/ozn_xtrct.sh delete mode 100755 src/Radiance_Monitor/nwprod/gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD delete mode 100755 src/Radiance_Monitor/nwprod/gdas_radmon/scripts/exgdas_atmos_verfrad.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_err_rpt.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh delete mode 100755 src/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh diff --git a/src/Minimization_Monitor/nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON b/src/Minimization_Monitor/nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON deleted file mode 100755 index 7e89f33d..00000000 --- a/src/Minimization_Monitor/nwprod/gdas/jobs/JGDAS_ATMOS_VMINMON +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/sh -########################################################### -# GDAS Minimization Monitor (MinMon) job -########################################################### -set -xa -echo `date` $0 `date -u` begin -export PS4='$SECONDS + ' - -############################### -# Specify NET, RUN, and COMPONENT name -############################## -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export COMPONENT=${COMPONENT:-atmos} - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export pid=$$ -export outid=${outid:-"LL$job"} - -export DATA=${DATA:-${DATAROOT}/${jobid:?}} -mkdir -p $DATA -cd $DATA - - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export MINMON_SUFFIX=${MINMON_SUFFIX:-${NET}} -export m_job=${m_job:-${MINMON_SUFFIX}_mmDE} - - -############################################## -# Specify Package Areas -############################################## -export HOMEgfs=${HOMEgfs:-${NWROOT}/gfs.${gfs_ver}} -export SCRgfs=${SCRgfs:-$HOMEgfs/scripts} - -export M_FIXgdas=${M_FIXgdas:-$HOMEgfs/fix/gdas} - -export HOMEminmon=${HOMEminmon:-${HOMEgfs}} -export EXECminmon=${EXECminmon:-$HOMEminmon/exec} -export USHminmon=${USHminmon:-$HOMEminmon/ush} - - -############################################# -# determine PDY and cyc for previous cycle -############################################# - -cdate=`${NDATE} -6 ${PDY}${cyc}` -echo 'pdate = ${pdate}' - -export P_PDY=`echo ${cdate} | cut -c1-8` -export p_cyc=`echo ${cdate} | cut -c9-10` - - -############################################# -# TANKverf - WHERE OUTPUT DATA WILL RESIDE -############################################# -export COM_IN=${COM_IN:-$(compath.py ${envir}/${NET}/${gfs_ver})} - -export M_TANKverf=${M_TANKverf:-${COM_IN}/${RUN}.${PDY}/${cyc}/${COMPONENT}/minmon} -export M_TANKverfM1=${M_TANKverfM1:-${COM_IN}/${RUN}.${P_PDY}/${p_cyc}/${COMPONENT}/minmon} - -export COMIN=${COMIN:-$COM_IN/${RUN}.${PDY}/${cyc}/$COMPONENT} - -mkdir -p -m 775 $M_TANKverf - - - -######################################## -# Set necessary environment variables -######################################## -export CYCLE_INTERVAL=6 -export gsistat=${gsistat:-${COMIN}/gdas.t${cyc}z.gsistat} - - -######################################################## -# Execute the script. -${GMONSH:-$SCRgfs/exgdas_atmos_vminmon.sh} ${PDY} ${cyc} -err=$? -[[ $err -ne 0 ]] && exit $err - - -################################ -# Remove the Working Directory -################################ -KEEPDATA=${KEEPDATA:-NO} -cd ${DATAROOT} - -if [ ${KEEPDATA} = NO ] ; then - rm -rf ${DATA} -fi - - diff --git a/src/Minimization_Monitor/nwprod/gdas/scripts/exgdas_atmos_vminmon.sh b/src/Minimization_Monitor/nwprod/gdas/scripts/exgdas_atmos_vminmon.sh deleted file mode 100755 index f5087e41..00000000 --- a/src/Minimization_Monitor/nwprod/gdas/scripts/exgdas_atmos_vminmon.sh +++ /dev/null @@ -1,131 +0,0 @@ -#/bin/sh -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgdas_vrfminmon.sh -# Script description: Runs data extract/validation for GSI normalization diag data -# -# Author: Ed Safford Org: NP23 Date: 2015-04-10 -# -# Abstract: This script runs the data extract/validation portion of the -# MinMon package. -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -################################################################################ - - -######################################## -# Set environment -######################################## -export VERBOSE=${VERBOSE:-"NO"} -if [[ "$VERBOSE" = "YES" ]] -then - set -x -fi - -export scr=exgdas_vrfyminmon.sh - - -export RUN_ENVIR=${RUN_ENVIR:-nco} -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export envir=${envir:-prod} - -######################################## -# Command line arguments -######################################## -export PDY=${1:-${PDY:?}} -export cyc=${2:-${cyc:?}} - -######################################## -# Directories -######################################## -export DATA=${DATA:-$(pwd)} - - -######################################## -# Filenames -######################################## -gsistat=${gsistat:-$COMIN/gdas.t${cyc}z.gsistat} -export mm_gnormfile=${gnormfile:-${M_FIXgdas}/gdas_minmon_gnorm.txt} -export mm_costfile=${costfile:-${M_FIXgdas}/gdas_minmon_cost.txt} - -######################################## -# Other variables -######################################## -export MINMON_SUFFIX=${MINMON_SUFFIX:-GDAS} -export PDATE=${PDY}${cyc} -export NCP=${NCP:-/bin/cp} -export pgm=exgdas_vrfminmon.sh - -if [[ ! -d ${DATA} ]]; then - mkdir $DATA -fi -cd $DATA - -###################################################################### - -data_available=0 - -if [[ -s ${gsistat} ]]; then - - data_available=1 - - #----------------------------------------------------------------------- - # Copy the $MINMON_SUFFIX.gnorm_data.txt file to the working directory - # It's ok if it doesn't exist; we'll create a new one if needed. - # - # Note: The logic below is to accomodate two different data storage - # methods. Some parallels (and formerly ops) dump all MinMon data for - # a given day in the same directory (if condition). Ops now separates - # data into ${cyc} subdirectories (elif condition). - #----------------------------------------------------------------------- - if [[ -s ${M_TANKverf}/gnorm_data.txt ]]; then - $NCP ${M_TANKverf}/gnorm_data.txt gnorm_data.txt - elif [[ -s ${M_TANKverfM1}/gnorm_data.txt ]]; then - $NCP ${M_TANKverfM1}/gnorm_data.txt gnorm_data.txt - fi - - - #------------------------------------------------------------------ - # Run the child sccripts. - #------------------------------------------------------------------ - ${USHminmon}/minmon_xtrct_costs.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_costs=$? - echo "rc_costs = $rc_costs" - - ${USHminmon}/minmon_xtrct_gnorms.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_gnorms=$? - echo "rc_gnorms = $rc_gnorms" - - ${USHminmon}/minmon_xtrct_reduct.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_reduct=$? - echo "rc_reduct = $rc_reduct" - -fi - -##################################################################### -# Postprocessing - -err=0 -if [[ ${data_available} -ne 1 ]]; then - err=1 -elif [[ $rc_costs -ne 0 ]]; then - err=$rc_costs -elif [[ $rc_gnorms -ne 0 ]]; then - err=$rc_gnorms -elif [[ $rc_reduct -ne 0 ]]; then - err=$rc_reduct -fi - -if [[ "$VERBOSE" = "YES" ]]; then - echo "end exgdas_vrfminmon.sh, exit value = ${err}" -fi - - -set +x -exit ${err} - diff --git a/src/Minimization_Monitor/nwprod/gfs/jobs/JGFS_ATMOS_VMINMON b/src/Minimization_Monitor/nwprod/gfs/jobs/JGFS_ATMOS_VMINMON deleted file mode 100755 index 420da031..00000000 --- a/src/Minimization_Monitor/nwprod/gfs/jobs/JGFS_ATMOS_VMINMON +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh -########################################################### -# GFS Minimization Monitor (MinMon) job -########################################################### -set -xa -echo `date` $0 `date -u` begin -export PS4='$SECONDS + ' - -############################### -# Specify NET and RUN name -############################## -export NET=${NET:-gfs} -export RUN=${RUN:-gfs} -export COMPONENT=${COMPONENT:-atmos} - - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export pid=$$ -export outid=${outid:-"LL$job"} -export DATA=${DATA:-${DATAROOT}/${jobid:?}} -mkdir -p $DATA -cd $DATA - - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export MINMON_SUFFIX=${MINMON_SUFFIX:-GFS} -export m_job=${m_job:-${MINMON_SUFFIX}_mmDE} - - -############################################## -# Specify Package Areas -############################################## -export HOMEgfs=${HOMEgfs:-${NWROOT}/gfs.${gfs_ver}} -export SCRgfs=${SCRgfs:-$HOMEgfs/scripts} -export M_FIXgfs=${M_FIXgfs:-$HOMEgfs/fix/product} - -export HOMEminmon=${HOMEminmon:-${HOMEgfs}} -export EXECminmon=${EXECminmon:-$HOMEminmon/exec} -export USHminmon=${USHminmon:-$HOMEminmon/ush} - - -############################################# -# determine PDY and cyc for previous cycle -############################################# - -cdate=`${NDATE} -6 ${PDY}${cyc}` -echo 'pdate = ${pdate}' - -export P_PDY=`echo ${cdate} | cut -c1-8` -export p_cyc=`echo ${cdate} | cut -c9-10` - - -############################################# -# TANKverf - WHERE OUTPUT DATA WILL RESIDE -############################################# -export COM_IN=${COM_IN:-$(compath.py ${envir}/${NET}/${gfs_ver})} - -M_TANKverf=${M_TANKverf:-${COM_IN}/${RUN}.${PDY}/${cyc}/${COMPONENT}/minmon} -export M_TANKverfM1=${M_TANKverfM1:-${COM_IN}/${RUN}.${P_PDY}/${p_cyc}/${COMPONENT}/minmon} - -export COMIN=${COMIN:-$COM_IN/${RUN}.${PDY}/${cyc}/$COMPONENT} - -mkdir -p -m 775 $M_TANKverf - - -######################################## -# Set necessary environment variables -######################################## -export CYCLE_INTERVAL=6 -export gsistat=${gsistat:-${COMIN}/gfs.t${cyc}z.gsistat} - - -######################################################## -# Execute the script. -${GMONSH:-$SCRgfs/exgfs_atmos_vminmon.sh} ${PDY} ${cyc} -err=$? -[[ $err -ne 0 ]] && exit $err - - -################################ -# Remove the Working Directory -################################ -KEEPDATA=${KEEPDATA:-NO} -cd ${DATAROOT} - -if [ ${KEEPDATA} = NO ] ; then - rm -rf ${DATA} -fi - - diff --git a/src/Minimization_Monitor/nwprod/gfs/scripts/exgfs_atmos_vminmon.sh b/src/Minimization_Monitor/nwprod/gfs/scripts/exgfs_atmos_vminmon.sh deleted file mode 100755 index eb0eac23..00000000 --- a/src/Minimization_Monitor/nwprod/gfs/scripts/exgfs_atmos_vminmon.sh +++ /dev/null @@ -1,127 +0,0 @@ -#/bin/sh -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgfs_vrfminmon.sh -# Script description: Runs data extract/validation for GSI normalization diag data -# -# Author: Ed Safford Org: NP23 Date: 2015-04-10 -# -# Abstract: This script runs the data extract/validation portion of the -# MinMon package. -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -################################################################################ - - -######################################## -# Set environment -######################################## -export VERBOSE=${VERBOSE:-"NO"} -if [[ "$VERBOSE" = "YES" ]] -then - set -x -fi - -export scr=exgfs_vrfyminmon.sh - -export RUN_ENVIR=${RUN_ENVIR:-nco} -export NET=${NET:-gfs} -export RUN=${RUN:-gfs} -export envir=${envir:-prod} - -######################################## -# Command line arguments -######################################## -export PDY=${1:-${PDY:?}} -export cyc=${2:-${cyc:?}} - -######################################## -# Directories -######################################## -export DATA=${DATA:-$(pwd)} - - -######################################## -# Filenames -######################################## -gsistat=${gsistat:-$COMIN/gfs.t${cyc}z.gsistat} -export mm_gnormfile=${gnormfile:-${M_FIXgfs}/gfs_minmon_gnorm.txt} -export mm_costfile=${costfile:-${M_FIXgfs}/gfs_minmon_cost.txt} - -######################################## -# Other variables -######################################## -export MINMON_SUFFIX=${MINMON_SUFFIX:-GFS} -export PDATE=${PDY}${cyc} -export NCP=${NCP:-/bin/cp} -export pgm=exgfs_vrfminmon.sh - - - -if [[ ! -d ${DATA} ]]; then - mkdir $DATA -fi -cd $DATA - -###################################################################### - -data_available=0 - -if [[ -s ${gsistat} ]]; then - - data_available=1 - - #------------------------------------------------------------------ - # Copy the $MINMON_SUFFIX.gnorm_data.txt file to the working directory - # It's ok if it doesn't exist; we'll create a new one if needed. - #------------------------------------------------------------------ - if [[ -s ${M_TANKverf}/gnorm_data.txt ]]; then - $NCP ${M_TANKverf}/gnorm_data.txt gnorm_data.txt - elif [[ -s ${M_TANKverfM1}/gnorm_data.txt ]]; then - $NCP ${M_TANKverfM1}/gnorm_data.txt gnorm_data.txt - fi - - - #------------------------------------------------------------------ - # Run the child sccripts. - #------------------------------------------------------------------ - ${USHminmon}/minmon_xtrct_costs.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_costs=$? - echo "rc_costs = $rc_costs" - - ${USHminmon}/minmon_xtrct_gnorms.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_gnorms=$? - echo "rc_gnorms = $rc_gnorms" - - ${USHminmon}/minmon_xtrct_reduct.pl ${MINMON_SUFFIX} ${PDY} ${cyc} ${gsistat} dummy - rc_reduct=$? - echo "rc_reduct = $rc_reduct" - -fi - -##################################################################### -# Postprocessing - -err=0 -if [[ ${data_available} -ne 1 ]]; then - err=1 -elif [[ $rc_costs -ne 0 ]]; then - err=$rc_costs -elif [[ $rc_gnorms -ne 0 ]]; then - err=$rc_gnorms -elif [[ $rc_reduct -ne 0 ]]; then - err=$rc_reduct -fi - -if [[ "$VERBOSE" = "YES" ]]; then - echo "end exgfs_vrfminmon.sh, exit value = ${err}" -fi - - -set +x -exit ${err} - diff --git a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_costs.pl b/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_costs.pl deleted file mode 100755 index 502032da..00000000 --- a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_costs.pl +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env perl - -#--------------------------------------------------------------------------- -# minmon_xtrct_costs.pl -# -# Extract cost data from gsistat file and load into cost -# and cost term files. -#--------------------------------------------------------------------------- - -use strict; -use warnings; - -#---------------------------------------------- -# subroutine to trim white space from strings -#---------------------------------------------- -sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s }; - - -#--------------------------- -# -# Main routine begins here -# -#--------------------------- - -if ($#ARGV != 4 ) { - print "usage: minmon_xtrct_costs.pl SUFFIX PDY cyc infile jlogfile\n"; - exit; -} -my $suffix = $ARGV[0]; - -my $pdy = $ARGV[1]; -my $cyc = $ARGV[2]; -my $infile = $ARGV[3]; -my $jlogfile = $ARGV[4]; - -my $use_costterms = 0; -my $no_data = 0.00; - -my $scr = "minmon_xtrct_costs.pl"; -print "$scr has started\n"; - - -my $rc = 0; -my $cdate = sprintf '%s%s', $pdy, $cyc; - -if( (-e $infile) ) { - - my $found_cost = 0; - my $found_costterms = 0; - my @cost_array; - my @jb_array; - my @jo_array; - my @jc_array; - my @jl_array; - my @term_array; - my @all_cost_terms; - - my $cost_target; - my $cost_number; - my $costterms_target; - my $jb_number = 5; - my $jo_number = 6; - my $jc_number = 7; - my $jl_number = 8; - - my $costfile = $ENV{"mm_costfile"}; - - if( (-e $costfile) ) { - open( COSTFILE, "<${costfile}" ) or die "Can't open ${costfile}: $!\n"; - my $line; - - while( $line = ) { - if( $line =~ /cost_target/ ) { - my @termsline = split( /:/, $line ); - $cost_target = $termsline[1]; - } elsif( $line =~ /cost_number/ ) { - my @termsline = split( /:/, $line ); - $cost_number = $termsline[1]; - } elsif( $line =~ /costterms_target/ ){ - my @termsline = split( /:/, $line ); - $costterms_target = $termsline[1]; - } - } - close( COSTFILE ); - } else { - $rc = 2; - } - - #------------------------------------------------------------------------ - # Open the infile and search for the $costterms_target and $cost_target - # strings. If found, parse out the cost information and push into - # holding arrays. - #------------------------------------------------------------------------ - if( $rc == 0 ) { - open( INFILE, "<${infile}" ) or die "Can't open ${infile}: $!\n"; - - my $line; - my $term_ctr=0; - - while( $line = ) { - - if( $line =~ /$costterms_target/ ) { - my @termsline = split( / +/, $line ); - push( @jb_array, $termsline[$jb_number] ); - push( @jo_array, $termsline[$jo_number] ); - push( @jc_array, $termsline[$jc_number] ); - push( @jl_array, $termsline[$jl_number] ); - $use_costterms = 1; - } - - if( $line =~ /$cost_target/ ) { - my @costline = split( / +/, $line ); - push( @cost_array, $costline[$cost_number] ); - } - - if( $term_ctr > 0 ) { - my @termline = split( / +/, $line ); - - if ( $term_ctr < 10 ) { - push( @term_array, trim($termline[1]) ); - push( @term_array, trim($termline[2]) ); - push( @term_array, trim($termline[3]) ); - $term_ctr++; - } else { - push( @term_array, trim($termline[1]) ); - push( @term_array, trim($termline[2]) ); - $term_ctr = 0; - } - - }elsif ( $line =~ "J=" && $line !~ "EJ=" ) { - my @termline = split( / +/, $line ); - push( @term_array, trim($termline[2]) ); - push( @term_array, trim($termline[3]) ); - push( @term_array, trim($termline[4]) ); - $term_ctr = 1; - } - } - - close( INFILE ); - - - #---------------------------------------------- - # move cost_array into all_costs by iteration - #---------------------------------------------- - my @all_costs; - for my $i (0 .. $#cost_array) { - my $iterline; - if( $use_costterms == 1 ){ - $iterline = sprintf ' %d,%e,%e,%e,%e,%e%s', - $i, $cost_array[$i], $jb_array[$i], $jo_array[$i], - $jc_array[$i], $jl_array[$i], "\n"; - } - else { - $iterline = sprintf ' %d,%e,%e,%e,%e,%e%s', - $i, $cost_array[$i], $no_data, $no_data, - $no_data, $no_data, "\n"; - } - - push( @all_costs, $iterline ); - } - - #--------------------------------------------------- - # move term_array into all_cost_terms by iteration - #--------------------------------------------------- - if( @term_array > 0 ) { - my $nterms = 32; - my $max_iter = ($#term_array+1)/$nterms; - my $niter = $max_iter -1; - - for my $iter (0 .. $niter ) { - my $step = $iter * $nterms; - my $iterline = sprintf '%d, %e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e,%e%s', - $iter, $term_array[$step], $term_array[$step+1], $term_array[$step+2], - $term_array[$step+3], $term_array[$step+4], $term_array[$step+5], - $term_array[$step+6], $term_array[$step+7], $term_array[$step+8], - $term_array[$step+9], $term_array[$step+10], $term_array[$step+11], - $term_array[$step+12], $term_array[$step+13], $term_array[$step+14], - $term_array[$step+15], $term_array[$step+16], $term_array[$step+17], - $term_array[$step+18], $term_array[$step+19], $term_array[$step+20], - $term_array[$step+21], $term_array[$step+22], $term_array[$step+23], - $term_array[$step+24], $term_array[$step+25], $term_array[$step+26], - $term_array[$step+27], $term_array[$step+28], $term_array[$step+29], - $term_array[$step+30], $term_array[$step+31], "\n"; - push( @all_cost_terms, $iterline ); - } - } - - #------------------------------------------ - # write all_costs array to costs.txt file - #------------------------------------------ - my $filename2 = "${cdate}.costs.txt"; - if( @all_costs > 0 ) { - open( OUTFILE, ">$filename2" ) or die "Can't open ${filename2}: $!\n"; - print OUTFILE @all_costs; - close( OUTFILE ); - } - - #----------------------------------------------------- - # write all_cost_terms array to costs_terms.txt file - #----------------------------------------------------- - my $filename3 = "${cdate}.cost_terms.txt"; - if( @all_cost_terms > 0 ) { - open( OUTFILE, ">$filename3" ) or die "Can't open ${filename3}: $!\n"; - print OUTFILE @all_cost_terms; - close( OUTFILE ); - } - - #-------------------------- - # move files to $M_TANKverf - #-------------------------- - my $tankdir = $ENV{"M_TANKverf"}; - if(! -d $tankdir) { - system( "mkdir -p $tankdir" ); - } - - if( -e $filename2 ) { - my $newfile2 = "${tankdir}/${filename2}"; - system("cp -f $filename2 $newfile2"); - } - if( -e $filename3 ) { - my $newfile3 = "${tankdir}/${filename3}"; - system("cp -f $filename3 $newfile3"); - } - - } # $rc still == 0 after reading gmon_cost.txt -} -else { # $infile does not exist - $rc = 1; -} - -print "$scr has ended, return code = $rc \n" diff --git a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_gnorms.pl b/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_gnorms.pl deleted file mode 100755 index 61da749a..00000000 --- a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_gnorms.pl +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use List::MoreUtils 'true'; -use List::MoreUtils 'first_index'; -use List::MoreUtils 'last_index'; - -#--------------------------------------------------------------------------- -# minmon_xtrct_gnorms.pl -# -# Update the gnorm_data.txt file with data from a new cycle. Add -# this new data to the last line of the gnorm_data.txt file. -# -# Note: If the gnorm_data.txt file does not exist, it will be created. -# -# The gnorm_data.txt file is used plotted directly by the javascript on -# the GSI stats page. -#--------------------------------------------------------------------------- -sub updateGnormData { - my $cycle = $_[0]; - my $igrad = $_[1]; - my $fgnorm = $_[2]; - my $avg_gnorm = $_[3]; - my $min_gnorm = $_[4]; - my $max_gnorm = $_[5]; - my $suffix = $_[6]; - - my $rc = 0; - my @filearray; - - my $gdfile = "gnorm_data.txt"; - - my $outfile = "new_gnorm_data.txt"; - my $yr = substr( $cycle, 0, 4); - my $mon = substr( $cycle, 4, 2); - my $day = substr( $cycle, 6, 2); - my $hr = substr( $cycle, 8, 2); - - my $newln = sprintf ' %04d,%02d,%02d,%02d,%e,%e,%e,%e,%e%s', - $yr, $mon, $day, $hr, $igrad, $fgnorm, - $avg_gnorm, $min_gnorm, $max_gnorm, "\n"; - - #------------------------------------------------------------- - # attempt to locate the latest $gdfile and copy it locally - # - if( -e $gdfile ) { - open( INFILE, "<${gdfile}" ) or die "Can't open ${gdfile}: $!\n"; - - @filearray = ; - -# This is the mechanism that limits the data to 30 days worth. Should I -# keep it or let the transfer script(s) truncate? 6/12/16 -- I'm going to keep -# it. I can add this as a later change once I add a user mechanism to vary the -# amount of data plotted (on the fly). - - my $cyc_interval = $ENV{'CYCLE_INTERVAL'}; - if( $cyc_interval eq "" ) { - $cyc_interval = 6; - } - - my $max_cyc = 119; # default 30 days worth of data = 120 cycles - # If CYCLE_INTERVAL is other than "" or 6 - # then set the $max_cyc using that interval - if( $cyc_interval != 6 && $cyc_interval != 0 ) { - my $cyc_per_day = 24 / $cyc_interval; - $max_cyc = (30 * $cyc_per_day) - 1; - } - - while( $#filearray > $max_cyc ) { - shift( @filearray ); - } - close( INFILE ); - } - - # Here is the problem Russ encountered after re-running the MinMon: - # If the cycle time in $newln is the same as an existing record in - # *.gnorm_data.txt then we end up with 2+ rows for the same cycle time. - # In that case $newln should replace the first existing line - # in @filearray and all other lines that might match should be deleted. - # Else when the cycle time doesn't already exist (the expected condition) - # it should be pushed into @filearray. - - # algorithm: - # ========= - # Establish $count of matches on "$yr,$mon,$day,$hr" - # if $count > 0 - # while $count > 1 - # get last_index and remove with splice - # replace first_index with $newln - # else - # push $newln - # - my $srch_strng = "$yr,$mon,$day,$hr"; - my $count = true { /$srch_strng/ } @filearray; - - if( $count > 0 ) { - while( $count > 1 ) { - my $l_index = last_index { /$srch_strng/ } @filearray; - splice @filearray, $l_index, 1; - $count = true { /$srch_strng/ } @filearray; - } - my $f_index = first_index { /$srch_strng/ } @filearray; - splice @filearray, $f_index, 1, $newln; - } - else { - push( @filearray, $newln ); - } - - open( OUTFILE, ">$outfile" ) or die "Can't open ${$outfile}: $!\n"; - print OUTFILE @filearray; - close( OUTFILE ); - - system("cp -f $outfile $gdfile"); - -} - -#--------------------------------------------------------------------------- -# makeErrMsg -# -# Apply a gross check on the final value of the gnorm for a specific -# cycle. If the final_gnorm value is greater than the gross_check value -# then put that in the error message file. Also check for resets or a -# premature halt, and journal those events to the error message file too. -# -# Note to self: reset_iter array is passed by reference -#--------------------------------------------------------------------------- -sub makeErrMsg { - my $suffix = $_[0]; - my $cycle = $_[1]; - my $final_gnorm = $_[2]; - my $stop_flag = $_[3]; - my $stop_iter = $_[4]; - my $reset_flag = $_[5]; - my $reset_iter = $_[6]; #reset iteration array - my $infile = $_[7]; - my $gross_check = $_[8]; - - my $mail_msg =""; - my $out_file = "${cycle}.errmsg.txt"; - - - if( $stop_flag > 0 ) { - my $stop_msg = " Gnorm check detected premature iteration stop: suffix = $suffix, cycle = $cycle, iteration = $stop_iter"; - $mail_msg .= $stop_msg; - } - - if( $reset_flag > 0 ) { - my $ctr=0; - my $reset_msg = "\n Gnorm check detected $reset_flag reset(s): suffix = $suffix, cycle = $cycle"; - $mail_msg .= $reset_msg; - $mail_msg .= "\n"; - $mail_msg .= " Reset(s) detected in iteration(s): @{$reset_iter}[$ctr] \n"; - - my $arr_size = @{$reset_iter}; - for( $ctr=1; $ctr < $arr_size; $ctr++ ) { - $mail_msg .= " @{$reset_iter}[$ctr]\n"; - } - } - - if( $final_gnorm >= $gross_check ){ - my $gnorm_msg = " Final gnorm gross check failure: suffix = $suffix, cycle = $cycle, final gnorm = $final_gnorm "; - - $mail_msg .= $gnorm_msg; - } - - if( length $mail_msg > 0 ){ - my $file_msg = " File source for report is: $infile"; - $mail_msg .= $file_msg; - } - - if( length $mail_msg > 0 ){ - my $mail_link = "http://www.emc.ncep.noaa.gov/gmb/gdas/gsi_stat/index.html?src=$suffix&typ=gnorm&cyc=$cycle"; - open( OUTFILE, ">$out_file" ) or die "Can't open ${$out_file}: $!\n"; - print OUTFILE $mail_msg; - print OUTFILE "\n\n $mail_link"; - close( OUTFILE ); - } -} - - -#--------------------------------------------------------------------------- -# -# Main routine begins here -# -#--------------------------------------------------------------------------- - -if ($#ARGV != 4 ) { - print "usage: minmon_xtrct_gnorms.pl SUFFIX pdy cyc infile jlogfile\n"; - exit; -} - - -my $suffix = $ARGV[0]; -my $pdy = $ARGV[1]; -my $cyc = $ARGV[2]; -my $infile = $ARGV[3]; -my $jlogfile = $ARGV[4]; - - -my $scr = "minmon_xtrct_gnorms.pl"; -print "$scr Has Started\n"; - -# -# This needs to be redesigned to get the gnorm value from the gsistat file -# using the line that starts "cost,grad,step,b,step?:". The line formerly -# used for the gnorm and reduction values may not be available if the the -# verbose output flag is set to FALSE. -# -# So, using the grad value on that line: -# gnorm[i] = (grad[i]**)/(grad[0]**) -# reduct[i] = sqrt(gnorm) - -my $igrad_target; -my $igrad_number; -my $expected_gnorms; -my $gross_check_val; - -my $rc = 0; -my $cdate = sprintf '%s%s', $pdy, $cyc; - -my $gnormfile = $ENV{"mm_gnormfile"}; - - -if( (-e $gnormfile) ) { - open( GNORMFILE, "<${gnormfile}" ) or die "Can't open ${gnormfile}: $!\n"; - my $line; - - while( $line = ) { - if( $line =~ /igrad_target/ ) { - my @termsline = split( /:/, $line ); - $igrad_target = $termsline[1]; - } elsif( $line =~ /igrad_number/ ) { - my @termsline = split( /:/, $line ); - $igrad_number = $termsline[1]; - } elsif( $line =~ /expected_gnorms/ ){ - my @termsline = split( /:/, $line ); - $expected_gnorms = $termsline[1]; - } elsif( $line =~ /gross_check_val/ ){ - my @termsline = split( /:/, $line ); - $gross_check_val = $termsline[1]; - } - } - close( GNORMFILE ); -} else { - $rc = 4; -} - -if( $rc == 0 ) { - if( (-e $infile) ) { - open( INFILE, "<${infile}" ) or die "Can't open ${infile}: $!\n"; - - my $found_igrad = 0; - my $final_gnorm = 0.0; - my $igrad = 0.0; - my $header = 4; - my $header2 = 0; - my @gnorm_array; - my @last_10_gnorm; - - my $reset_flag = 0; - my $stop_flag = 0; - my $warn_str = "WARNING"; - my $stop_str = "Stopping"; - my $stop_iter = ""; - my $reset_str = "Reset"; - my @reset_iter; # reset iteration array - - my $stop_iter_flag = 0; - my $reset_iter_flag = 0; - my $line; - while( $line = ) { - - ############################################## - # if the reset_iter_flag is 1 then record the - # current outer & inner iteration number - ############################################## - if( $reset_iter_flag == 1 ) { - if( $line =~ /${igrad_target}/ ) { - my @iterline = split( / +/, $line ); - my $iter_str = $iterline[2] . "," . $iterline[3]; - push( @reset_iter, $iter_str); - $reset_iter_flag = 0; - } - } - - - if( $line =~ /${igrad_target}/ ) { - my @gradline = split( / +/, $line ); - - my $grad = $gradline[$igrad_number]; - - if( $found_igrad == 0 ){ - $igrad = $grad; - $found_igrad = 1; - } - - my $igrad_sqr = $igrad**2; - my $grad_sqr = $grad**2; - my $gnorm = $grad_sqr/$igrad_sqr; - - push( @gnorm_array, $gnorm ); - } - - - if( $line =~ /${warn_str}/ ) { - if( $line =~ /${stop_str}/ ) { - $stop_flag++; - $stop_iter_flag=1; - } - elsif( $line =~ /${reset_str}/ ){ - $reset_flag++; - $reset_iter_flag = 1; - } - } - - } - close( INFILE ); - - ######################################################################## - # If the stop_flag is >0 then record the last outer & inner - # iteration number. The trick is that it's the last iteration in the - # log file and we just passed it when we hit the stop warning message, - # so we have to reopen the file and get the last iteration number. - ######################################################################## - if( $stop_flag > 0 ) { - open( INFILE, "<${infile}" ) or die "Can't open ${infile}: $!\n"; - - my @lines = reverse ; - foreach $line (@lines) { - if( $line =~ /${igrad_target}/ ){ - my @iterline = split( / +/, $line ); - $stop_iter = $iterline[2] . "," . $iterline[3]; - last; - } - } - close( INFILE ); - } - - - my @all_gnorm = @gnorm_array; - - ############################################################################## - ## - ## If the iterations were halted due to error then the @all_gnorm array won't - ## be the expected size. In that case we need to pad the array out with - ## RMISS values so GrADS won't choke when it tries to read the data file. - ## - ## Note that we're padding @all_gnorm. The @gnorm_array is examined below - ## and we don't want to pad that and mess up the min/max calculation. - ## - ############################################################################### - my $arr_size = @all_gnorm; - - if( $arr_size < $expected_gnorms ) { - for( my $ctr = $arr_size; $ctr < $expected_gnorms; $ctr++ ) { - push( @all_gnorm, -999.0 ); - } - } - - my $sum_10_gnorm = 0.0; - my $min_gnorm = 9999999.0; - my $max_gnorm = -9999999.0; - my $avg_gnorm = 0.0; - - for( my $ctr = 9; $ctr >= 0; $ctr-- ) { - my $new_gnorm = pop( @gnorm_array ); - $sum_10_gnorm = $sum_10_gnorm + $new_gnorm; - if( $new_gnorm > $max_gnorm ) { - $max_gnorm = $new_gnorm; - } - if( $new_gnorm < $min_gnorm ) { - $min_gnorm = $new_gnorm; - } - if( $ctr == 9 ) { - $final_gnorm = $new_gnorm; - } - } - - $avg_gnorm = $sum_10_gnorm / 10; - - - ##################################################################### - # Update the gnorm_data.txt file with information on the - # initial gradient, final gnorm, and avg/min/max for the last 10 - # iterations. - ##################################################################### - updateGnormData( $cdate,$igrad,$final_gnorm,$avg_gnorm,$min_gnorm,$max_gnorm,$suffix ); - - - ##################################################################### - # Call makeErrMsg to build the error message file to record any - # abnormalities in the minimization. This file can be mailed by - # a calling script. - ##################################################################### - makeErrMsg( $suffix, $cdate, $final_gnorm, $stop_flag, $stop_iter, $reset_flag, \@reset_iter, $infile, $gross_check_val ); - - - ######################################################### - # write to GrADS ready output data file - # - # Note: this uses pack to achieve the same results as - # an unformatted binary Fortran file. - ######################################################### - my $filename2 = "${cdate}.gnorms.ieee_d"; - - open( OUTFILE, ">$filename2" ) or die "Can't open ${filename2}: $!\n"; - binmode OUTFILE; - - print OUTFILE pack( 'f*', @all_gnorm); - - close( OUTFILE ); - - #-------------------------- - # move files to $M_TANKverf - #-------------------------- - my $tankdir = $ENV{"M_TANKverf"}; - print "M_TANKverf = $tankdir \n"; - if(! -d $tankdir) { - print "making $tankdir\n"; - system( "mkdir -p $tankdir" ); - } - - if( -e $filename2 ) { - system("cp -f $filename2 ${tankdir}/."); - } - - my $gdfile = "gnorm_data.txt"; - if( -e $gdfile ) { - system("cp -f $gdfile ${tankdir}/."); - } - - my $errmsg = "${cdate}.errmsg.txt"; - if( -e $errmsg ) { - system("cp -f $errmsg ${tankdir}/."); - } - - } # $rc still == 0 after reading gmon_gnorm.txt - -}else { # $infile does not exist - $rc = 3; -} - -print "$scr has ended, return code = $rc \n" diff --git a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_reduct.pl b/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_reduct.pl deleted file mode 100755 index 1b8186b6..00000000 --- a/src/Minimization_Monitor/nwprod/minmon_shared/ush/minmon_xtrct_reduct.pl +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env perl - -use strict; - -#--------------------------------------------------------------------------- -# minmon_xtrct_reduct.pl -# -# Extract the reduction stats for a GSI minimization run and store in -# reduction.ieee_d files ready for GrADS use. -#--------------------------------------------------------------------------- - -if ($#ARGV != 4 ) { - print "usage: minmon_xtrct_reduct.pl SUFFIX pdy cyc infile jlogfile\n"; - print " suffix is data source identifier\n"; - print " pdy is YYYYMMDD of the cycle to be processed\n"; - print " cyc is HH of the cycle to be processed\n"; - print " infile is the data file containing the reduction stats\n"; - print " jlogfile is the job log file\n"; - exit; -} -my $suffix = $ARGV[0]; -my $pdy = $ARGV[1]; -my $cyc = $ARGV[2]; -my $infile = $ARGV[3]; -my $jlogfile = $ARGV[4]; - -my $scr = "minmon_xtrct_reduct.pl"; -print "$scr has started\n"; - -my $rc = 0; -my $cdate = sprintf '%s%s', $pdy, $cyc; -my $initial_gradient = -999.0; -my $iter_gradient; - -if( (-e $infile) ) { - - my $reduct_target = "cost,grad,step,b,step?"; - my $gradient_num = 5; - my $reduct; - - open( INFILE, "<${infile}" ) or die "Can't open ${infile}: $!\n"; - - my @reduct_array; - - while( my $line = ) { - if( $line =~ /$reduct_target/ ) { - my @reduct_ln = split( / +/, $line ); - $iter_gradient = $reduct_ln[$gradient_num]; - if( $initial_gradient == -999.0 ){ - $initial_gradient = $iter_gradient; - } - - $reduct = $iter_gradient / $initial_gradient; - - push( @reduct_array, $reduct ); - } - } - - close( INFILE ); - - - ################################# - # write reduct_array to outfile - ################################# - my $outfile = "${cdate}.reduction.ieee_d"; - open( OUTFILE, ">$outfile" ) or die "Can't open ${outfile}: $!\n"; - binmode OUTFILE; - - print OUTFILE pack( 'f*', @reduct_array); - close( OUTFILE ); - - #---------------------------- - # copy outfile to $M_TANKverf - #---------------------------- - my $tankdir = $ENV{"M_TANKverf"}; - if(! -d $tankdir) { - system( "mkdir -p $tankdir" ); - } - - if( -e $outfile ) { - my $newfile = "${tankdir}/${outfile}"; - system("cp -f $outfile $newfile"); - } - -} else { # $infile does not exist - $rc = 5; -} - -print "$scr has ended, return code = $rc \n" diff --git a/src/Ozone_Monitor/nwprod/gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN b/src/Ozone_Monitor/nwprod/gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN deleted file mode 100755 index 1acd7432..00000000 --- a/src/Ozone_Monitor/nwprod/gdas_oznmon/jobs/JGDAS_ATMOS_VERFOZN +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh -############################################################# -# Set up environment for GDAS Ozone Monitor job -############################################################# -set -xa -echo `date` $0 `date -u` begin - - -############################### -# Specify NET, RUN, and COMPONENT name -############################## -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export COMPONENT=${COMPONENT:-atmos} - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export pid=$$ -export outid=${outid:-"LL$job"} -export DATA=${DATA:-${DATAROOT}/${jobid:?}} - -export OZNMON_SUFFIX=${OZNMON_SUFFIX:-${NET}} - -mkdir -p ${DATA} -cd ${DATA} - - -#################################### -# Determine Job Output Name on System -#################################### -export pgmout="OUTPUT.${pid}" -export pgmerr=errfile -export cycle=t${cyc}z - - -#--------------------------------------------- -# Specify Execution Areas -# -export HOMEgfs_ozn=${HOMEgfs:-${NWROOT}/gfs.${gfs_ver}} -export HOMEgdas_ozn=${HOMEgfs_ozn:-${NWROOT}/gfs.${gfs_ver}} -export PARMgdas_ozn=${PARMgfs_ozn:-$HOMEgfs_ozn/parm/mon} -export SCRgdas_ozn=${SCRgfs_ozn:-$HOMEgfs_ozn/scripts} -export FIXgdas_ozn=${FIXgfs_ozn:-$HOMEgfs_ozn/fix/gdas} - -export HOMEoznmon=${HOMEoznmon:-${HOMEgfs_ozn}} -export EXECoznmon=${EXECoznmon:-$HOMEoznmon/exec} -export FIXoznmon=${FIXoznmon:-${HOMEoznmon}/fix} -export USHoznmon=${USHoznmon:-$HOMEoznmon/ush} - - -#----------------------------------- -# source the parm file -# -. ${PARMgdas_ozn}/gdas_oznmon.parm - - -############################################# -# determine PDY and cyc for previous cycle -############################################# - -cdate=`${NDATE} -6 ${PDY}${cyc}` -echo 'pdate = ${pdate}' - -export P_PDY=`echo ${cdate} | cut -c1-8` -export p_cyc=`echo ${cdate} | cut -c9-10` - -#--------------------------------------------- -# OZN_TANKDIR - WHERE OUTPUT DATA WILL RESIDE -# -export OZN_TANKDIR=${OZN_TANKDIR:-$(compath.py ${envir}/${NET}/${gfs_ver})} -export TANKverf_ozn=${TANKverf_ozn:-${OZN_TANKDIR}/${RUN}.${PDY}/${cyc}/${COMPONENT}/oznmon} -export TANKverf_oznM1=${TANKverf_oznM1:-${OZN_TANKDIR}/${RUN}.${P_PDY}/${p_cyc}/${COMPONENT}/oznmon} -export COM_IN=${COM_IN:-$(compath.py ${envir}/${NET}/${gfs_ver})} -export COMIN=${COMIN:-${COM_IN}/${RUN}.${PDY}/${cyc}/${COMPONENT}} - -if [[ ! -d ${TANKverf_ozn} ]]; then - mkdir -p -m 775 ${TANKverf_ozn} -fi - -#--------------------------------------- -# set up validation file -# -if [[ ${VALIDATE_DATA} -eq 1 ]]; then - export ozn_val_file=${ozn_val_file:-${FIXgdas_ozn}/gdas_oznmon_base.tar} -fi - -#--------------------------------------- -# Set necessary environment variables -# -export OZN_AREA=${OZN_AREA:-glb} -export oznstat=${oznstat:-$COMIN/gdas.t${cyc}z.oznstat} - - -#------------------------------------------------------- -# Execute the script. -# -${OZNMONSH:-${SCRgdas_ozn}/exgdas_atmos_verfozn.sh} ${PDY} ${cyc} -err=$? -[[ $err -ne 0 ]] && exit $err - - -################################ -# Remove the Working Directory -################################ -KEEPDATA=${KEEPDATA:-YES} -cd $DATAROOT -if [ ${KEEPDATA} = NO ] ; then - rm -rf $DATA -fi - -date - diff --git a/src/Ozone_Monitor/nwprod/gdas_oznmon/scripts/exgdas_atmos_verfozn.sh b/src/Ozone_Monitor/nwprod/gdas_oznmon/scripts/exgdas_atmos_verfozn.sh deleted file mode 100755 index 939b03a3..00000000 --- a/src/Ozone_Monitor/nwprod/gdas_oznmon/scripts/exgdas_atmos_verfozn.sh +++ /dev/null @@ -1,100 +0,0 @@ -#/bin/sh - -set -ax - -################################################################################ -# exgdas_vrfyozn.sh -# -# This script runs the data extract/validation portion of the Ozone Monitor -# (OznMon) DA package. -# -################################################################################ -export scr=exgdas_vrfyozn.sh - -err=0 - -#------------------------------------------------------------------------------- -# Set environment -# -export RUN_ENVIR=${RUN_ENVIR:-nco} -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export envir=${envir:-prod} -export COMPONENT=${COMPONENT:-atmos} - -# Command line arguments -export PDY=${1:-${PDY:?}} -export cyc=${2:-${cyc:?}} - - -# Other variables -export SATYPE_FILE=${SATYPE_FILE:-$FIXgdas_ozn/gdas_oznmon_satype.txt} -export PDATE=${PDY}${cyc} -export DO_DATA_RPT=${DO_DATA_RPT:-1} -export NCP=${NCP:-/bin/cp} - - -#----------------------------------------------------------------- -# ensure work and TANK dirs exist, verify oznstat is available -# -export OZN_WORK_DIR=${OZN_WORK_DIR:-$(pwd)} - -if [[ ! -d ${OZN_WORK_DIR} ]]; then - mkdir $OZN_WORK_DIR -fi -cd $OZN_WORK_DIR - -if [[ ! -d ${TANKverf_ozn} ]]; then - mkdir -p $TANKverf_ozn -fi - -if [[ -s ${oznstat} ]]; then - echo ${oznstat} is available -fi - - - -data_available=0 - -if [[ -s ${oznstat} ]]; then - data_available=1 - - #------------------------------------------------------------------ - # Copy data files file to local data directory. - # Untar oznstat file. - #------------------------------------------------------------------ - - $NCP $oznstat ./oznstat.$PDATE - - tar -xvf oznstat.$PDATE - rm oznstat.$PDATE - - netcdf=0 - count=`ls diag* | grep ".nc4" | wc -l` - if [ $count -gt 0 ] ; then - netcdf=1 - for filenc4 in `ls diag*nc4.gz`; do - file=`echo $filenc4 | cut -d'.' -f1-2`.gz - mv $filenc4 $file - done - fi - - export OZNMON_NETCDF=${netcdf} - - ${HOMEoznmon}/ush/ozn_xtrct.sh - err=$? - -else - # oznstat file not found - err=1 -fi - - -if [[ "$VERBOSE" = "YES" ]]; then - echo "end exgdas_vrfyozn.sh, exit value = ${err}" -fi - - -set +x -exit ${err} - diff --git a/src/Ozone_Monitor/nwprod/oznmon_shared/ush/ozn_xtrct.sh b/src/Ozone_Monitor/nwprod/oznmon_shared/ush/ozn_xtrct.sh deleted file mode 100755 index f301a95f..00000000 --- a/src/Ozone_Monitor/nwprod/oznmon_shared/ush/ozn_xtrct.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash -#------------------------------------------------------------------ -# ozn_xtrct.sh -# -# This script performs the data extraction from the oznstat -# diagnostic files. The resulting data (*.ieee_d) files, GrADS -# control files and stdout files will be moved to the -# $TANKverf_ozn. -# -# Calling scripts must define: -# $TANKverf_ozn -# $HOMEoznmon -# $PDATE -# -# Return values are -# 0 = normal -# 2 = unable to generate satype list; may indicate no diag -# files found in oznstat file -#------------------------------------------------------------------ - -set -ax - - -#-------------------------------------------------- -# check_diag_files -# -# Compare $satype (which contains the contents of -# gdas_oznmon_satype.txt to $avail_satype which is -# determined by the contents of the oznstat file. -# Report any missing diag files in a file named -# bad_diag.$PDATE -# -check_diag_files() { - pdate=$1 - found_satype=$2 - avail_satype=$3 - - out_file="bad_diag.${pdate}" - - echo ""; echo ""; echo "--> check_diag_files" - - for type in ${found_satype}; do - check=`echo ${avail_satype} | grep ${type}` - len_check=`echo -n "${check}" | wc -c` - - if [[ ${len_check} -le 1 ]]; then - echo "missing diag file -- diag_${type}_ges.${pdate}.gz not found " >> ./${out_file} - fi - done - - echo "<-- check_diag_files"; echo ""; echo "" -} - - - -echo "start ozn_xtrct.sh" - -iret=0 -export NCP=${NCP:-/bin/cp} -VALIDATE_DATA=${VALIDATE_DATA:-0} -nregion=${nregion:-6} -DO_DATA_RPT=${DO_DATA_RPT:-0} - -netcdf_boolean=".false." -if [[ $OZNMON_NETCDF -eq 1 ]]; then - netcdf_boolean=".true." -fi - -OZNMON_NEW_HDR=${OZNMON_NEW_HDR:-0} -new_hdr="F" -if [[ $OZNMON_NEW_HDR -eq 1 ]]; then - new_hdr="T" -fi - -#------------------------------------------------------------------ -# if VALIDATE_DATA then locate and untar base file -# -validate=".FALSE." -if [[ $VALIDATE_DATA -eq 1 ]]; then - if [[ ! -e $ozn_val_file && ! -h $ozn_val_file ]]; then - echo "WARNING: VALIDATE_DATA set to 1, but unable to locate $ozn_val_file" - echo " Setting VALIDATE_DATA to 0/OFF" - VALIDATE_DATA=0 - else - validate=".TRUE." - val_file=`basename ${ozn_val_file}` - ${NCP} $ozn_val_file $val_file - tar -xvf $val_file - fi -fi -echo "VALIDATE_DATA, validate = $VALIDATE_DATA, $validate " - - - -#------------------------------------------------------------------ -# ozn_ptype here is the processing type which is intended to be "ges" -# or "anl". Default is "ges". -# -ozn_ptype=${ozn_ptype:-"ges anl"} - - -#--------------------------------------------------------------------------- -# Build satype list from the available diag files. -# -# An empty satype list means there are no diag files to process. That's -# a problem, reported by an iret value of 2 -# - -avail_satype=`ls -l d*ges* | sed -e 's/_/ /g;s/\./ /' | gawk '{ print $11 "_" $12 }'` - -if [[ ${DO_DATA_RPT} -eq 1 ]]; then - if [[ -e ${SATYPE_FILE} ]]; then - satype=`cat ${SATYPE_FILE}` - check_diag_files ${PDATE} "${satype}" "${avail_satype}" - else - echo "WARNING: missing ${SATYPE_FILE}" - fi -fi - -len_satype=`echo -n "${satype}" | wc -c` - -if [[ ${len_satype} -le 1 ]]; then - satype=${avail_satype} -fi - -echo ${satype} - - -len_satype=`echo -n "${satype}" | wc -c` - -if [[ ${DO_DATA_RPT} -eq 1 && ${len_satype} -lt 1 ]]; then - iret=2 - -else - - #-------------------------------------------------------------------- - # Copy extraction programs to working directory - # - ${NCP} ${HOMEoznmon}/exec/oznmon_time.x ./oznmon_time.x - if [[ ! -e oznmon_time.x ]]; then - iret=2 - exit ${iret} - fi - ${NCP} ${HOMEoznmon}/exec/oznmon_horiz.x ./oznmon_horiz.x - if [[ ! -e oznmon_horiz.x ]]; then - iret=3 - exit ${iret} - fi - - - #--------------------------------------------------------------------------- - # Outer loop over $ozn_ptype (default values 'ges', 'anl') - # - echo "ozn_ptype = ${ozn_ptype}" - for ptype in ${ozn_ptype}; do - echo "ptype = ${ptype}" - - - for type in ${satype}; do - mv diag_${type}_${ptype}.${PDATE}.gz ${type}.${ptype}.gz - gunzip ./${type}.${ptype}.gz - done - - - #-------------------------------------------------------------------- - # Run programs for given time - - iyy=`echo ${PDATE} | cut -c1-4` - imm=`echo ${PDATE} | cut -c5-6` - idd=`echo ${PDATE} | cut -c7-8` - ihh=`echo ${PDATE} | cut -c9-10` - - for type in ${satype}; do - echo "processing ptype, type: ${ptype}, ${type}" - rm -f input - -cat << EOF > input - &INPUT - satname='${type}', - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-720, - incr=6, - nregion=${nregion}, - region(1)='global', rlonmin(1)=-180.0,rlonmax(1)=180.0,rlatmin(1)=-90.0,rlatmax(1)= 90.0, - region(2)='70N-90N', rlonmin(2)=-180.0,rlonmax(2)=180.0,rlatmin(2)= 70.0,rlatmax(2)= 90.0, - region(3)='20N-70N', rlonmin(3)=-180.0,rlonmax(3)=180.0,rlatmin(3)= 20.0,rlatmax(3)= 70.0, - region(4)='20S-20N', rlonmin(4)=-180.0,rlonmax(4)=180.0,rlatmin(4)=-20.0,rlatmax(4)= 20.0, - region(5)='20S-70S', rlonmin(5)=-180.0,rlonmax(5)=180.0,rlatmin(5)=-70.0,rlatmax(5)=-20.0, - region(6)='70S-90S', rlonmin(6)=-180.0,rlonmax(6)=180.0,rlatmin(6)=-90.0,rlatmax(6)=-70.0, - validate=${validate}, - new_hdr=${new_hdr}, - ptype=${ptype}, - netcdf=${netcdf_boolean} - / -EOF - - - echo "oznmon_time.x HAS STARTED ${type}" - - ./oznmon_time.x < input > stdout.time.${type}.${ptype} - - echo "oznmon_time.x HAS ENDED ${type}" - - if [[ ! -d ${TANKverf_ozn}/time ]]; then - mkdir -p ${TANKverf_ozn}/time - fi - $NCP ${type}.${ptype}.ctl ${TANKverf_ozn}/time/ - $NCP ${type}.${ptype}.${PDATE}.ieee_d ${TANKverf_ozn}/time/ - - $NCP bad* ${TANKverf_ozn}/time/ - - rm -f input - -cat << EOF > input - &INPUT - satname='${type}', - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-18, - incr=6, - new_hdr=${new_hdr}, - ptype=${ptype}, - netcdf=${netcdf_boolean} - / -EOF - - echo "oznmon_horiz.x HAS STARTED ${type}" - - ./oznmon_horiz.x < input > stdout.horiz.${type}.${ptype} - - echo "oznmon_horiz.x HAS ENDED ${type}" - - if [[ ! -d ${TANKverf_ozn}/horiz ]]; then - mkdir -p ${TANKverf_ozn}/horiz - fi - $NCP ${type}.${ptype}.ctl ${TANKverf_ozn}/horiz/ - - $COMPRESS ${type}.${ptype}.${PDATE}.ieee_d - $NCP ${type}.${ptype}.${PDATE}.ieee_d.${Z} ${TANKverf_ozn}/horiz/ - - - echo "finished processing ptype, type: $ptype, $type" - done # type in satype - - done # ptype in $ozn_ptype - - tar -cvf stdout.horiz.tar stdout.horiz* - ${COMPRESS} stdout.horiz.tar - ${NCP} stdout.horiz.tar.${Z} ${TANKverf_ozn}/horiz/ - - tar -cvf stdout.time.tar stdout.time* - ${COMPRESS} stdout.time.tar - ${NCP} stdout.time.tar.${Z} ${TANKverf_ozn}/time/ -fi - -#------------------------------------------------------- -# Conditionally remove data files older than 40 days -# -if [[ ${CLEAN_TANKDIR} -eq 1 ]]; then - ${HOMEoznmon}/ush/clean_tankdir.sh glb 40 -fi - - -echo "ozn_xtrct.sh HAS ENDED, iret = ${iret}" - -exit ${iret} diff --git a/src/Radiance_Monitor/nwprod/gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD b/src/Radiance_Monitor/nwprod/gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD deleted file mode 100755 index 39c7b666..00000000 --- a/src/Radiance_Monitor/nwprod/gdas_radmon/jobs/JGDAS_ATMOS_VERFRAD +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/sh -############################################################# -# Set up environment for GDAS Radiance Monitor job -############################################################# -set -xa -echo `date` $0 `date -u` begin -export PS4='$SECONDS + ' - -############################### -# Specify NET, RUN, and COMPONENT name -############################## -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export COMPONENT=${COMPONENT:-atmos} - -########################################################### -# obtain unique process id (pid) and make temp directories -########################################################### -export pid=$$ -export outid=${outid:-"LL$job"} -export RAD_DATA_IN=${RAD_DATA_IN:-${DATAROOT}/${jobid:?}} - -export RADMON_SUFFIX=${RADMON_SUFFIX:-${RUN}} -export CYCLE_INTERVAL=${CYCLE_INTERVAL:-6} - -mkdir -p $RAD_DATA_IN -cd $RAD_DATA_IN - -#################################### -# Determine Job Output Name on System -#################################### -export pgmout="OUTPUT.${pid}" -export pgmerr=errfile -export cycle=t${cyc}z - -############################################## -# Specify Execution Areas -############################################## -export HOMEgfs=${HOMEgfs:-${NWROOT}/gfs.${gfs_ver}} -export SCRgfs=${SCRgfs:-$HOMEgfs/scripts} - -export FIXgdas=${FIXgdas:-$HOMEgfs/fix/gdas} -export PARMmon=${PARMmon:-$HOMEgfs/parm/mon} - -export HOMEradmon=${HOMEradmon:-${HOMEgfs}} -export EXECradmon=${EXECradmon:-$HOMEradmon/exec} -export FIXradmon=${FIXradmon:-${FIXgfs}} -export USHradmon=${USHradmon:-$HOMEradmon/ush} - - -################################### -# source the parm file -################################### -parm_file=${parm_file:-${PARMmon}/da_mon.parm} -. ${parm_file} - - -############################################# -# Run setpdy and initialize PDY variables -############################################# -if [[ $MY_MACHINE != "HERA" && $MY_MACHINE != "hera" ]]; then - setpdy.sh - . ./PDY -fi - -############################################# -# determine PDY and cyc for previous cycle -############################################# - -cdate=`${NDATE} -6 ${PDY}${cyc}` -echo 'pdate = ${pdate}' - -export P_PDY=`echo ${cdate} | cut -c1-8` -export p_cyc=`echo ${cdate} | cut -c9-10` - -############################################# -# COMOUT - WHERE GSI OUTPUT RESIDES -# TANKverf - WHERE OUTPUT DATA WILL RESIDE -############################################# -export TANKverf=${TANKverf:-$(compath.py ${envir}/${NET}/${gfs_ver})} -export TANKverf_rad=${TANKverf_rad:-${TANKverf}/${RUN}.${PDY}/${cyc}/$COMPONENT/radmon} -export TANKverf_radM1=${TANKverf_radM1:-${TANKverf}/${RUN}.${P_PDY}/${p_cyc}/$COMPONENT/radmon} -export COM_IN=${COM_IN:-$(compath.py ${envir}/${NET}/${gfs_ver})} -export COMIN=${COMIN:-${COM_IN}/${RUN}.${PDY}/${cyc}/$COMPONENT} - -################################ -# backwards compatibility for -# gfs v15 which doesn't have -# a $COMPONENT in output path -################################ -if [[ ! -d ${COMIN} ]]; then - export COMIN=${COM_IN}/${RUN}.${PDY}/${cyc} -fi - - -mkdir -p -m 775 $TANKverf_rad - -env - -######################################## -# Set necessary environment variables -######################################## -export RAD_AREA=${RAD_AREA:-glb} - -export biascr=${biascr:-$COMIN/gdas.t${cyc}z.abias} -export radstat=${radstat:-$COMIN/gdas.t${cyc}z.radstat} - -echo " " -echo "JOB HAS STARTED" -echo " " - - -######################################################## -# Execute the script. -${RADMONSH:-${SCRgfs}/exgdas_atmos_verfrad.sh} ${PDY} ${cyc} -err=$? - -if [[ $err -ne 0 ]] ; then - exit $err -else - echo " " - echo "JOB HAS COMPLETED NORMALLY" - echo " " -fi - -################################ -# Remove the Working Directory -################################ -KEEPDATA=${KEEPDATA:-YES} -cd $DATAROOT -if [ ${KEEPDATA} = NO ] ; then - rm -rf $RAD_DATA_IN -fi - -date - diff --git a/src/Radiance_Monitor/nwprod/gdas_radmon/scripts/exgdas_atmos_verfrad.sh b/src/Radiance_Monitor/nwprod/gdas_radmon/scripts/exgdas_atmos_verfrad.sh deleted file mode 100755 index 7aec2543..00000000 --- a/src/Radiance_Monitor/nwprod/gdas_radmon/scripts/exgdas_atmos_verfrad.sh +++ /dev/null @@ -1,228 +0,0 @@ -#/bin/sh -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: exgdas_vrfyrad.sh -# Script description: Runs data extract/validation for global radiance diag data -# -# Author: Ed Safford Org: NP23 Date: 2012-01-18 -# -# Abstract: This script runs the data extract/validation portion of the -# RadMon package. -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -################################################################################ -scr=exgdas_vrfyrad.sh -echo "${scr} HAS STARTED" - -export VERBOSE=${VERBOSE:-"NO"} -if [[ "$VERBOSE" = "YES" ]] -then - set -x -fi - - -export RUN_ENVIR=${RUN_ENVIR:-nco} -export NET=${NET:-gfs} -export RUN=${RUN:-gdas} -export envir=${envir:-prod} -export COMPONENT=${COMPONENT:-atmos} - -# Command line arguments -export PDY=${1:-${PDY:?}} -export cyc=${2:-${cyc:?}} - -# Directories -export COM_IN=${COM_IN:-$(compath.py ${envir}/${NET}/${gfs_ver})} -export COMIN=${COMIN:-$COM_IN/${RUN}.${PDY}/${cyc}/$COMPONENT} - - -# Filenames -export biascr=${biascr:-$COMIN/gdas.t${cyc}z.abias} -export radstat=${radstat:-$COMIN/gdas.t${cyc}z.radstat} -export satype_file=${satype_file:-${FIXgdas}/gdas_radmon_satype.txt} - -# Other variables -export RAD_AREA=${RAD_AREA:-glb} -export MAKE_CTL=${MAKE_CTL:-1} -export MAKE_DATA=${MAKE_DATA:-1} -export USE_ANL=${USE_ANL:-1} -export PDATE=${PDY}${cyc} -export DO_DIAG_RPT=${DO_DIAG_RPT:-1} -export DO_DATA_RPT=${DO_DATA_RPT:-1} -export USE_MAIL=${USE_MAIL:-0} -export MAIL_TO=${MAIL_TO:-" "} -export MAIL_CC=${MAIL_CC:-" "} -export NCP=${NCP:-/bin/cp} - -########################################################################### -# ensure TANK dir exists, verify radstat and biascr are available -# -if [[ ! -d ${TANKverf_rad} ]]; then - mkdir -p $TANKverf_rad -fi - -if [[ "$VERBOSE" = "YES" ]]; then - if [[ -s ${radstat} ]]; then - echo ${radstat} is available - fi - if [[ -s ${biascr} ]]; then - echo ${biascr} is available - fi -fi -##################################################################### - -data_available=0 -if [[ -s ${radstat} && -s ${biascr} ]]; then - data_available=1 - - #------------------------------------------------------------------ - # Copy data files file to local data directory. - # Untar radstat file. - #------------------------------------------------------------------ - - $NCP $biascr ./biascr.$PDATE - $NCP $radstat ./radstat.$PDATE - - tar -xvf radstat.$PDATE - rm radstat.$PDATE - - #------------------------------------------------------------------ - # SATYPE is the list of expected satellite/instrument sources - # in the radstat file. It should be stored in the $TANKverf - # directory. If it isn't there then use the $FIXgdas copy. In all - # cases write it back out to the radmon.$PDY directory. Add any - # new sources to the list before writing back out. - #------------------------------------------------------------------ - - radstat_satype=`ls d*ges* | awk -F_ '{ print $2 "_" $3 }'` - if [[ "$VERBOSE" = "YES" ]]; then - echo $radstat_satype - fi - - echo satype_file = $satype_file - - #------------------------------------------------------------------ - # Get previous cycle's date, and look for the satype_file. Using - # the previous cycle will get us the previous day's directory if - # the cycle being processed is 00z. - #------------------------------------------------------------------ - if [[ $cyc = "00" ]]; then - use_tankdir=${TANKverf_radM1} - else - use_tankdir=${TANKverf_rad} - fi - - echo satype_file = $satype_file - export SATYPE=`cat ${satype_file}` - - - #------------------------------------------------------------- - # Update the SATYPE if any new sat/instrument was - # found in $radstat_satype. Write the SATYPE contents back - # to $TANKverf/radmon.$PDY. - #------------------------------------------------------------- - satype_changes=0 - new_satype=$SATYPE - for type in ${radstat_satype}; do - test=`echo $SATYPE | grep $type | wc -l` - - if [[ $test -eq 0 ]]; then - if [[ "$VERBOSE" = "YES" ]]; then - echo "Found $type in radstat file but not in SATYPE list. Adding it now." - fi - satype_changes=1 - new_satype="$new_satype $type" - fi - done - - - #------------------------------------------------------------------ - # Rename the diag files and uncompress - #------------------------------------------------------------------ - netcdf=0 - - for type in ${SATYPE}; do - - if [[ netcdf -eq 0 && -e diag_${type}_ges.${PDATE}.nc4.${Z} ]]; then - netcdf=1 - fi - - mv diag_${type}_ges.${PDATE}*.${Z} ${type}.${Z} - ${UNCOMPRESS} ./${type}.${Z} - - if [[ $USE_ANL -eq 1 ]]; then - mv diag_${type}_anl.${PDATE}*.${Z} ${type}_anl.${Z} - ${UNCOMPRESS} ./${type}_anl.${Z} - fi - done - - export RADMON_NETCDF=$netcdf - - - #------------------------------------------------------------------ - # Run the child sccripts. - #------------------------------------------------------------------ - ${USHradmon}/radmon_verf_angle.sh ${PDATE} - rc_angle=$? - - ${USHradmon}/radmon_verf_bcoef.sh ${PDATE} - rc_bcoef=$? - - ${USHradmon}/radmon_verf_bcor.sh ${PDATE} - rc_bcor=$? - - ${USHradmon}/radmon_verf_time.sh ${PDATE} - rc_time=$? - - #-------------------------------------- - # optionally run clean_tankdir script - # - if [[ ${CLEAN_TANKVERF} -eq 1 ]]; then - ${USHradmon}/clean_tankdir.sh glb 60 - rc_clean_tankdir=$? - echo "rc_clean_tankdir = $rc_clean_tankdir" - fi - -fi - - - -##################################################################### -# Postprocessing - -err=0 -if [[ ${data_available} -ne 1 ]]; then - err=1 -elif [[ $rc_angle -ne 0 ]]; then - err=$rc_angle -elif [[ $rc_bcoef -ne 0 ]]; then - err=$rc_bcoef -elif [[ $rc_bcor -ne 0 ]]; then - err=$rc_bcor -elif [[ $rc_time -ne 0 ]]; then - err=$rc_time -fi - -##################################################################### -# Restrict select sensors and satellites -export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} -rlist="saphir" -for rtype in $rlist; do - ${CHGRP_CMD} $TANKverf_rad/*${rtype}* -done - - -if [[ "$VERBOSE" = "YES" ]]; then - echo "end exgdas_vrfyrad.sh, exit value = ${err}" -fi - -echo "${scr} HAS ENDED" - - -set +x -exit ${err} - diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_err_rpt.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_err_rpt.sh deleted file mode 100755 index 93cf376c..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_err_rpt.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/ksh - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: radmon_err_rpt.sh -# Script description: Compare the contents of error files from two different -# cycles. -# -# Author: Ed Safford Org: NP23 Date: 2012-02-02 -# -# Abstract: This script compares the contents of two error files from two different -# sets of radiance diagnostic files (which are an output from GSI runs). -# All unique satellite instrument/channel/region combinations that appear -# in both files are reported. -# -# This script is run as a child script of radmon_verf_time.sh. The parent -# script creates/copies the error files into a temporary working -# directory before invoking this script. -# -# -# Usage: radmon_err_rpt.sh file1 file2 type cycle1 cycle2 diag_rpt outfile -# -# Input script positional parameters: -# file1 obs, penalty, or channel error file -# required -# file2 obs, penalty, or channel error file -# required -# type type of error file -# choices are obs, pen, chan, or cnt; required -# cycle1 first cycle processing date -# yyyymmddcc format; required -# cycle2 second cycle processing date -# yyyymmddcc format; required -# diag_rpt diagnostic report text file -# required -# outfile output file name -# required -# -# Remarks: -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -#################################################################### - -echo "--> radmon_err_rpt.sh" - -# Command line arguments. -file1=${1:-${file1:?}} -file2=${2:-${file2:?}} -type=${3:-${type:?}} -cycle1=${4:-${cycle1:?}} -cycle2=${5:-${cycle2:?}} -diag_rpt=${6:-${diag_rpt:?}} -outfile=${7:-${outfile:?}} - -# Directories -HOMEradmon=${HOMEradmon:-$(pwd)} - -# Other variables -VERBOSE=${VERBOSE:-NO} -err=0 -RADMON_SUFFIX=${RADMON_SUFFIX} - -if [[ "$VERBOSE" = "YES" ]]; then - echo EXECUTING $0 $* >&2 - set -ax -fi - - -have_diag_rpt=0 -if [[ -s $diag_rpt ]]; then - have_diag_rpt=1 -else - err=1 -fi -echo "have_diag_rpt = $have_diag_rpt" - -#----------------------------------------------------------------------------- -# read each line in the $file1 -# search $file2 for the same satname, channel, and region -# if same combination is in both files, add the values to the output file -# -{ while read myline; do - echo "myline = $myline" - bound="" - - echo $myline - satname=`echo $myline | gawk '{print $1}'` - channel=`echo $myline | gawk '{print $3}'` - region=`echo $myline | gawk '{print $5}'` - value1=`echo $myline | gawk '{print $7}'` - bound=`echo $myline | gawk '{print $9}'` - -# -# Check findings against diag_report. If the satellite/instrument is on the -# diagnostic report it means the diagnostic file file for the -# satelite/instrument is missing for this cycle, so skip any additional -# error checking for that source. Otherwise, evaluate as per normal. -# - - diag_match="" - diag_match_len=0 - - if [[ $have_diag_rpt == 1 ]]; then - diag_match=`gawk "/$satname/" $diag_rpt` - diag_match_len=`echo ${#diag_match}` - fi - - - if [[ $diag_match_len == 0 ]]; then - - if [[ $type == "chan" ]]; then - echo "looking for match for $satname and $channel" - { while read myline2; do - satname2=`echo $myline2 | gawk '{print $1}'` - channel2=`echo $myline2 | gawk '{print $3}'` - - if [[ $satname == $satname2 && $channel == $channel2 ]]; then - match="$satname channel= $channel" - echo "match from gawk = $match" - break; - else - match="" - fi - - done } < $file2 - - - else - match=`gawk "/$satname/ && /channel= $channel / && /region= $region /" $file2` - echo match = $match - - match_len=`echo ${#match}` - if [[ $match_len > 0 ]]; then - channel2=`echo $match | gawk '{print $3}'` - - if [[ $channel2 != $channel ]]; then - match="" - fi - fi - - fi - match_len=`echo ${#match}` - - if [[ $match_len > 0 ]]; then - - value2=`echo $match | gawk '{print $7}'` - bound2=`echo $match | gawk '{print $9}'` - - if [[ $type == "chan" ]]; then - tmpa=" $satname channel= $channel" - tmpb="" - - elif [[ $type == "pen" ]]; then - tmpa="$satname channel= $channel region= $region" - tmpb="$cycle1 $value1 $bound" - - elif [[ $type == "cnt" ]]; then - tmpa="$satname channel= $channel region= $region" - tmpb="$cycle1 $value1 $bound" - - else - tmpa="$satname channel= $channel region= $region" - tmpb="$cycle1: $type= $value1" - fi - - line1="$tmpa $tmpb" - echo "$line1" >> $outfile - - if [[ $type != "chan" ]]; then - tmpc=`echo $tmpa |sed 's/[a-z]/ /g' | sed 's/[0-9]/ /g' | sed 's/=/ /g' | sed 's/_/ /g' | sed 's/-/ /g'` - - if [[ $type == "pen" || $type == "cnt" ]]; then - line2=" $tmpc $cycle2 $value2 $bound2" - else - line2=" $tmpc $cycle2: $type= $value2" - fi - - echo "$line2" >> $outfile - fi - - #----------------------------------------- - # add hyperlink to warning entry - # - line3=" http://www.emc.ncep.noaa.gov/gmb/gdas/radiance/es_rad/${RADMON_SUFFIX}/index.html?sat=${satname}®ion=${region}&channel=${channel}&stat=${type}" - if [[ $channel -gt 0 ]]; then - echo "$line3" >> $outfile - echo "" >> $outfile - fi - fi - fi -done } < $file1 - - -################################################################################ -# Post processing -if [[ "$VERBOSE" = "YES" ]]; then - echo $(date) EXITING $0 with error code ${err} >&2 -fi - -echo "<-- radmon_err_rpt.sh" - -set +x -exit ${err} - diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh deleted file mode 100755 index b67aea21..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_angle.sh +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/ksh - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: radmon_verf_angle.sh -# Script description: Extract angle dependent data from radiance -# diagnostic files. -# -# Author: Ed Safford Org: NP23 Date: 2012-02-02 -# -# Abstract: This script extracts angle dependent data from radiance -# diagnostic files (which are an output from GSI runs), -# storing the extracted data in small binary files. -# -# This script is a child script of exgdas_vrfyrad.sh.sms. The parent -# script opens and uncompresses the radiance diagnostic file and copies -# other supporting files into a temporary working directory. -# -# -# Usage: radmon_verf_angle.sh PDATE -# -# Input script positional parameters: -# PDATE processing date -# yyyymmddcc format; required -# -# Imported Shell Variables: -# RADMON_SUFFIX data source suffix -# defauls to opr -# EXECradmon executable directory -# defaults to current directory -# RAD_AREA global or regional flag -# defaults to global -# TANKverf_rad data repository -# defaults to current directory -# SATYPE list of satellite/instrument sources -# defaults to none -# VERBOSE Verbose flag (YES or NO) -# defaults to NO -# LITTLE_ENDIAN flag to indicate LE machine -# defaults to 0 (big endian) -# USE_ANL use analysis files as inputs in addition to -# the ges files. Default is 0 (ges only) -# -# Modules and files referenced: -# scripts : -# -# programs : $NCP -# $angle_exec -# -# fixed data : $scaninfo -# -# input data : $data_file -# -# output data: $angle_file -# $angle_ctl -# $pgmout -# -# Remarks: -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -#################################################################### - -# Command line arguments. -RAD_AREA=${RAD_AREA:-glb} -REGIONAL_RR=${REGIONAL_RR:-0} # rapid refresh model flag -rgnHH=${rgnHH:-} -rgnTM=${rgnTM:-} - -export PDATE=${1:-${PDATE:?}} - -echo " REGIONAL_RR, rgnHH, rgnTM = $REGIONAL_RR, $rgnHH, $rgnTM" -netcdf_boolean=".false." -if [[ $RADMON_NETCDF -eq 1 ]]; then - netcdf_boolean=".true." -fi -echo " RADMON_NETCDF, netcdf_boolean = ${RADMON_NETCDF}, $netcdf_boolean" - -which prep_step -which startmsg - -if [[ "$VERBOSE" = "YES" ]]; then - set -ax -fi - -# Directories -FIXgdas=${FIXgdas:-$(pwd)} -EXECradmon=${EXECradmon:-$(pwd)} -TANKverf_rad=${TANKverf_rad:-$(pwd)} - -# File names -export pgmout=${pgmout:-${jlogfile}} -touch $pgmout - -# Other variables -SATYPE=${SATYPE:-} -VERBOSE=${VERBOSE:-NO} -LITTLE_ENDIAN=${LITTLE_ENDIAN:-0} -USE_ANL=${USE_ANL:-0} - - -if [[ $USE_ANL -eq 1 ]]; then - gesanl="ges anl" -else - gesanl="ges" -fi - -err=0 -angle_exec=radmon_angle.x -shared_scaninfo=${shared_scaninfo:-$FIXgdas/gdas_radmon_scaninfo.txt} -scaninfo=scaninfo.txt - -#-------------------------------------------------------------------- -# Copy extraction program and supporting files to working directory - -$NCP ${EXECradmon}/${angle_exec} ./ -$NCP $shared_scaninfo ./${scaninfo} - -if [[ ! -s ./${angle_exec} || ! -s ./${scaninfo} ]]; then - err=2 -else -#-------------------------------------------------------------------- -# Run program for given time - - export pgm=${angle_exec} - - iyy=`echo $PDATE | cut -c1-4` - imm=`echo $PDATE | cut -c5-6` - idd=`echo $PDATE | cut -c7-8` - ihh=`echo $PDATE | cut -c9-10` - - ctr=0 - fail=0 - touch "./errfile" - - for type in ${SATYPE}; do - - if [[ ! -s ${type} ]]; then - echo "ZERO SIZED: ${type}" - continue - fi - - for dtype in ${gesanl}; do - - echo "pgm = $pgm" - echo "pgmout = $pgmout" - prep_step - - ctr=`expr $ctr + 1` - - if [[ $dtype == "anl" ]]; then - data_file=${type}_anl.${PDATE}.ieee_d - ctl_file=${type}_anl.ctl - angl_ctl=angle.${ctl_file} - else - data_file=${type}.${PDATE}.ieee_d - ctl_file=${type}.ctl - angl_ctl=angle.${ctl_file} - fi - - if [[ $REGIONAL_RR -eq 1 ]]; then - angl_file=${rgnHH}.${data_file}.${rgnTM} - fi - - - rm input - - nchanl=-999 -cat << EOF > input - &INPUT - satname='${type}', - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-720, - incr=${CYCLE_INTERVAL}, - nchanl=${nchanl}, - suffix='${RADMON_SUFFIX}', - gesanl='${dtype}', - little_endian=${LITTLE_ENDIAN}, - rad_area='${RAD_AREA}', - netcdf=${netcdf_boolean}, - / -EOF - - startmsg - ./${angle_exec} < input >> ${pgmout} 2>>errfile - export err=$?; err_chk - if [[ $err -ne 0 ]]; then - fail=`expr $fail + 1` - fi - -#------------------------------------------------------------------- -# move data, control, and stdout files to $TANKverf_rad and compress - - #----------------------------------------------------------- - # For cfp use, instead of executing these file manipulation - # comands directly, journel them instead to a flat file. - # Then execute cfp (as appropriate for each machine) to - # perform the file manipulations. Those manipulation - # commands should probably go in a parm file so I can not - # rely on a machine dependent decision here. - # - # Note that in my esafford_RadMon_45526 branch I've already - # modified the order of operation to compress before copy. - # That change should be delivered to trunk soon. - #----------------------------------------------------------- - - - if [[ -s ${angl_file} ]]; then - ${COMPRESS} -f ${angl_file} - fi - - if [[ -s ${angl_ctl} ]]; then - ${COMPRESS} -f ${angl_ctl} - fi - - - done # for dtype in ${gesanl} loop - - done # for type in ${SATYPE} loop - - - ${USHradmon}/rstprod.sh - - tar_file=radmon_angle.tar - tar -cf $tar_file angle*.ieee_d* angle*.ctl* - ${COMPRESS} ${tar_file} - mv $tar_file.${Z} ${TANKverf_rad}/. - - if [[ $RAD_AREA = "rgn" ]]; then - cwd=`pwd` - cd ${TANKverf_rad} - tar -xf ${tar_file}.${Z} - rm ${tar_file}.${Z} - cd ${cwd} - fi - - if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then - err=3 - fi -fi - -################################################################################ -# Post processing - -if [[ "$VERBOSE" = "YES" ]]; then - echo $(date) EXITING $0 error code ${err} >&2 -fi - - -echo "<-- radmon_verf_angle.sh" -exit ${err} diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh deleted file mode 100755 index 20cbee45..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcoef.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/ksh - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: radmon_verf_bcoef.sh -# Script description: Extract bias correction coefficients data from radiance -# diagnostic files. -# -# Author: Ed Safford Org: NP23 Date: 2012-02-02 -# -# Abstract: This script extracts bias correction coefficient related data from -# radiance diagnostic files (which are an output from GSI runs), -# storing the extracted data in small binary files. -# -# This script is a child script of exgdas_vrfyrad.sh.sms. The parent -# script opens and uncompresses the radiance diagnostic file and copies -# other supporting files into a temporary working directory. -# -# -# Usage: radmon_verf_bcoef.sh PDATE -# -# Input script positional parameters: -# PDATE processing date -# yyyymmddcc format; required -# -# Imported Shell Variables: -# RADMON_SUFFIX data source suffix -# defauls to opr -# EXECradmon executable directory -# defaults to current directory -# FIXradmon fixed data directory -# defaults to current directory -# RAD_AREA global or regional flag -# defaults to global -# TANKverf_rad data repository -# defaults to current directory -# SATYPE list of satellite/instrument sources -# defaults to none -# VERBOSE Verbose flag (YES or NO) -# defaults to NO -# LITTLE_ENDIAN flag for LE machine -# defaults to 0 (big endian) -# USE_ANL use analysis files as inputs in addition to -# the ges files. Default is 0 (ges only) -# -# Modules and files referenced: -# scripts : -# -# programs : $NCP -# $bcoef_exec -# -# fixed data : $biascr -# -# input data : $data_file -# -# output data: $bcoef_file -# $bcoef_ctl -# $pgmout -# -# Remarks: -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -#################################################################### -# Command line arguments. -export PDATE=${1:-${PDATE:?}} - -netcdf_boolean=".false." -if [[ $RADMON_NETCDF -eq 1 ]]; then - netcdf_boolean=".true." -fi -echo " RADMON_NETCDF, netcdf_boolean = ${RADMON_NETCDF}, $netcdf_boolean" - -if [[ "$VERBOSE" = "YES" ]]; then - set -ax -fi - - -# Directories -FIXgdas=${FIXgdas:-$(pwd)} -EXECradmon=${EXECradmon:-$(pwd)} -TANKverf_rad=${TANKverf_rad:-$(pwd)} - -# File names -pgmout=${pgmout:-${jlogfile}} -touch $pgmout - -# Other variables -RAD_AREA=${RAD_AREA:-glb} -REGIONAL_RR=${REGIONAL_RR:-0} -rgnHH=${rgnHH:-} -rgnTM=${rgnTM:-} -SATYPE=${SATYPE:-} -VERBOSE=${VERBOSE:-NO} -LITTLE_ENDIAN=${LITTLE_ENDIAN:-0} -USE_ANL=${USE_ANL:-0} - - -err=0 -bcoef_exec=radmon_bcoef.x - -if [[ $USE_ANL -eq 1 ]]; then - gesanl="ges anl" -else - gesanl="ges" -fi - -#-------------------------------------------------------------------- -# Copy extraction program and supporting files to working directory - -$NCP $EXECradmon/${bcoef_exec} ./${bcoef_exec} -$NCP ${biascr} ./biascr.txt - -if [[ ! -s ./${bcoef_exec} || ! -s ./biascr.txt ]]; then - err=4 -else - - -#-------------------------------------------------------------------- -# Run program for given time - - export pgm=${bcoef_exec} - - iyy=`echo $PDATE | cut -c1-4` - imm=`echo $PDATE | cut -c5-6` - idd=`echo $PDATE | cut -c7-8` - ihh=`echo $PDATE | cut -c9-10` - - ctr=0 - fail=0 - - nchanl=-999 - npredr=5 - - for type in ${SATYPE}; do - - if [[ ! -s ${type} ]]; then - echo "ZERO SIZED: ${type}" - continue - fi - - for dtype in ${gesanl}; do - - prep_step - - ctr=`expr $ctr + 1` - - if [[ $dtype == "anl" ]]; then - data_file=${type}_anl.${PDATE}.ieee_d - ctl_file=${type}_anl.ctl - bcoef_ctl=bcoef.${ctl_file} - else - data_file=${type}.${PDATE}.ieee_d - ctl_file=${type}.ctl - bcoef_ctl=bcoef.${ctl_file} - fi - - if [[ $REGIONAL_RR -eq 1 ]]; then - bcoef_file=${rgnHH}.bcoef.${data_file}.${rgnTM} - else - bcoef_file=bcoef.${data_file} - fi - - - rm input - - -cat << EOF > input - &INPUT - satname='${type}', - npredr=${npredr}, - nchanl=${nchanl}, - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-720, - incr=${CYCLE_INTERVAL}, - suffix='${RADMON_SUFFIX}', - gesanl='${dtype}', - little_endian=${LITTLE_ENDIAN}, - netcdf=${netcdf_boolean}, - / -EOF - startmsg - ./${bcoef_exec} < input >>${pgmout} 2>>errfile - export err=$?; err_chk - if [[ $err -ne 0 ]]; then - fail=`expr $fail + 1` - fi - - -#------------------------------------------------------------------- -# move data, control, and stdout files to $TANKverf_rad and compress -# - - if [[ -s ${bcoef_file} ]]; then - ${COMPRESS} ${bcoef_file} - fi - - if [[ -s ${bcoef_ctl} ]]; then - ${COMPRESS} ${bcoef_ctl} - fi - - - done # dtype in $gesanl loop - done # type in $SATYPE loop - - - ${USHradmon}/rstprod.sh - - tar_file=radmon_bcoef.tar - tar -cf $tar_file bcoef*.ieee_d* bcoef*.ctl* - ${COMPRESS} ${tar_file} - mv $tar_file.${Z} ${TANKverf_rad} - - if [[ $RAD_AREA = "rgn" ]]; then - cwd=`pwd` - cd ${TANKverf_rad} - tar -xf ${tar_file}.${Z} - rm ${tar_file}.${Z} - cd ${cwd} - fi - - if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then - err=5 - fi -fi - - -################################################################################ -# Post processing -if [[ "$VERBOSE" = "YES" ]]; then - echo $(date) EXITING $0 with error code ${err} >&2 -fi - - -exit ${err} diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh deleted file mode 100755 index ecf3e4a3..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_bcor.sh +++ /dev/null @@ -1,234 +0,0 @@ -#!/bin/ksh - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: radmon_verf_bcor.sh -# Script description: Extract bias correction data from radiance diagnostic -# files. -# -# Author: Ed Safford Org: NP23 Date: 2012-02-02 -# -# Abstract: This script extracts bias correction related data from radiance -# diagnostic files (which are an output from GSI runs), storing the -# extracted data in small binary files. -# -# This script is a child script of exgdas_vrfyrad.sh.sms. The parent -# script opens and uncompresses the radiance diagnostic file and copies -# other supporting files into a temporary working directory. -# -# -# Usage: radmon_verf_bcor.sh PDATE -# -# Input script positional parameters: -# PDATE processing date -# yyyymmddcc format; required -# -# Imported Shell Variables: -# RADMON_SUFFIX data source suffix -# defauls to opr -# EXECradmon executable directory -# defaults to current directory -# RAD_AREA global or regional flag -# defaults to global -# TANKverf_rad data repository -# defaults to current directory -# SATYPE list of satellite/instrument sources -# defaults to none -# VERBOSE Verbose flag (YES or NO) -# defaults to NO -# LITTLE_ENDIAN flag for little endian machine -# defaults to 0 (big endian) -# USE_ANL use analysis files as inputs in addition to -# the ges files. Default is 0 (ges only) -# -# Modules and files referenced: -# scripts : -# -# programs : $NCP -# $bcor_exec -# -# fixed data : none -# -# input data : $data_file -# -# output data: $bcor_file -# $bcor_ctl -# $pgmout -# -# Remarks: -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -#################################################################### - -# Command line arguments. -export PDATE=${1:-${PDATE:?}} - - -if [[ "$VERBOSE" = "YES" ]]; then - set -ax -fi - -# Directories -EXECradmon=${EXECradmon:-$(pwd)} -TANKverf_rad=${TANKverf_rad:-$(pwd)} - -# File names -pgmout=${pgmout:-${jlogfile}} -touch $pgmout - -# Other variables -RAD_AREA=${RAD_AREA:-glb} -SATYPE=${SATYPE:-} -VERBOSE=${VERBOSE:-NO} -LITTLE_ENDIAN=${LITTLE_ENDIAN:-0} -USE_ANL=${USE_ANL:-0} - -bcor_exec=radmon_bcor.x -err=0 - -netcdf_boolean=".false." -if [[ $RADMON_NETCDF -eq 1 ]]; then - netcdf_boolean=".true." -fi - -if [[ $USE_ANL -eq 1 ]]; then - gesanl="ges anl" -else - gesanl="ges" -fi - - -#-------------------------------------------------------------------- -# Copy extraction program to working directory - -$NCP ${EXECradmon}/${bcor_exec} ./${bcor_exec} - -if [[ ! -s ./${bcor_exec} ]]; then - err=6 -else - - -#-------------------------------------------------------------------- -# Run program for given time - - export pgm=${bcor_exec} - - iyy=`echo $PDATE | cut -c1-4` - imm=`echo $PDATE | cut -c5-6` - idd=`echo $PDATE | cut -c7-8` - ihh=`echo $PDATE | cut -c9-10` - - ctr=0 - fail=0 - touch "./errfile" - - for type in ${SATYPE}; do - - for dtype in ${gesanl}; do - - prep_step - - ctr=`expr $ctr + 1` - - if [[ $dtype == "anl" ]]; then - data_file=${type}_anl.${PDATE}.ieee_d - bcor_file=bcor.${data_file} - ctl_file=${type}_anl.ctl - bcor_ctl=bcor.${ctl_file} - stdout_file=stdout.${type}_anl - bcor_stdout=bcor.${stdout_file} - input_file=${type}_anl - else - data_file=${type}.${PDATE}.ieee_d - bcor_file=bcor.${data_file} - ctl_file=${type}.ctl - bcor_ctl=bcor.${ctl_file} - stdout_file=stdout.${type} - bcor_stdout=bcor.${stdout_file} - input_file=${type} - fi - - rm input - - # Check for 0 length input file here and avoid running - # the executable if $input_file doesn't exist or is 0 bytes - # - if [[ -s $input_file ]]; then - nchanl=-999 - -cat << EOF > input - &INPUT - satname='${type}', - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-720, - incr=6, - nchanl=${nchanl}, - suffix='${RADMON_SUFFIX}', - gesanl='${dtype}', - little_endian=${LITTLE_ENDIAN}, - rad_area='${RAD_AREA}', - netcdf=${netcdf_boolean}, - / -EOF - - startmsg - ./${bcor_exec} < input >> ${pgmout} 2>>errfile - export err=$?; err_chk - if [[ $? -ne 0 ]]; then - fail=`expr $fail + 1` - fi - - -#------------------------------------------------------------------- -# move data, control, and stdout files to $TANKverf_rad and compress -# - - if [[ -s ${bcor_file} ]]; then - ${COMPRESS} ${bcor_file} - fi - - if [[ -s ${bcor_ctl} ]]; then - ${COMPRESS} ${bcor_ctl} - fi - - fi - done # dtype in $gesanl loop - done # type in $SATYPE loop - - - ${USHradmon}/rstprod.sh - tar_file=radmon_bcor.tar - - tar -cf $tar_file bcor*.ieee_d* bcor*.ctl* - ${COMPRESS} ${tar_file} - mv $tar_file.${Z} ${TANKverf_rad}/. - - if [[ $RAD_AREA = "rgn" ]]; then - cwd=`pwd` - cd ${TANKverf_rad} - tar -xf ${tar_file}.${Z} - rm ${tar_file}.${Z} - cd ${cwd} - fi - - if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then - err=7 - fi -fi - -################################################################################ -# Post processing - -if [[ "$VERBOSE" = "YES" ]]; then - echo $(date) EXITING $0 error code ${err} >&2 -fi - -exit ${err} - diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh deleted file mode 100755 index 6dbb8070..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/radmon_verf_time.sh +++ /dev/null @@ -1,575 +0,0 @@ -#!/bin/ksh - -################################################################################ -#### UNIX Script Documentation Block -# . . -# Script name: radmon_verf_time.sh -# Script description: Extract time data from radiance diagnostic files, -# perform data integrity checks. -# -# Author: Ed Safford Org: NP23 Date: 2012-02-02 -# -# Abstract: This script extracts time related data from radiance diagnostic -# files (which are an output from GSI runs), storing the extracted -# data in small binary files. Data integrity checks are performed -# on the data and mail messages are sent if potential errors are -# detected. -# -# This script is a child script of exgdas_vrfyrad.sh.sms. The parent -# script opens and uncompresses the radiance diagnostic file and copies -# other supporting files into a temporary working directory. -# -# -# Usage: radmon_verf_time.sh PDATE -# -# Input script positional parameters: -# PDATE processing date -# yyyymmddcc format; required -# -# Imported Shell Variables: -# DO_DATA_RPT switch to build the data report -# defaults to 1 (on) -# RADMON_SUFFIX data source suffix -# defauls to opr -# EXECradmon executable directory -# defaults to current directory -# FIXgdas fixed data directory -# defaults to current directory -# RAD_AREA global or regional flag -# defaults to global -# TANKverf_rad data repository -# defaults to current directory -# SATYPE list of satellite/instrument sources -# defaults to none -# MAIL_TO email recipients -# defaults to none -# MAIL_CC email cc recipients -# defaults to none -# VERBOSE Verbose flag (YES or NO) -# defaults to NO -# LITTLE_ENDIAN flag for little endian machine -# defaults to 0 (big endian) -# USE_ANL use analysis files as inputs in addition to -# the ges files. Default is 0 (ges only) -# -# Modules and files referenced: -# scripts : -# -# programs : $NCP -# $time_exec -# -# fixed data : gdas_radmon_base.tar -# -# input data : $data_file -# -# output data: $time_file -# $time_ctl -# $pgmout -# $bad_pen -# $bad_chan -# $report -# $diag_report -# -# -# Remarks: -# -# Condition codes -# 0 - no problem encountered -# >0 - some problem encountered -# -#################################################################### - -# Command line arguments. -export PDATE=${1:-${PDATE:?}} - - -if [[ "$VERBOSE" = "YES" ]]; then - set -ax -fi - -# Directories -FIXgdas=${FIXgdas:-$(pwd)} -EXECradmon=${EXECradmon:-$(pwd)} -TANKverf_rad=${TANKverf_rad:-$(pwd)} - -# File names -#pgmout=${pgmout:-${jlogfile}} -#touch $pgmout - -radmon_err_rpt=${radmon_err_rpt:-${USHradmon}/radmon_err_rpt.sh} -base_file=${base_file:-$FIXgdas/gdas_radmon_base.tar} -report=report.txt -disclaimer=disclaimer.txt - -diag_report=diag_report.txt -diag_hdr=diag_hdr.txt -diag=diag.txt - -obs_err=obs_err.txt -obs_hdr=obs_hdr.txt -pen_err=pen_err.txt -pen_hdr=pen_hdr.txt - -chan_err=chan_err.txt -chan_hdr=chan_hdr.txt -count_hdr=count_hdr.txt -count_err=count_err.txt - -netcdf_boolean=".false." -if [[ $RADMON_NETCDF -eq 1 ]]; then - netcdf_boolean=".true." -fi - -DO_DATA_RPT=${DO_DATA_RPT:-1} -RADMON_SUFFIX=${RADMON_SUFFIX:-opr} -RAD_AREA=${RAD_AREA:-glb} -REGIONAL_RR=${REGIONAL_RR:-0} -rgnHH=${rgnHH:-} -rgnTM=${rgnTM:-} -SATYPE=${SATYPE:-} -MAIL_TO=${MAIL_TO:-} -MAIL_CC=${MAIL_CC:-} -VERBOSE=${VERBOSE:-NO} -LITTLE_ENDIAN=${LITTLE_ENDIAN:-0} -USE_MAIL=${USE_MAIL:-1} - -time_exec=radmon_time.x -USE_ANL=${USE_ANL:-0} -err=0 - -if [[ $USE_ANL -eq 1 ]]; then - gesanl="ges anl" -else - gesanl="ges" -fi - - -#-------------------------------------------------------------------- -# Copy extraction program and base files to working directory -#------------------------------------------------------------------- -$NCP ${EXECradmon}/${time_exec} ./ -if [[ ! -s ./${time_exec} ]]; then - err=8 -fi - -iyy=`echo $PDATE | cut -c1-4` -imm=`echo $PDATE | cut -c5-6` -idd=`echo $PDATE | cut -c7-8` -ihh=`echo $PDATE | cut -c9-10` -cyc=$ihh -CYCLE=$cyc - -local_base="local_base" -if [[ $DO_DATA_RPT -eq 1 ]]; then - - if [[ -e ${base_file}.${Z} ]]; then - $NCP ${base_file}.${Z} ./${local_base}.{Z} - ${UNCOMPRESS} ${local_base}.${Z} - else - $NCP ${base_file} ./${local_base} - fi - - if [[ ! -s ./${local_base} ]]; then - echo "RED LIGHT: local_base file not found" - else - echo "Confirming local_base file is good = ${local_base}" - tar -xf ./${local_base} - echo "local_base is untarred" - fi -fi - -if [[ $err -eq 0 ]]; then - ctr=0 - fail=0 - - export pgm=${time_exec} -#-------------------------------------------------------------------- -# Loop over each entry in SATYPE -#-------------------------------------------------------------------- - for type in ${SATYPE}; do - - if [[ ! -s ${type} ]]; then - echo "ZERO SIZED: ${type}" - continue - fi - - ctr=`expr $ctr + 1` - - for dtype in ${gesanl}; do - - rm input - - if [[ $dtype == "anl" ]]; then - data_file=${type}_anl.${PDATE}.ieee_d - ctl_file=${type}_anl.ctl - time_ctl=time.${ctl_file} - else - data_file=${type}.${PDATE}.ieee_d - ctl_file=${type}.ctl - time_ctl=time.${ctl_file} - fi - - if [[ $REGIONAL_RR -eq 1 ]]; then - time_file=${rgnHH}.time.${data_file}.${rgnTM} - else - time_file=time.${data_file} - fi - -#-------------------------------------------------------------------- -# Run program for given satellite/instrument -#-------------------------------------------------------------------- - nchanl=-999 -cat << EOF > input - &INPUT - satname='${type}', - iyy=${iyy}, - imm=${imm}, - idd=${idd}, - ihh=${ihh}, - idhh=-720, - incr=${CYCLE_INTERVAL}, - nchanl=${nchanl}, - suffix='${RADMON_SUFFIX}', - gesanl='${dtype}', - little_endian=${LITTLE_ENDIAN}, - rad_area='${RAD_AREA}', - netcdf=${netcdf_boolean}, - / -EOF - - ./${time_exec} < input >> stdout.${type} 2>>errfile - - if [[ $err -ne 0 ]]; then - fail=`expr $fail + 1` - fi - -#------------------------------------------------------------------- -# move data, control, and stdout files to $TANKverf_rad and compress -#------------------------------------------------------------------- - cat stdout.${type} >> stdout.time - - if [[ -s ${time_file} ]]; then - ${COMPRESS} ${time_file} - fi - - if [[ -s ${time_ctl} ]]; then - ${COMPRESS} ${time_ctl} - fi - - done - done - - - ${USHradmon}/rstprod.sh - - tar_file=radmon_time.tar - tar -cf $tar_file time*.ieee_d* time*.ctl* - ${COMPRESS} ${tar_file} - mv $tar_file.${Z} ${TANKverf_rad}/. - - if [[ $RAD_AREA = "rgn" ]]; then - cwd=`pwd` - cd ${TANKverf_rad} - tar -xf ${tar_file}.${Z} - rm ${tar_file}.${Z} - cd ${cwd} - fi - - if [[ $fail -eq $ctr || $fail -gt $ctr ]]; then - echo "fail, ctr = $fail, $ctr" - err=10 - fi - -fi - - - -#################################################################### -#------------------------------------------------------------------- -# Begin error analysis and reporting -#------------------------------------------------------------------- -#################################################################### - -if [[ $DO_DATA_RPT -eq 1 ]]; then - -#--------------------------- -# build report disclaimer -# - cat << EOF > ${disclaimer} - - -*********************** WARNING *************************** -THIS IS AN AUTOMATED EMAIL. REPLIES TO SENDER WILL NOT BE -RECEIVED. PLEASE DIRECT REPLIES TO edward.safford@noaa.gov -*********************** WARNING *************************** -EOF - - -#------------------------------------------------------------------- -# Check for missing diag files -# - tmp_satype="./tmp_satype.txt" - echo ${SATYPE} > ${tmp_satype} - ${USHradmon}/radmon_diag_ck.sh --rad ${radstat} --sat ${tmp_satype} --out ${diag} - - if [[ -s ${diag} ]]; then - cat << EOF > ${diag_hdr} - - Problem Reading Diagnostic File - - - Problems were encountered reading the diagnostic file for - the following sources: - -EOF - - cat ${diag_hdr} >> ${diag_report} - cat ${diag} >> ${diag_report} - - echo >> ${diag_report} - - rm ${diag_hdr} - fi - -#------------------------------------------------------------------- -# move warning notification to TANKverf -# - if [[ -s ${diag} ]]; then - lines=`wc -l <${diag}` - echo "lines in diag = $lines" - - if [[ $lines -gt 0 ]]; then - cat ${diag_report} - cp ${diag} ${TANKverf_rad}/bad_diag.${PDATE} - else - rm ${diag_report} - fi - fi - - - - #---------------------------------------------------------------- - # Identify bad_pen and bad_chan files for this cycle and - # previous cycle - - bad_pen=bad_pen.${PDATE} - bad_chan=bad_chan.${PDATE} - low_count=low_count.${PDATE} - - qdate=`$NDATE -${CYCLE_INTERVAL} $PDATE` - pday=`echo $qdate | cut -c1-8` - - prev_bad_pen=bad_pen.${qdate} - prev_bad_chan=bad_chan.${qdate} - prev_low_count=low_count.${qdate} - - prev_bad_pen=${TANKverf_radM1}/${prev_bad_pen} - prev_bad_chan=${TANKverf_radM1}/${prev_bad_chan} - prev_low_count=${TANKverf_radM1}/${prev_low_count} - - if [[ -s $bad_pen ]]; then - echo "pad_pen = $bad_pen" - fi - if [[ -s $prev_bad_pen ]]; then - echo "prev_pad_pen = $prev_bad_pen" - fi - - if [[ -s $bad_chan ]]; then - echo "bad_chan = $bad_chan" - fi - if [[ -s $prev_bad_chan ]]; then - echo "prev_bad_chan = $prev_bad_chan" - fi - if [[ -s $low_count ]]; then - echo "low_count = $low_count" - fi - if [[ -s $prev_low_count ]]; then - echo "prev_low_count = $prev_low_count" - fi - - do_pen=0 - do_chan=0 - do_cnt=0 - - if [[ -s $bad_pen && -s $prev_bad_pen ]]; then - do_pen=1 - fi - - if [[ -s $low_count && -s $prev_low_count ]]; then - do_cnt=1 - fi - - #-------------------------------------------------------------------- - # avoid doing the bad_chan report for REGIONAL_RR sources -- because - # they run hourly they often have 0 count channels for off-hour runs. - # - if [[ -s $bad_chan && -s $prev_bad_chan && REGIONAL_RR -eq 0 ]]; then - do_chan=1 - fi - - #-------------------------------------------------------------------- - # Remove extra spaces in new bad_pen & low_count files - # - gawk '{$1=$1}1' $bad_pen > tmp.bad_pen - mv -f tmp.bad_pen $bad_pen - - gawk '{$1=$1}1' $low_count > tmp.low_count - mv -f tmp.low_count $low_count - - echo " do_pen, do_chan, do_cnt = $do_pen, $do_chan, $do_cnt" - echo " diag_report = $diag_report " - if [[ $do_pen -eq 1 || $do_chan -eq 1 || $do_cnt -eq 1 || -s ${diag_report} ]]; then - - if [[ $do_pen -eq 1 ]]; then - - echo "calling radmon_err_rpt for pen" - ${radmon_err_rpt} ${prev_bad_pen} ${bad_pen} pen ${qdate} \ - ${PDATE} ${diag_report} ${pen_err} - fi - - if [[ $do_chan -eq 1 ]]; then - - echo "calling radmon_err_rpt for chan" - ${radmon_err_rpt} ${prev_bad_chan} ${bad_chan} chan ${qdate} \ - ${PDATE} ${diag_report} ${chan_err} - fi - - if [[ $do_cnt -eq 1 ]]; then - - echo "calling radmon_err_rpt for cnt" - ${radmon_err_rpt} ${prev_low_count} ${low_count} cnt ${qdate} \ - ${PDATE} ${diag_report} ${count_err} - fi - - #------------------------------------------------------------------- - # put together the unified error report with any obs, chan, and - # penalty problems and mail it - - if [[ -s ${obs_err} || -s ${pen_err} || -s ${chan_err} || -s ${count_err} || -s ${diag_report} ]]; then - - echo DOING ERROR REPORTING - - - cat << EOF > $report -Radiance Monitor warning report - - Net: ${RADMON_SUFFIX} - Run: ${RUN} - Cycle: $PDATE - -EOF - - if [[ -s ${diag_report} ]]; then - echo OUTPUTING DIAG_REPORT - cat ${diag_report} >> $report - fi - - if [[ -s ${chan_err} ]]; then - - echo OUTPUTING CHAN_ERR - - cat << EOF > ${chan_hdr} - - The following channels report 0 observational counts over the past two cycles: - - Satellite/Instrument Channel - ==================== ======= - -EOF - - cat ${chan_hdr} >> $report - cat ${chan_err} >> $report - - fi - - if [[ -s ${count_err} ]]; then - - cat << EOF > ${count_hdr} - - - - The following channels report abnormally low observational counts in the latest 2 cycles: - -Satellite/Instrument Obs Count Avg Count -==================== ========= ========= - -EOF - - cat ${count_hdr} >> $report - cat ${count_err} >> $report - fi - - - if [[ -s ${pen_err} ]]; then - - cat << EOF > ${pen_hdr} - - - Penalty values outside of the established normal range were found - for these sensor/channel/regions in the past two cycles: - - Questionable Penalty Values - ============ ======= ====== Cycle Penalty Bound - ----- ------- ----- -EOF - cat ${pen_hdr} >> $report - cat ${pen_err} >> $report - rm -f ${pen_hdr} - rm -f ${pen_err} - fi - - echo >> $report - cat ${disclaimer} >> $report - echo >> $report - fi - - #------------------------------------------------------------------- - # dump report to log file - # - if [[ -s ${report} ]]; then - lines=`wc -l <${report}` - if [[ $lines -gt 2 ]]; then - cat ${report} - - $NCP ${report} ${TANKverf_rad}/warning.${PDATE} - fi - fi - - - fi - - #------------------------------------------------------------------- - # copy new bad_pen, bad_chan, and low_count files to $TANKverf_rad - # - if [[ -s ${bad_chan} ]]; then - mv ${bad_chan} ${TANKverf_rad}/. - fi - - if [[ -s ${bad_pen} ]]; then - mv ${bad_pen} ${TANKverf_rad}/. - fi - - if [[ -s ${low_count} ]]; then - mv ${low_count} ${TANKverf_rad}/. - fi - - -fi - - for type in ${SATYPE}; do - rm -f stdout.${type} - done - - ################################################################################ - #------------------------------------------------------------------- - # end error reporting section - #------------------------------------------------------------------- - ################################################################################ - - ################################################################################ - # Post processing - if [[ "$VERBOSE" = "YES" ]]; then - echo $(date) EXITING $0 error code ${err} >&2 - fi - -exit ${err} diff --git a/src/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh b/src/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh deleted file mode 100755 index dd5a09ec..00000000 --- a/src/Radiance_Monitor/nwprod/radmon_shared/ush/rstprod.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -#--------------------------------------------------------- -# rstprod.sh -# -# Restrict data from select sensors and satellites -#--------------------------------------------------------- - -# Restrict select sensors and satellites -echo "--> rstprod.sh" - -export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} -rlist="saphir abi_g16" -for rtype in $rlist; do - ${CHGRP_CMD} *${rtype}* -done - -echo "<-- rstprod.sh"