Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

PR for #18: Improve error handling #19

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

PR for #18: Improve error handling #19

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
796332a
#18 changed error handling in run_python.sh
lucamlouzada Sep 18, 2024
fc0e95e
#18 changed error handling in run_R.sh
lucamlouzada Sep 18, 2024
182cce5
#18 changed error handling in run_stata.sh
lucamlouzada Sep 18, 2024
efc8d95
#18 changed error handling in run_latex.sh
lucamlouzada Sep 18, 2024
404e2d7
#18 changed error handling in run_shell.sh
lucamlouzada Sep 18, 2024
af61c3e
#18 changed error handling in ./make.sh
lucamlouzada Sep 18, 2024
98dc2cb
#18 changed shell error handling for zsh-bash consistency
lucamlouzada Sep 27, 2024
6240a8d
#18 changed regex in run_xx for zsh-bash consistency
lucamlouzada Sep 27, 2024
888557d
#18 minor fix for zsh consistency
lucamlouzada Sep 30, 2024
30a11ed
#18 added check for missing scripts in run_xxx
lucamlouzada Sep 30, 2024
f61cf15
#18 add blank line in the end of scripts
lucamlouzada Oct 1, 2024
cd9fdbf
#18 add blank line in the end of scripts
lucamlouzada Oct 1, 2024
f263307
#18 fix variable name in run_latex
lucamlouzada Oct 1, 2024
584c3d4
#18 fix default python command
lucamlouzada Oct 1, 2024
9e7d338
#18 add exit 1 to run_xx errors
lucamlouzada Oct 1, 2024
1f6cb71
#18 change regex in output scan to improve speed
lucamlouzada Oct 2, 2024
5e466e3
Fix #18 LFS pointer issue for slides.pdf
ShiqiYang2022 Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions 1_data/make.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
#!/bin/bash
set -e

# Trap to handle shell script errors
trap 'error_handler' ERR
error_handler() {
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\n\033[0;31mWarning\033[0m: make.sh failed at ${error_time}. Check above for details." # display warning in terminal
exit 1 # early exit with error code
}

# Set paths
# (Make sure REPO_ROOT is set to point to the root of the repository!)
Expand All @@ -11,6 +18,9 @@ LOGFILE="${MAKE_SCRIPT_DIR}/output/make.log"
# Check setup
source "${REPO_ROOT}/lib/shell/check_setup.sh"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Load settings & tools
source "${REPO_ROOT}/local_env.sh"
source "${REPO_ROOT}/lib/shell/run_shell.sh"
Expand All @@ -27,20 +37,20 @@ rm -rf "${MAKE_SCRIPT_DIR}/input"
mkdir -p "${MAKE_SCRIPT_DIR}/input"
# cp my_source_files "${MAKE_SCRIPT_DIR}/input/"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Run scripts
# (Do this in a subshell so we return to the original working directory
# after scripts are run)
echo -e "\nmake.sh started at $(date '+%Y-%m-%d %H:%M:%S')"

(
cd "${MAKE_SCRIPT_DIR}"
echo -e "make.sh started at $(date '+%Y-%m-%d %H:%M:%S')"
trap 'error_handler' ERR # reactivate trap in subshell

cd "${MAKE_SCRIPT_DIR}/source"

cd source
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
)

) 2>&1 | tee "${LOGFILE}"
cd "${MAKE_SCRIPT_DIR}" # return to original working directory
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved

echo -e "make.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
echo -e "\nmake.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
32 changes: 21 additions & 11 deletions 2_analysis/make.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
#!/bin/bash
set -e

# Trap to handle shell script errors
trap 'error_handler' ERR
error_handler() {
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\n\033[0;31mWarning\033[0m: make.sh failed at ${error_time}. Check above for details." # display warning in terminal
exit 1 # early exit with error code
}

# Set paths
# (Make sure REPO_ROOT is set to point to the root of the repository!)
Expand All @@ -11,6 +18,9 @@ LOGFILE="${MAKE_SCRIPT_DIR}/output/make.log"
# Check setup
source "${REPO_ROOT}/lib/shell/check_setup.sh"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Load settings & tools
source "${REPO_ROOT}/local_env.sh"
source "${REPO_ROOT}/lib/shell/run_shell.sh"
Expand All @@ -27,20 +37,20 @@ rm -rf "${MAKE_SCRIPT_DIR}/input"
mkdir -p "${MAKE_SCRIPT_DIR}/input"
# cp my_source_files "${MAKE_SCRIPT_DIR}/input/"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Run scripts
# (Do this in a subshell so we return to the original working directory
# after scripts are run)
echo -e "\nmake.sh started at $(date '+%Y-%m-%d %H:%M:%S')"

(
cd "${MAKE_SCRIPT_DIR}"
echo -e "make.sh started at $(date '+%Y-%m-%d %H:%M:%S')"
trap 'error_handler' ERR # reactivate trap in subshell

cd "${MAKE_SCRIPT_DIR}/source"

cd source
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
)

) 2>&1 | tee "${LOGFILE}"
cd "${MAKE_SCRIPT_DIR}" # return to original working directory

echo -e "make.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
echo -e "\nmake.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
32 changes: 21 additions & 11 deletions 3_slides/make.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
#!/bin/bash
set -e

# Trap to handle shell script errors
trap 'error_handler' ERR
error_handler() {
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\n\033[0;31mWarning\033[0m: make.sh failed at ${error_time}. Check above for details." # display warning in terminal
exit 1 # early exit with error code
}

# Set paths
# (Make sure REPO_ROOT is set to point to the root of the repository!)
Expand All @@ -11,6 +18,9 @@ LOGFILE="${MAKE_SCRIPT_DIR}/output/make.log"
# Check setup
source "${REPO_ROOT}/lib/shell/check_setup.sh"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Load settings & tools
source "${REPO_ROOT}/local_env.sh"
source "${REPO_ROOT}/lib/shell/run_shell.sh"
Expand All @@ -27,20 +37,20 @@ rm -rf "${MAKE_SCRIPT_DIR}/input"
mkdir -p "${MAKE_SCRIPT_DIR}/input"
# cp my_source_files "${MAKE_SCRIPT_DIR}/input/"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Run scripts
# (Do this in a subshell so we return to the original working directory
# after scripts are run)
echo -e "\nmake.sh started at $(date '+%Y-%m-%d %H:%M:%S')"

(
cd "${MAKE_SCRIPT_DIR}"
echo -e "make.sh started at $(date '+%Y-%m-%d %H:%M:%S')"
trap 'error_handler' ERR # reactivate trap in subshell

cd "${MAKE_SCRIPT_DIR}/source"

cd source
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
)

) 2>&1 | tee "${LOGFILE}"
cd "${MAKE_SCRIPT_DIR}" # return to original working directory

echo -e "make.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
echo -e "\nmake.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
32 changes: 21 additions & 11 deletions 4_paper/make.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
#!/bin/bash
set -e

# Trap to handle shell script errors
trap 'error_handler' ERR
error_handler() {
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\n\033[0;31mWarning\033[0m: make.sh failed at ${error_time}. Check above for details." # display warning in terminal
exit 1 # early exit with error code
}

# Set paths
# (Make sure REPO_ROOT is set to point to the root of the repository!)
Expand All @@ -11,6 +18,9 @@ LOGFILE="${MAKE_SCRIPT_DIR}/output/make.log"
# Check setup
source "${REPO_ROOT}/lib/shell/check_setup.sh"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Load settings & tools
source "${REPO_ROOT}/local_env.sh"
source "${REPO_ROOT}/lib/shell/run_shell.sh"
Expand All @@ -27,20 +37,20 @@ rm -rf "${MAKE_SCRIPT_DIR}/input"
mkdir -p "${MAKE_SCRIPT_DIR}/input"
# cp my_source_files "${MAKE_SCRIPT_DIR}/input/"

# Tell user what we're doing
echo -e "\n\nMaking module \033[35m${MODULE}\033[0m with shell ${SHELL}"

# Run scripts
# (Do this in a subshell so we return to the original working directory
# after scripts are run)
echo -e "\nmake.sh started at $(date '+%Y-%m-%d %H:%M:%S')"

(
cd "${MAKE_SCRIPT_DIR}"
echo -e "make.sh started at $(date '+%Y-%m-%d %H:%M:%S')"
trap 'error_handler' ERR # reactivate trap in subshell

cd "${MAKE_SCRIPT_DIR}/source"

cd source
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
run_shell my_shell_script.sh "${LOGFILE}"
# run_xxx my_script.xx "${LOGFILE}"
)

) 2>&1 | tee "${LOGFILE}"
cd "${MAKE_SCRIPT_DIR}" # return to original working directory

echo -e "make.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
echo -e "\nmake.sh finished at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${LOGFILE}"
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
59 changes: 46 additions & 13 deletions lib/shell/run_R.sh
View file
Open in desktop
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One general comment: we sometime don't need very much comments, the scripts should be self documenting. Maybe we can tidy up the scripts for a little bit?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this makes sense. I can remove some comments (and please let me know if you see specific ones that you think should be removed). Thanks @ShiqiYang2022

lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,60 @@

unset run_R
run_R () {
trap - ERR # allow internal error handling

# get arguments
program="$1"
logfile="$2"
OUTPUT_DIR=$(dirname "$logfile")

# set R command if unset
if [ -z "$rCmd" ]; then
echo "No R command set. Using default: Rscript"
echo -e "\nNo R command set. Using default: Rscript"
rCmd="Rscript"
fi

# run program, add output to logfile
echo "Executing: ${rCmd} ${program}"
(
${rCmd} "${program}" 1>> "${logfile}" 2>> "${logfile}"
# check if the command exists before running, log error if does not
if ! command -v ${rCmd} &> /dev/null; then
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\033[0;31mProgram error\033[0m at ${error_time}: ${rCmd} not found. Make sure command line usage is properly set up."
echo "Program Error at ${error_time}: ${rCmd} not found." >> "${logfile}"
return 1 # exit early with an error code
fi

# report on errors
return_code=$?
if [ $return_code -ne 0 ]; then
# Log an error message if the command failed
echo "Error: ${program} failed with exit code $return_code"
fi
)
}
# capture the content of output folder before running the script
files_before=$(ls -1 "$OUTPUT_DIR" | grep -v "make.log" | tr '\n' ' ')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My sense is ls and grep are slow. If we have many many small files as output (maybe for example, television transcripts), then this need to take a lot of time to process. Shall we consider using more efficient methods to handle file lists, such as directly using regular expressions within the find command.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar comment to the all other scripts below. Also, I don't know whether we need to separate this functionality into a seperate part of script? Say, we can do separate this part into check_file_dependencies.sh.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @ShiqiYang2022 , I will investigate different approaches for handling the file lists.

Copy link
Collaborator Author

@lucamlouzada lucamlouzada Oct 2, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ShiqiYang2022 I just pushed a change in 1f6cb71 to replace the code that handles the file lists. Let me know what you think. In the same commit I also added a minor fix to the Program Error warnings so that they skip a line when printing in the terminal. Thanks!


# log start time for the script
echo -e "\nScript ${program} in ${rCmd} started at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${logfile}"

# run command and capture both stdout and stderr in the output variable
output=$(${rCmd} "${program}" 2>&1)
return_code=$? # capture the exit status

# capture the content of output folder after running the script
files_after=$(ls -1 "$OUTPUT_DIR" | grep -v "make.log" | tr '\n' ' ')

# determine the new files that were created
created_files=$(comm -13 <(echo "$files_before") <(echo "$files_after"))

# report on errors or success and display the output
if [ $return_code -ne 0 ]; then
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\033[0;31mWarning\033[0m: ${program} failed at ${error_time}. Check log for details." # display error warning in terminal
echo "Error in ${program} at ${error_time}: $output" >> "${logfile}" # log error output
if [ -n "$created_files" ]; then
echo -e "\033[0;31mWarning\033[0m: there was an error, but files where created. Check log."
echo -e "\nWarning: There was an error, but these files were created: $created_files" >> "${logfile}" # log created files
fi
exit 1
else
echo "Script ${program} finished successfully at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${logfile}"
echo "Output: $output" >> "${logfile}" # log output

if [ -n "$created_files" ]; then
echo -e "\nThe following files were created in ${program}:" >> "${logfile}"
echo "$created_files" >> "${logfile}" # list files in output folder
fi
fi
}
lucamlouzada marked this conversation as resolved.
Show resolved Hide resolved
65 changes: 58 additions & 7 deletions lib/shell/run_latex.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,25 @@
unset run_latex
unset cleanup
run_latex() {

trap - ERR # allow internal error handling

# Get arguments
programname=$(basename "$1" .tex)
logfile="$2"

# Set output directory
# (If no third argument is provided, use the default)
if [ -n "$3" ]; then
outputdir="$3"
OUTPUT_DIR="$3"
else
outputdir="../output"
OUTPUT_DIR="../output"
fi

# Clean up
cleanup() {
mv "${programname}.pdf" ${outputdir}
if [ -f "${programname}.pdf" ]; then
mv "${programname}.pdf" "${OUTPUT_DIR}"
fi
rm -f "${programname}.aux" \
"${programname}.bbl" \
"${programname}.blg" \
Expand All @@ -35,9 +38,57 @@ run_latex() {
# Ensure cleanup is called on exit
trap 'cleanup' EXIT


echo "Executing: latexmk ${programname}.tex -pdf -bibtex"
(latexmk "${programname}.tex" -pdf -bibtex >> "${logfile}" 2>&1)
# Check if latexmk command exists
if ! command -v latexmk &> /dev/null; then
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\033[0;31mProgram error\033[0m at ${error_time}: latexmk not found. Ensure LaTeX is installed."
echo "Program Error at ${error_time}: latexmk not found." >> "${logfile}"
return 1
fi

# capture the content of output folder before running the script
files_before=$(ls -1 "$OUTPUT_DIR" | grep -v "make.log" | tr '\n' ' ')

# log start time for the script
echo -e "\nScript ${programname}.tex in latexmk -pdf -bibtex started at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${logfile}"

# run command and capture both stdout and stderr in the output variable
output=$(latexmk -interaction=nonstopmode -f "${programname}.tex" -pdf -bibtex 2>&1)
return_code=$? # capture the exit status

# perform cleanup
cleanup

# capture the content of output folder after running the script
files_after=$(ls -1 "$OUTPUT_DIR" | grep -v "make.log" | tr '\n' ' ')

# determine the new files that were created
created_files=$(comm -13 <(echo "$files_before") <(echo "$files_after"))

# report on errors or success and display the output
if [ $return_code -ne 0 ]; then
error_time=$(date '+%Y-%m-%d %H:%M:%S')
echo -e "\033[0;31mWarning\033[0m: ${programname}.tex failed at ${error_time}. Check log for details." # display error warning in terminal
echo "Error in ${programname}.tex at ${error_time}: $output" >> "${logfile}" # log error output
if [ -n "$created_files" ]; then
echo -e "\033[0;31mWarning\033[0m: there was an error, but files where created. Check log."
echo -e "\nWarning: There was an error, but these files were created: $created_files" >> "${logfile}" # log created files
fi
exit 1
else
echo "Script ${programname}.tex finished successfully at $(date '+%Y-%m-%d %H:%M:%S')" | tee -a "${logfile}"
echo "Output: $output" >> "${logfile}" # log output

if [ -n "$created_files" ]; then
echo -e "\nThe following files were created in ${programname}.tex:" >> "${logfile}"
echo "$created_files" >> "${logfile}" # list files in output folder
fi

if [ ! -f "${OUTPUT_DIR}/${programname}.pdf" ]; then
echo -e "\033[0;31mWarning\033[0m: No PDF file was created. Check output in log."
echo -e "\nWarning: No PDF file was created. Check output above." >> "${logfile}" # log warning
fi
fi

}

Loading