From f2a980292071a4700ddd403f95284ed06055b731 Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Mon, 13 Sep 2021 18:49:21 -0700 Subject: [PATCH 1/6] Make it easier to reprocess images on other machines `rm -rf /tmp/nonexistent` doesn't fail on nonexistent files, so unconditionally nuke the sequence directory ``` $ rm -rf /tmp/nonexistent && echo that worked that worked ``` provide some default values for the framerate and encoder, just in case you don't have them in your configs. --- scripts/timelapse.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index 2443dd6b0..4f878b737 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -5,17 +5,21 @@ source $ALLSKY_HOME/scripts/ftp-settings.sh cd $ALLSKY_HOME +FPS=${FPS:-25} +ENCODER=${ENCODER:-libx264} + ME="$(basename "$BASH_ARGV0")" # Include script name in output so it's easier to find in the log file RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' # No Color -if [ $# -lt 1 ] - then - echo -en "${ME}: ${RED}You need to pass a day argument\n" - echo -en " ex: timelapse.sh 20180119${NC}\n" - exit 3 +if [ $# -lt 1 -o $# -gt 2 -o "x$1" = "x-h" ] ; then + TODAY=$(date +%Y%m%d) + echo -en "${RED}Usage: ${ME} [directory]${NC}\n" + echo -en " ex: timelapse.sh ${TODAY}\n" + echo -en " or: timelapse.sh ${TODAY} /media/external/allsky/${TODAY}\n" + exit 3 fi # Allow timelapses of pictures not in ALLSKY_HOME. @@ -27,7 +31,7 @@ else DIR="$2" fi echo -en "${ME}: ${GREEN}Creating symlinks to generate timelapse${NC}\n" -[ -d $DIR/sequence ] && rm -fr $DIR/sequence # remove in case it was left over from last run +rm -fr $DIR/sequence # remove in case it was left over from last run mkdir -p $DIR/sequence/ # find images, make symlinks sequentially and start avconv to build mp4; upload mp4 and move directory From f5c4abb64440883647af6db1529196200c3d74bd Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Tue, 14 Sep 2021 12:25:29 -0700 Subject: [PATCH 2/6] More reprocessing fun Now this sort of thing works: ``` ~$ env FFLOG=info VCODEC=h264_nvenc ~/allsky/scripts/timelapse.sh 20210913 /nas/home/allsky/20210913/ timelapse.sh: Creating symlinks to generate timelapse ``` --- config.sh.repo | 3 +++ scripts/timelapse.sh | 15 +++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config.sh.repo b/config.sh.repo index a60f7fcdb..fd5136dc0 100644 --- a/config.sh.repo +++ b/config.sh.repo @@ -25,6 +25,9 @@ TIMELAPSE_BITRATE="2000k" # Timelapse frame rate (number of frames per second) FPS=25 +# Encoder for timelapse. May be changed to use a hardware encoder or different codec +VCODEC=libx264 + # Set to true to generate a keogram at the end of the night (image summary of the night) KEOGRAM=true diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index 4f878b737..fed40178a 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -1,13 +1,15 @@ #!/bin/bash + +if [ -z "$ALLSKY_HOME" ] ; then + export ALLSKY_HOME=$(realpath $(dirname "$BASH_ARGV0")/..) +fi + source $ALLSKY_HOME/config.sh source $ALLSKY_HOME/scripts/filename.sh source $ALLSKY_HOME/scripts/ftp-settings.sh cd $ALLSKY_HOME -FPS=${FPS:-25} -ENCODER=${ENCODER:-libx264} - ME="$(basename "$BASH_ARGV0")" # Include script name in output so it's easier to find in the log file RED='\033[0;31m' @@ -71,11 +73,12 @@ fi # "-loglevel warning" gets rid of the dozens of lines of garbage output # but doesn't get rid of "deprecated pixel format" message when -pix_ftm is "yuv420p". +# set FFLOG=info in config.sh if you want to see what's going on for debugging ffmpeg -y -f image2 \ - -loglevel warning \ - -r $FPS \ + -loglevel ${FFLOG:-warning} \ + -r ${FPS:-25} \ -i $DIR/sequence/%04d.$EXTENSION \ - -vcodec libx264 \ + -vcodec ${VCODEC:libx264} \ -b:v ${TIMELAPSE_BITRATE:-2000k} \ -pix_fmt yuv420p \ -movflags +faststart \ From 89e094a20eacb79bdc1e3c172c025f6d8f25cb04 Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Tue, 14 Sep 2021 13:35:23 -0700 Subject: [PATCH 3/6] allow sequence directory to be retained Makes tuning encoder settings a little easier by keeping the sequence. Not meant for general consumption which is why it's only documented in a comment. --- scripts/timelapse.sh | 77 +++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index fed40178a..24a680653 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -14,6 +14,7 @@ ME="$(basename "$BASH_ARGV0")" # Include script name in output so it's easier to RED='\033[0;31m' GREEN='\033[0;32m' +YELLOW='\033[0;33m' NC='\033[0m' # No Color if [ $# -lt 1 -o $# -gt 2 -o "x$1" = "x-h" ] ; then @@ -32,35 +33,47 @@ if [ "$2" = "" ] ; then else DIR="$2" fi -echo -en "${ME}: ${GREEN}Creating symlinks to generate timelapse${NC}\n" -rm -fr $DIR/sequence # remove in case it was left over from last run -mkdir -p $DIR/sequence/ - -# find images, make symlinks sequentially and start avconv to build mp4; upload mp4 and move directory -find "$DIR" -name "*.$EXTENSION" -size 0 -delete - -TMP_DIR="$ALLSKY_HOME/tmp" -mkdir -p "$TMP_DIR" -TMP="$TMP_DIR/timelapseTMP.txt" # capture the "ln" commands in case the user needs to debug -> $TMP -ls -rt $DIR/*.$EXTENSION | -gawk 'BEGIN{ a=1 } - { - printf "ln -s %s '$DIR'/sequence/%04d.'$EXTENSION'\n", $0, a; - printf "ln -s %s '$DIR'/sequence/%04d.'$EXTENSION'\n", $0, a >> "'$TMP'"; - # if (a % 100 == 0) printf "echo '$ME': %d links created so far\n", a; - a++; - }' | -bash - -# Make sure there's at least one link. -NUM_FILES=$(wc -l < ${TMP}) -if [ $NUM_FILES -eq 0 ]; then - echo -en "*** ${ME}: ${RED}ERROR: No links found!${NC}\n" - rmdir "${DIR}/sequence" - exit 1 + +# If you are tuning encoder settings, run this script with KEEP_SEQUENCE, eg. +# $ env KEEP_SEQUENCE=1 VCODEC=h264_nvenc ~/allsky/scripts/timelapse.sh ${TODAY} /media/external/allsky/${TODAY}/ +# to keep the sequence directory from being deleted and to reuse the contents +# of the sequence directory if it looks ok (contains at least 100 files). This +# might save you some time when running your encoder repeatedly. + +NSEQ=$(ls "/${DIR}/sequence" | wc -l ) +if [ -z "$KEEP_SEQUENCE" -o $NSEQ -lt 100 ] ; then + echo -en "${ME}: ${GREEN}Creating symlinks to generate timelapse${NC}\n" + rm -fr $DIR/sequence + mkdir -p $DIR/sequence + + # find images, make symlinks sequentially and start avconv to build mp4; upload mp4 and move directory + find "$DIR" -name "*.$EXTENSION" -size 0 -delete + + TMP_DIR="$ALLSKY_HOME/tmp" + mkdir -p "$TMP_DIR" + TMP="$TMP_DIR/timelapseTMP.txt" # capture the "ln" commands in case the user needs to debug + > $TMP + ls -rt $DIR/*.$EXTENSION | + gawk 'BEGIN{ a=1 } + { + printf "ln -s %s '$DIR'/sequence/%04d.'$EXTENSION'\n", $0, a; + printf "ln -s %s '$DIR'/sequence/%04d.'$EXTENSION'\n", $0, a >> "'$TMP'"; + # if (a % 100 == 0) printf "echo '$ME': %d links created so far\n", a; + a++; + }' | + bash + + # Make sure there's at least one link. + NUM_FILES=$(wc -l < ${TMP}) + if [ $NUM_FILES -eq 0 ]; then + echo -en "*** ${ME}: ${RED}ERROR: No links found!${NC}\n" + rmdir "${DIR}/sequence" + exit 1 + else + echo -e "$ME: Created $NUM_FILES links total\n" + fi else - echo -e "$ME: Created $NUM_FILES links total\n" + echo -en "${ME}: ${YELLOW}Not regenerating sequence because KEEP_SEQUENCE was given and $NSEQ links are present ${NC}\n" fi SCALE="" @@ -123,7 +136,11 @@ if [ "$UPLOAD_VIDEO" = true ] ; then fi fi -echo -en "${ME}: ${GREEN}Deleting sequence${NC}\n" -rm -rf $DIR/sequence +if [ -z "$KEEP_SEQUENCE" ] ; then + echo -en "${ME}: ${GREEN}Deleting sequence${NC}\n" + rm -rf $DIR/sequence +else + echo -en "${ME}: ${GREEN}Keeping sequence${NC}\n" +fi echo -en "${ME}: ${GREEN}Timelapse was created${NC}\n" From 21621ddda96f937e1130c3aeabcdb9529613170e Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Tue, 14 Sep 2021 13:49:47 -0700 Subject: [PATCH 4/6] try to guess image extension if unconfigured ``` ~/allsky$ env VCODEC=h264_nvenc ~/allsky/scripts/timelapse.sh 20210913 /nas/homes/allsky/20210913/ /home/ckuethe/allsky/scripts/timelapse.sh: line 7: /home/ckuethe/allsky/config.sh: No such file or directory jq: error: Could not open file : No such file or directory timelapse.sh: file EXTENSION not found in configuration, guessing jpg timelapse.sh: Creating symlinks to generate timelapse timelapse.sh: Created 6340 links total ... ``` --- scripts/timelapse.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index 24a680653..e32998a29 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -34,6 +34,14 @@ else DIR="$2" fi +# Guess what the likely image extension is (unless specified in the config) by +# looking at the most common extension in the target day directory +if [ -z "$EXTENSION" ] ; then + EXT_GUESS=$(ls $DIR | sed -e 's/.*[.]//' | sort | uniq -c | head -1 | sed -e 's/.* //') + echo -en "${RED}${ME}: file EXTENSION not found in configuration, guessing ${EXT_GUESS}${NC}\n" + EXTENSION=$EXT_GUESS +fi + # If you are tuning encoder settings, run this script with KEEP_SEQUENCE, eg. # $ env KEEP_SEQUENCE=1 VCODEC=h264_nvenc ~/allsky/scripts/timelapse.sh ${TODAY} /media/external/allsky/${TODAY}/ # to keep the sequence directory from being deleted and to reuse the contents From 5e7b3f1ca60cc767d5fbb22b4e05c5386c4ed486 Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Tue, 14 Sep 2021 16:58:30 -0700 Subject: [PATCH 5/6] fix typo --- scripts/timelapse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index e32998a29..9ba73b15c 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -99,7 +99,7 @@ ffmpeg -y -f image2 \ -loglevel ${FFLOG:-warning} \ -r ${FPS:-25} \ -i $DIR/sequence/%04d.$EXTENSION \ - -vcodec ${VCODEC:libx264} \ + -vcodec ${VCODEC:-libx264} \ -b:v ${TIMELAPSE_BITRATE:-2000k} \ -pix_fmt yuv420p \ -movflags +faststart \ From be9c8cc25dfc01bb3837db47e4ce02fd12f231c4 Mon Sep 17 00:00:00 2001 From: Chris Kuethe Date: Wed, 15 Sep 2021 12:03:34 -0700 Subject: [PATCH 6/6] silence stderr from ls --- scripts/timelapse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/timelapse.sh b/scripts/timelapse.sh index 9ba73b15c..381b606c8 100755 --- a/scripts/timelapse.sh +++ b/scripts/timelapse.sh @@ -48,7 +48,7 @@ fi # of the sequence directory if it looks ok (contains at least 100 files). This # might save you some time when running your encoder repeatedly. -NSEQ=$(ls "/${DIR}/sequence" | wc -l ) +NSEQ=$(ls "/${DIR}/sequence" 2>/dev/null | wc -l ) if [ -z "$KEEP_SEQUENCE" -o $NSEQ -lt 100 ] ; then echo -en "${ME}: ${GREEN}Creating symlinks to generate timelapse${NC}\n" rm -fr $DIR/sequence