Merge pull request #872 from thomasjacquin/saveImages.sh

Save images.sh

Author: EricClaeys

.gitignore

@@ -19,6 +19,8 @@ scripts/startrails_manual.sh
 scripts/startrails_only.sh
 scripts/stretch.sh
 scripts/endOfNight_additionalSteps.sh
+scripts/saveImageDay.sh
+scripts/saveImageNight.sh
 settings_RPiHQ.json
 settings_ZWO.json
 
@@ -35,4 +37,4 @@ capture
 capture_RPiHQ
 keogram
 startrails
-sunwait
+sunwait

README.md

@@ -1,11 +1,15 @@
-# Allsky Camera ![Release 0.8.1](https://img.shields.io/badge/Release-0.8.1-green.svg) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=MEBU2KN75G2NG&source=url)
-
+# Allsky Camera ![Release 0.8.3](https://img.shields.io/badge/Release-0.8.3-green.svg) [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=MEBU2KN75G2NG&source=url)
 
 This is the source code for the Allsky Camera project described [on Instructables](http://www.instructables.com/id/Wireless-All-Sky-Camera/).
 
+## Required Action Needed
+When upgrading from a release _prior to_ 0.8.3 you **MUST** follow the steps [here](https://github.com/thomasjacquin/allsky/wiki/Upgrade-from-0.8.2-or-prior-versions).
+
+
 
 ![](http://www.thomasjacquin.com/allsky-portal/screenshots/camera-header-photo.jpg)
 
+
 ## Requirements
 
 In order to get the camera working properly you will need the following hardware:
@@ -196,7 +200,7 @@ Set to "false" to keep all nights (requires manual management of SD card free sp
 
 ## Logging issues
 
-When using the allsky service, issues are written to a log file. In case the program stopped, crashed, or behaved in an abnormal way, take a look at this log file:
+When using the allsky service, issues are written to a log file. In case the program stopped, crashed, or behaved in an abnormal way, take a look at:
 ```
 tail /var/log/allsky.log
 ```
@@ -232,8 +236,7 @@ Once you've installed the website, either on your Pi or another machine, look at
 Experienced users may want to add some additional processing steps at the end of nighttime.
 To do so, copy `scripts/endOfNight_additionalSteps.repo` to 
 `scripts/endOfNight_additionalSteps.sh` and then add your additional processing steps which
-will be run after the usual end-of-night processing, but before the deletion of any
-old image files.
+will be run after the usual end-of-night processing, but before the deletion of any old image files.
 
 Edit this file via the "Editor" link on the left side of the WebUI page.
 
@@ -301,7 +304,19 @@ If you've built an allsky camera, please send me a message and I'll add you to t
 	* The WebUI will now show the Pi's throttle and low-voltage states, which is useful for debugging.
 	* Darks work better.
 	* Many bug fixes, error checks, and warnings added.
-
+* version **0.8.3**: Works on Bullseye operating system.
+	* RPiHQ version:
+	  * Has an improved auto-exposure algorithm.  To use it, set `CAPTURE_EXTRA_PARAMETERS="-mean-value 0.3 -autoexposure 1"` in config.sh (a future version will allow this to be set via the WebUI).
+	  * Has many new settings including support for most of the text overlay features that are supported by the ZWO version.  The "extra text" feature will be supported in a future version).
+	* New and changed config.sh variables, see the [Software Settings](https://github.com/thomasjacquin/allsky/wiki/allsky-Settings) Wiki page for more information:
+	  * `IMG_UPLOAD_FREQUENCY`: how often the image should be uploaded to a website.  Useful with slow uplinks or metered Internet connections.
+	  * `IMG_CREATE_THUMBNAILS`: specifies whether or not thumbnails should be created for each image.
+	  * `REMOVE_BAD_IMAGES` now defaults to "true" since bad-image detection is now done after a picture is saved rather than once for all pictures at the end of the night.  This helps decrease problems when creating startrails, keograms, and timelapse videos.
+	  * `IMG_PREFIX`: no longer used - the name of the image used by the websites is now whatever you specify in the WebUI (default: image.jpg).
+	  * **NOTE**: When upgrading to 0.8.3 you MUST follow the steps listed [here](https://github.com/thomasjacquin/allsky/wiki/Upgrade-from-0.8.2-or-prior-versions).
+	* Replaced `saveImageDay.sh` and `saveImageNight.sh` with `saveImage.sh` that has improved functionality, including passing the sensor temperature to the dark subtraction commands, thereby eliminating the need for the "temperature.txt" file.
+	* The image used by the websites (default: image.jpg) as well as all temporary files are now written to `allsky/tmp`.  **NOTE**: if you are using the Allsky Website you will need to change the "imageName" variable in `/var/www/html/allsky/config.js` to `"/current/tmp/"`.
+	* You can **significanly** reduce wear on your SD card by making `allsky/tmp` a [memory-based filesystem](https://github.com/thomasjacquin/allsky/wiki/Miscellaneous-Tips) for instructions).
 ## Donation
 
 If you found this project useful, here's a link to send me a cup of coffee :)

allsky.sh

@@ -25,16 +25,16 @@ fi
 # If not found, force the user to upgrade config.sh
 if [ -f "${ALLSKY_CONFIG}/config.sh" ]; then
 	source "${ALLSKY_CONFIG}/config.sh"
-	RET=$?
-	if [ -z "${ALLSKY_DEBUG_LEVEL}" ]; then
+	if [ -z "${CAPTURE_SAVE_DIR}" ]; then
 		echo "${RED}*** ERROR: old version of config.sh detected.${NC}"
-		RET=1
+		echo "See https://github.com/thomasjacquin/allsky/wiki/Upgrade-from-0.8.2-or-prior-versions"
+
+		"${ALLSKY_SCRIPTS}/copy_notification_image.sh" "Error" 2>&1
+		sudo systemctl stop allsky
+		exit 1
 	fi
 else
-	echo "${RED}*** ERROR: cannot find config.sh.${NC}"
-	RET=1
-fi
-if [ ${RET} -ne 0 ]; then
+	echo "${RED}*** ERROR: config.sh not in ${ALLSKY_CONFIG}.${NC}"
 	echo "Please make a backup of your config.sh, ftp-settings.sh, and settings_*.json files,"
 	echo "then do a full re-install of AllSky."
 	echo "After the re-install, copy your settings from the backup files to the new files."
@@ -45,24 +45,22 @@ if [ ${RET} -ne 0 ]; then
 	exit 1
 fi
 
-mkdir -p "${ALLSKY_TMP}"	# Re-create in case it's on a memory filesystem that gets wiped out at reboot
-
 # Make sure allsky.sh is not already running.
 ps -ef | grep allsky.sh | grep -v $$ | xargs "sudo kill -9" 2>/dev/null
 
 # old/regular manual camera selection mode => exit if no requested camera was found
 # Buster and Bullseye have different output so only check the part they have in common.
 # TODO: this check only needs to be done if CAMERA = RPiHQ
-# vcgencmd get_camera | grep --silent "supported=1"
-# bullseye has problems to dedect cameras - workaround
-which libcamera-still
+# Bullseye has problems detecting RPiHQ cameras - workaround
+which libcamera-still > /dev/null
 if [ $? -eq 0 ]; then
-        LIBCAMERA_LOG_LEVELS="ERROR,FATAL" libcamera-still -t 1 --nopreview
+	LIBCAMERA_LOG_LEVELS="ERROR,FATAL" libcamera-still -t 1 --nopreview
+	RET=$?
 else
-        vcgencmd get_camera | grep --silent "supported=1" 
+	vcgencmd get_camera | grep --silent "supported=1" 
+	RET=$?
 fi
-
-if [ $? -eq 0 ]; then
+if [ $RET -eq 0 ]; then
 	RPiHQIsPresent=1
 else
 	RPiHQIsPresent=0
@@ -147,15 +145,21 @@ echo "export CAMERA=${CAMERA}" > "${ALLSKY_CONFIG}/autocam.sh"
 # This must be called after CAMERA AUTOSELECT above to refresh the file name.
 source "${ALLSKY_SCRIPTS}/filename.sh"
 
+if [ -d "${ALLSKY_TMP}" ]; then
+	# remove any lingering old image files.
+	rm -f "${ALLSKY_TMP}/${FILENAME}"-202*.${EXTENSION}	# "202" for 2021 and later
+else
+	# Re-create in case it's on a memory filesystem that gets wiped out at reboot
+	mkdir -p "${ALLSKY_TMP}"
+fi
+
 # Optionally display a notification image. This must come after the creation of "autocam.sh" above.
 USE_NOTIFICATION_IMAGES=$(jq -r '.notificationimages' "$CAMERA_SETTINGS")
 if [ "$USE_NOTIFICATION_IMAGES" = "1" ] ; then
 	# Can do this in the background to speed up startup
 	"${ALLSKY_SCRIPTS}/copy_notification_image.sh" "StartingUp" 2>&1 &
 fi
 
-echo "Starting allsky camera..."
-
 # Building the arguments to pass to the capture binary.
 # Want to allow spaces in arguments so need to put quotes around them,
 # but in order for it to work need to make ARGUMENTS an array.
@@ -181,7 +185,6 @@ ARGUMENTS+=(-debuglevel ${ALLSKY_DEBUG_LEVEL})
 # This argument should come next so the capture program knows if it should use colors.
 ARGUMENTS+=(-tty ${ON_TTY})
 
-
 KEYS=( $(jq -r 'keys[]' $CAMERA_SETTINGS) )
 for KEY in ${KEYS[@]}
 do
@@ -195,6 +198,18 @@ if [[ $1 == "preview" ]] ; then
 	ARGUMENTS+=(-preview 1)
 fi
 
+ARGUMENTS+=(-save_dir "${CAPTURE_SAVE_DIR}")
+
+FREQUENCY_FILE="${ALLSKY_TMP}/IMG_UPLOAD_FREQUENCY.txt"
+# If the user wants images uploaded only every n times, save that number to a file.
+if [ "${IMG_UPLOAD_FREQUENCY}" != "1" ]; then
+	# Save "1" so we upload the first image.
+	# saveImage.sh will write ${IMG_UPLOAD_FREQUENCY} to the file as needed.
+	echo "1" > "${FREQUENCY_FILE}"
+else
+	rm -f "${FREQUENCY_FILE}"
+fi
+
 # "capture" expects 0 or 1; newer versions of config.sh use "true" and "false".
 # DAYTIME is the old name for DAYTIME_CAPTURE.
 # TODO: These checks will go away in the future.

config_repo/config.sh.repo

@@ -15,20 +15,18 @@ CAMERA=""
 IMG_UPLOAD="false"
 
 # If UPLOAD_IMG is "true", upload images every IMG_UPLOAD_FREQUENCY frames, e.g., every 5 frames.
-# 0 uploades every frame.
-IMG_UPLOAD_FREQUENCY=0
+# 1 uploades every frame.
+IMG_UPLOAD_FREQUENCY=1
 
 # The websites look in IMG_DIR for the current image.
-# When using the Allsky website from the "allsky-website" packge, "allsky" is an
-# alias for "/var/www/html/allsky/"  and   "current" is an alias for "/home/pi/allsky".
-# If you use "current" and have the Allsky website, prepend "/current/" to
-# the "imageName" in /var/www/html/allsky/config.js, for example:  imageName: "/current/image.jpg",
-# Using "current" avoids copying the image to the website.
-IMG_DIR="current"
-
-# Set an optional prefix on the image file name, before "$FILENAME.$EXTENSION".
-# This must match what the local web site thinks the name is (see /var/www/html/allsky/config.js).
-IMG_PREFIX="liveview-"
+# When using the Allsky website from the "allsky-website" packge,
+# "current" is an alias for "/home/pi/allsky".
+# If you use the default IMG_DIR and have the Allsky website, set "imageName" in
+# /var/www/html/allsky/config.js to:
+#     imageName: "/current/tmp/image.jpg",
+# This avoids copying the image to the website.
+# Only change IMG_DIR if you know what you are doing.
+IMG_DIR="current/tmp"
 
 # Resize images before cropping, stretching, and saving.
 # Adjust IMG_WIDTH and IMG_HEIGHT according to your camera's sensor ratio.
@@ -54,10 +52,12 @@ AUTO_STRETCH_MID_POINT="10%"
 RESIZE_UPLOADS="false"
 RESIZE_UPLOADS_SIZE="962x720"
 
-# Remove corrupt images before generating keograms and startrails.
-# This can take tens of minutes to run and isn't necessary unless your system produces
-# corrupt images which then generate funny colors in the summary images...
-REMOVE_BAD_IMAGES="false"
+# Create thumbnails of images.  If you are not running the WebUI consider changing this to "false".
+IMG_CREATE_THUMBNAILS="true"
+
+# Remove corrupt images before generating keograms, startrails, and timelapse videos.
+# This only adds a few seconds per image as it's being processed.
+REMOVE_BAD_IMAGES="true"
 
 # If REMOVE_BAD_IMAGES is "true", images whose mean brightness is
 # less than THRESHOLD_LOW or greater than THRESHOLD_HIGH percent (max: 100) will be removed.
@@ -142,6 +142,7 @@ AUTO_DELETE="true"
 NIGHTS_TO_KEEP=14
 
 # Set to "true" to upload data to your server at the end of each night.
+# This is needed if you are running the Allsky Website.
 POST_END_OF_NIGHT_DATA="false"
 
 # Set to "true" if you want to enable dark frame subtraction on your night-time images.
@@ -175,6 +176,13 @@ elif [ "${CAMERA}" = "auto" ]; then
 	exit 1
 fi
 
+# This is needed in case the user changed the default location the current image is saved to.
+if [ "${IMG_DIR}" = "current/tmp" ]; then
+	CAPTURE_SAVE_DIR="${ALLSKY_TMP}"
+else
+	CAPTURE_SAVE_DIR="${IMG_DIR}"
+fi
+
 # !!! The next few lines are for backwards compatibility and will be removed in the future - do not change !!!
 UPLOAD_IMG="${IMG_UPLOAD}"	# UPLOAD_IMG is th eold name for IMG_UPLOAD
 if [ "${DAYTIME_CAPTURE}" = "true" ]; then

scripts/copy_notification_image.sh

@@ -19,7 +19,7 @@ if [ ! -e "${NOTIFICATIONFILE}" ] ; then
 	exit 2
 fi
 
-IMAGE_TO_USE="${ALLSKY_TMP}/notification-${FULL_FILENAME}"
+IMAGE_TO_USE="${CAPTURE_SAVE_DIR}/notification-${FULL_FILENAME}"
 # Don't overwrite notification image so create a temporary copy and use that.
 cp "${NOTIFICATIONFILE}" "${IMAGE_TO_USE}"
 
@@ -32,32 +32,34 @@ if [ "${IMG_RESIZE}" = "true" ]; then
 	fi
 fi
 
-# If daytime saving is desired, save the image in today's thumbnail directory
+# If daytime saving is desired, save the image in today's thumbnail directory if desired
 # so the user can see when things changed.
 # Don't save in main image directory because we don't want the notification image in timelapses.
 # If at nighttime, save them in (possibly) yesterday's directory.
 # If during day, save in today's directory.
-if [ "${DAYTIME_SAVE}" = "true" -o "${CAPTURE_24HR}" = "true" ] ; then	# CAPTURE_24HR is OLD name
+if [ "${DAYTIME_SAVE}" = "true" -a "${IMG_CREATE_THUMBNAILS}" = "true" ] ; then
 	DATE_DIR="${ALLSKY_IMAGES}/$(date +'%Y%m%d')"
 	# Use today's folder if it exists, otherwise yesterday's
 	[ ! -d "${DATE_DIR}" ] && DATE_DIR="${ALLSKY_IMAGES}/$(date -d '12 hours ago' +'%Y%m%d')"
-	THUMB="${DATE_DIR}/thumbnails/${FILENAME}-$(date +'%Y%m%d%H%M%S').${EXTENSION}"
+	THUMBNAILS_DIR="${DATE_DIR}/thumbnails"
+	mkdir -p ${THUMBNAILS_DIR}
+	THUMB="${THUMBNAILS_DIR}/${FILENAME}-$(date +'%Y%m%d%H%M%S').${EXTENSION}"
 
 	convert "${IMAGE_TO_USE}" -resize "${THUMBNAIL_SIZE_X}x${THUMBNAIL_SIZE_Y}" "${THUMB}"
 	if [ $? -ne 0 ] ; then
 		echo "${YELLOW}*** ${ME}: WARNING: THUMBNAIL resize failed; continuing.${NC}"
-    	fi
+	fi
 fi
 
 FULL_FILENAME="${IMG_PREFIX}${FULL_FILENAME}"
-FINAL_IMAGE="${ALLSKY_HOME}/${FULL_FILENAME}"	# final resting place - no more changes to it.  TODO: put in ALLSKY_TMP
+FINAL_IMAGE="${CAPTURE_SAVE_DIR}/${FULL_FILENAME}"	# final resting place - no more changes to it.
 mv -f "${IMAGE_TO_USE}" "${FINAL_IMAGE}"	# so web severs can see it.
 
 # If upload is true, optionally create a smaller version of the image, either way, upload it.
 if [ "${UPLOAD_IMG}" = "true" ] ; then
 	if [ "${RESIZE_UPLOADS}" = "true" ]; then
 		# Don't overwrite FINAL_IMAGE since the web server(s) may be looking at it.
-		TEMP_FILE="${ALLSKY_TMP}/resize-${FULL_FILENAME}"
+		TEMP_FILE="${CAPTURE_SAVE_DIR}/resize-${FULL_FILENAME}"
 		cp "${FINAL_IMAGE}" "${TEMP_FILE}"  # create temporary copy to resize
 		convert "${TEMP_FILE}" -resize "${RESIZE_UPLOADS_SIZE}" -gravity East -chop 2x0 "${TEMP_FILE}"
 		if [ $? -ne 0 ] ; then

scripts/darkCapture.sh

@@ -2,36 +2,43 @@
 
 # This file is "source"d into another.
 # "${CURRENT_IMAGE}" is the name of the current image we're working on and is passed to us.
+ME2="$(basename "${BASH_SOURCE[0]}")"
 
-# ${TEMPERATURE} is passed to us by saveImage.sh, but may be null.
-# If ${TEMPERATURE} is set, use it as the temperature, otherwise read the ${TEMPERATURE_FILE}.
+# Make sure the input file exists; if not, something major is wrong so exit.
+if [ "${CURRENT_IMAGE}" = "" ]; then
+	echo "*** ${ME2}: ERROR: 'CURRENT_IMAGE' not set; aborting."
+	exit 1
+fi
+if [ ! -f "${CURRENT_IMAGE}" ]; then
+	echo "*** ${ME2}: ERROR: '${CURRENT_IMAGE}' does not exist; aborting."
+	exit 2
+fi
+
+# ${THIS_TEMPERATURE} is passed to us by saveImage.sh, but may be null.
+# If ${THIS_TEMPERATURE} is set, use it as the temperature, otherwise read the ${TEMPERATURE_FILE}.
 # If the ${TEMPERATURE_FILE} file doesn't exist, set the temperature to "n/a".
-if [ "${TEMPERATURE}" = "" ]; then
-	TEMPERATURE_FILE="${ALLSKY_HOME}/temperature.txt"
+if [ "${THIS_TEMPERATURE}" = "" ]; then
+	TEMPERATURE_FILE="${ALLSKY_TMP}/temperature.txt"
 	if [ -s "${TEMPERATURE_FILE}" ]; then	# -s so we don't use an empty file
-		TEMPERATURE=$(printf "%02.0f" "$(< ${TEMPERATURE_FILE})")
+		THIS_TEMPERATURE=$( < ${TEMPERATURE_FILE})
 	else
-		TEMPERATURE="n/a"
+		THIS_TEMPERATURE="n/a"
 	fi
 fi
 
 DARK_MODE=$(jq -r '.darkframe' "${CAMERA_SETTINGS}")
 if [ "${DARK_MODE}" = "1" ] ; then
-	# XXXXX in future release $CURRENT_IMAGE will be set by saveImage.sh
-	# and passed to us.  Doing this conditional set works either way.
-	CURRENT_IMAGE=${CURRENT_IMAGE:-"dark.${EXTENSION}"}
-
 	# The extension on $CURRENT_IMAGE may not be $EXTENSION.
 	DARK_EXTENSION="${CURRENT_IMAGE##*.}"
 
 	DARKS_DIR="${ALLSKY_DARKS}"
 	mkdir -p "${DARKS_DIR}"
 	# If the camera doesn't support temperature, we will keep overwriting the file until
 	# the user creates a temperature.txt file.
-	if [ "${TEMPERATURE}" = "n/a" ]; then
+	if [ "${THIS_TEMPERATURE}" = "n/a" ]; then
 		cp "${CURRENT_IMAGE}" "${DARKS_DIR}"
 	else
-		cp "${CURRENT_IMAGE}" "${DARKS_DIR}/${TEMPERATURE}.${DARK_EXTENSION}"
+		cp "${CURRENT_IMAGE}" "${DARKS_DIR}/${THIS_TEMPERATURE}.${DARK_EXTENSION}"
 	fi
 
 	# If the user has notification images on, the current image says we're taking dark frames,
@@ -45,7 +52,4 @@ if [ "${DARK_MODE}" = "1" ] ; then
 	fi
 
 	exit 0	# exit so the calling script exits and doesn't try to process the file.
-else	
-	# XXXXX Need this until saveImage.sh passes $CURRENT_IMAGE to us.
-	CURRENT_IMAGE="${CURRENT_IMAGE:-${FULL_FILENAME}}"	# Not capturing darks so use standard file name
 fi