Merge branch 'master' into makelossless-updates

Author: dericed

README.md

@@ -162,7 +162,7 @@ set this variable to the directory where your youtube access copy is delivered.
 set this variable to a temporary directory. it is used in the uploadomneon microservice as a temporary place for a file to live before it is uploaded to the omneon server.
 
 **10. REGEX4PODCAST**
-this varible holds regular expressions that are queried when makepodcast is run, in order to determine if a file qualifies for podcast creation. If you want all of your files to qualify for podcast creation, enter a "." which matches (almost) any character. Learn more about [regex](https://en.wikipedia.org/wiki/Regular_expression).
+this variable holds regular expressions that are queried when makepodcast is run, in order to determine if a file qualifies for podcast creation. If you want all of your files to qualify for podcast creation, enter a "." which matches (almost) any character. Learn more about [regex](https://en.wikipedia.org/wiki/Regular_expression).
 
 **11. DVDLABELPREFIX**
 this variable is for adding a set prefix to the filename for DVDs in makedvd. You may leave this variable blank if you do not want to have a prefix uniformly assigned.
@@ -201,7 +201,7 @@ This variable must be set to yes (Y) or no (N). If set to yes, volume will be ru
 This variable must be set to yes (Y) or no (N). If set to yes, database reporting for mm will be enabled.
 
 **23. SQL_ERROR_EMAIL_TO**
-This can be set to include an email address to recieve error reports for database insertion.
+This can be set to include an email address to receive error reports for database insertion.
 
 **24. PREMIS_PROFILE**
 Enter the login path information for the database here. (Use the output supplied by the createpremisdb script, or create your own using mysql_config_editor).
@@ -236,7 +236,7 @@ To finalize the database setup, run `mmconfig` (GUI mode) or `mmconfig -t` (CLI)
 ### Database Backup
 MM includes a script for database backup.  This can be either run manually or set up as an automated process.  After configuring the backup script, running `dbbackup` will create a zipped backup in the chosen location.  To automate this process use Brew Services.  Typing `brew services start mm` will cause the db to be backed up automatically using the included plist file.  By default, backups will occur daily at 2:00 AM.
 
-To configure database backup use the command `dbbackup -e`. This will open the config file in a terminal editor. Set the neccessary variables to your desired values.
+To configure database backup use the command `dbbackup -e`. This will open the config file in a terminal editor. Set the necessary variables to your desired values.
 
 ***
 
@@ -385,10 +385,10 @@ To view the specific ffmpeg encoding options for each file, view the sourcecode
 * Creates a frame md5 for video files in target package. Relies on package structure used in mm.
 
 #### makeframes
-* makeframes creates 10 still images from a video file or package input. Your command will look like:  makeframes [input]. To deliver still images to a specific directory, use this command  `ingestfile -d [path/to/directory] [input]`.
+* makeframes creates 10 still images from a video file or package input. Your command will look like: `makeframes [input]`. To deliver still images to a specific directory, use this command  `ingestfile -d [path/to/directory] [input]`.
 
 #### makelossless
-* makelossless losslessly transcodes a video file or package input. Your command will look like this: makelossless [input].
+* makelossless losslessly transcodes a video file or package input. Your command will look like this: `makelossless [input]`.
 * if you would like to transcode using lossless jpeg2000 instead of ffv1 version 3, use option j. Your command will look like this: `makelossless -j [input]`.
 * if you want to run makelossless in "dry-run" mode, which means that the commands will be shown in the terminal but not run, use option n. Your command will look like this:  `makelossless -n [input]`.
 

audiotest

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+echo "Package Name,First Object File Name,Count of Object Files,Count of Image Files,integrated loudness,loudness range,true peak,threshold,\
+target_offset,cuny loudness adjustment,silent ranges,phase issues,interstitial,duration,samplerate,channels,codec,peak level ch1,peak level ch2,peak level diff,bitscope,dcoffset,flatfactor,peakcount,bwfmetaedit"
+while [[ "${@}" != "" ]] ; do
+    PACKAGE="$1"
+    shift
+    FILECOUNT=$(find "${PACKAGE}/objects" -maxdepth 2 -type f ! -name .DS_Store | wc -l | sed 's/ //g')
+
+    find "${PACKAGE}/objects" -maxdepth 2 -type f ! -name .DS_Store | while read FILENAME ; do
+        ffmpeg -nostdin -i "${FILENAME}" -vn -filter_complex "silencedetect,astats,loudnorm=print_format=json,agate=attack=0.01:release=0.01:ratio=9000,aphasemeter=r=0.2:video=0,ametadata=mode=print:key=lavfi.aphasemeter.phase:value=-0.18:function=less" -f null - 2>&1 |  tr "\r" "\n" > /tmp/loud
+        image_count=$(find "${PACKAGE}/metadata/depictions" -maxdepth 2 -type f ! -name .DS_Store | wc -l | sed 's/ //g')
+        measured_I=$(cat /tmp/loud | grep "input_i" | awk -F\" '{print $4}')
+        measured_LRA=$(cat /tmp/loud | grep "input_lra" | awk -F\" '{print $4}')
+        measured_TP=$(cat /tmp/loud | grep "input_tp" | awk -F\" '{print $4}')
+        measured_thresh=$(cat /tmp/loud | grep "input_thresh" | awk -F\" '{print $4}')
+        target_offset=$(cat /tmp/loud | grep "target_offset" | awk -F\" '{print $4}')
+        LOUD_ADJ=$(echo "($measured_I - -24)*-1" | bc)
+        DURATION=$(ffprobe "$FILENAME" -show_entries format=duration -of default=nw=1:nk=1 2>/dev/null)
+        SAMPLERATE=$(ffprobe "$FILENAME" -show_entries stream=sample_rate -of default=nw=1:nk=1 2>/dev/null)
+        CHANNELS=$(ffprobe "$FILENAME" -show_entries stream=channels -of default=nw=1:nk=1 2>/dev/null)
+        CODEC=$(ffprobe "$FILENAME" -show_entries stream=codec_name -of default=nw=1:nk=1 2>/dev/null)
+
+        PEAKLEVEL1=$(cat /tmp/loud | grep "Peak level dB:" | tail -n +1 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+        PEAKLEVEL2=$(cat /tmp/loud | grep "Peak level dB:" | tail -n +2 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+        PEAKLEVELDIFF=$(echo "scale=3; $PEAKLEVEL1 - $PEAKLEVEL2" | bc)
+
+        BITSCOPE=$(cat /tmp/loud | grep "Bit depth: " | tail -n +3 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+        DCOFFSET=$(cat /tmp/loud | grep "DC offset: " | tail -n +3 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+        FLATFACTOR=$(cat /tmp/loud | grep "Flat factor: " | tail -n +3 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+        PEAKCOUNT=$(cat /tmp/loud | grep "Peak count: " | tail -n +3 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+
+        bwfmetaedit=$(bwfmetaedit "$FILENAME" 2>&1 | awk '{printf "%s+",$0} END {print ""}')
+
+        SILENCE=$(cat /tmp/loud | grep " silence_end: " | while read line ; do
+            silence_duration=$(echo "$line" | awk '{print $8}')
+            silence_end=$(echo "$line" | awk '{print $5}')
+            echo -n "$silence_end ($silence_duration);"
+        done)
+
+        PHASEERROR=$(grep Parsed_ametadata /tmp/loud | paste - - | sed "s/lavfi.aphasemeter.phase=//g" | cut -d ":" -f 4 | awk '{print $1 "(" $5 ")"}' | xargs)
+        INTERSTITIAL=$(ffprobe -f lavfi -i "amovie=${FILENAME},loudnorm=tp=0,astats=metadata=1:reset=1" -show_entries frame=pkt_pts_time:frame_tags=lavfi.astats.Overall.Max_difference -of csv 2>/dev/null | awk -F  "," '$3>=0.07' | awk -F  "," '{print $2 "(" $3 ")"}' | xargs)
+
+        echo "$(basename "$PACKAGE"),$(basename "$FILENAME"),$FILECOUNT,$image_count,$measured_I,$measured_LRA,$measured_TP,$measured_thresh,$target_offset,$LOUD_ADJ,$SILENCE,$PHASEERROR,$INTERSTITIAL,$DURATION,$SAMPLERATE,$CHANNELS,$CODEC,$PEAKLEVEL1,$PEAKLEVEL2,$PEAKLEVELDIFF,$BITSCOPE,$DCOFFSET,$FLATFACTOR,$PEAKCOUNT,$bwfmetaedit"
+    done
+done

checksumpackage

@@ -64,10 +64,10 @@ _check_dfxml(){
     PACKAGE="${ABS_PATH}"
     # make temp file that lists filenames and filesizes from dfxml.xml
     DFXML_FILELIST="$(_maketemp)"
-    if [ ! -z "$(xml sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" 2>/dev/null)" ] ; then
-        xml sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" | sort > "${DFXML_FILELIST}"
+    if [ ! -z "$(xmlstarlet sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" 2>/dev/null)" ] ; then
+        xmlstarlet sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" | sort > "${DFXML_FILELIST}"
     else
-        xml sel -T -t -m "/dfxml/fileobject" -v "filename" -o " " -v "filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" | sort > "${DFXML_FILELIST}"
+        xmlstarlet sel -T -t -m "/dfxml/fileobject" -v "filename" -o " " -v "filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" | sort > "${DFXML_FILELIST}"
     fi
     PWD=$(pwd)
     # make temp file that lists filenames and filesizes from current objects directory
@@ -177,10 +177,10 @@ while [ "${*}" != "" ] ; do
             _version_dfxml "${PACKAGE}"
         fi
         cat "${DFXMLTMP}" > "${PACKAGE}/${OUTDIR}/${DFXMLNAME}"
-        if [ ! -z "$(xml sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" 2>/dev/null)" ] ; then
-            xml sel -T -t -m "/_:dfxml/_:fileobject" -v "_:hashdigest" -o "  " -v "_:filename" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" > "${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}"
+        if [ ! -z "$(xmlstarlet sel -T -t -m "/_:dfxml/_:fileobject" -v "_:filename" -o " " -v "_:filesize" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" 2>/dev/null)" ] ; then
+            xmlstarlet sel -T -t -m "/_:dfxml/_:fileobject" -v "_:hashdigest" -o "  " -v "_:filename" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" > "${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}"
         else
-            xml sel -T -t -m "/dfxml/fileobject" -v "hashdigest" -o "  " -v "filename" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" > "${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}"
+            xmlstarlet sel -T -t -m "/dfxml/fileobject" -v "hashdigest" -o "  " -v "filename" -n "${PACKAGE}/${OUTDIR}/${DFXMLNAME}" > "${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}"
         fi
         db_source="${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}"
         db_fixity=$(cat "${PACKAGE}/${OUTDIR}/${CHECKSUMNAME}")

ingestfile

@@ -82,7 +82,7 @@ _ask_digitization_logs(){
 # spreadsheet fields are currently CUNY-specific
 _parse_spreadsheet(){
     DIGITIZATIONLOG=$(_maketemp)
-    paste -d"*" <(ssconvert -T  Gnumeric_html:xhtml_range "${FILES}" fd://1 | xml fo | xml select --no-doc-namespace -t -m "_:html/_:body/_:table[1]/_:tr[1]/_:td" -v "." -n) <(ssconvert -T  Gnumeric_html:xhtml_range "${FILES}" fd://1 | xml fo | xml select --no-doc-namespace -t -m "_:html/_:body/_:table[1]/_:tr[_:td[1]='${MEDIAID}']/_:td" -v "." -n) | sed "s/*/\: /g" | tee "${DIGITIZATIONLOG}"
+    paste -d"*" <(ssconvert -T Gnumeric_html:xhtml_range "${FILES}" fd://1 | xmlstarlet fo | xmlstarlet select --no-doc-namespace -t -m "_:html/_:body/_:table[1]/_:tr[1]/_:td" -v "." -n) <(ssconvert -T  Gnumeric_html:xhtml_range "${FILES}" fd://1 | xmlstarlet fo | xmlstarlet select --no-doc-namespace -t -m "_:html/_:body/_:table[1]/_:tr[_:td[1]='${MEDIAID}']/_:td" -v "." -n) | sed "s/*/\: /g" | tee "${DIGITIZATIONLOG}"
     cat "${DIGITIZATIONLOG}" >> "${INGESTLOG}"
     OBJECTSOUNDFIELD=$(_readingestlog "object soundfield")
     DECKHEADSETTING=$(_readingestlog "deck head setting")
@@ -100,11 +100,10 @@ _parse_spreadsheet(){
 }
 
 _get_xmp_info(){
-    WAV="${INPUT}"
-    XMP_SIZE=$(mediaconch -mt "${WAV}" | xmlstarlet sel -N "mt=https://mediaarea.net/mediatrace" -t -m "mt:MediaTrace/mt:media/mt:block[@name='Wave']/mt:block[@name='_PMX']" -v "mt:block[@name='Header']/mt:data[@name='Size']" -n)
-    XMP_OFFSET=$(mediaconch -mt "${WAV}" | xmlstarlet sel -N "mt=https://mediaarea.net/mediatrace" -t -m "mt:MediaTrace/mt:media/mt:block[@name='Wave']/mt:block[@name='_PMX']" -v "mt:data/@offset" -n)
+    XMP_SIZE=$(mediaconch -mt "${INPUT}" | xmlstarlet sel -N "mt=https://mediaarea.net/mediatrace" -t -m "mt:MediaTrace/mt:media/mt:block[@name='Wave']/mt:block[@name='_PMX']" -v "mt:block[@name='Header']/mt:data[@name='Size']" -n)
+    XMP_OFFSET=$(mediaconch -mt "${INPUT}" | xmlstarlet sel -N "mt=https://mediaarea.net/mediatrace" -t -m "mt:MediaTrace/mt:media/mt:block[@name='Wave']/mt:block[@name='_PMX']" -v "mt:data/@offset" -n)
     XMP_OFFSET=$(echo "${XMP_OFFSET} + 1" | bc)
-    XMP_DATA=$(tail -c "+${XMP_OFFSET}" "$WAV" | head -c "${XMP_SIZE}" | xml fo)
+    XMP_DATA=$(tail -c "+${XMP_OFFSET}" "${INPUT}" | head -c "${XMP_SIZE}" | xmlstarlet fo)
     GET_INFO=$(echo "${XMP_DATA}" | xmlstarlet sel -N rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" -N xmpDM="http://ns.adobe.com/xmp/1.0/DynamicMedia/" -t -m "//rdf:li[@rdf:parseType='Resource'][xmpDM:trackName='CuePoint Markers']" -o "inSample:" -v "xmpDM:markers/rdf:Seq/rdf:li[@rdf:parseType='Resource']/xmpDM:startTime" -n -o "durationSample:" -v "xmpDM:markers/rdf:Seq/rdf:li[@rdf:parseType='Resource']/xmpDM:duration" -n)
     INSAMPLE=$(echo "${XMP_DATA}" | xmlstarlet sel -N rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" -N xmpDM="http://ns.adobe.com/xmp/1.0/DynamicMedia/" -t -m "//rdf:li[@rdf:parseType='Resource'][xmpDM:trackName='CuePoint Markers']" -v "xmpDM:markers/rdf:Seq/rdf:li[@rdf:parseType='Resource']/xmpDM:startTime" -n)
     DURATIONSAMPLE=$(echo "${XMP_DATA}" | xmlstarlet sel -N rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" -N xmpDM="http://ns.adobe.com/xmp/1.0/DynamicMedia/" -t -m "//rdf:li[@rdf:parseType='Resource'][xmpDM:trackName='CuePoint Markers']" -v "xmpDM:markers/rdf:Seq/rdf:li[@rdf:parseType='Resource']/xmpDM:duration" -n)
@@ -115,6 +114,30 @@ _get_xmp_info(){
     echo "The intime in samples is $INTIME, the outtime in seconds is $OUTTIME, and the duration in seconds is $DURATION."
 }
 
+_audioreport(){
+    ffmpeg -nostdin -i "${INPUT}" -vn -filter_complex "silencedetect,astats,loudnorm=print_format=json,agate=attack=0.01:release=0.01:ratio=9000,aphasemeter=r=0.2:video=0,ametadata=mode=print:key=lavfi.aphasemeter.phase:value=-0.18:function=less" -f null - 2>&1 |  tr "\r" "\n" > /tmp/loud
+    MEASURED_I=$(cat /tmp/loud | grep "input_i" | awk -F\" '{print $4}')
+    MEASURED_LRA=$(cat /tmp/loud | grep "input_lra" | awk -F\" '{print $4}')
+    MEASURED_TP=$(cat /tmp/loud | grep "input_tp" | awk -F\" '{print $4}')
+    MEASURED_THRESH=$(cat /tmp/loud | grep "input_thresh" | awk -F\" '{print $4}')
+    TARGET_OFFSET=$(cat /tmp/loud | grep "target_offset" | awk -F\" '{print $4}')
+    BWFMETAEDIT=$(bwfmetaedit "${INPUT}" 2>&1 | awk '{printf "%s+",$0} END {print ""}')
+    FLATFACTOR=$(cat /tmp/loud | grep "Flat factor: " | tail -n +3 | head -n 1 | cut -d ":" -f 2 | sed 's/ //g')
+    echo "  MEASURED_I = ${MEASURED_I}"
+    echo "  MEASURED_LRA = ${MEASURED_LRA}"
+    echo "  MEASURED_TP = ${MEASURED_TP}"
+    echo "  MEASURED_THRESH = ${MEASURED_THRESH}"
+    echo "  TARGET_OFFSET = ${TARGET_OFFSET}"
+    echo "  BWFMETAEDIT = ${BWFMETAEDIT}"
+    if [[ "${FLATFACTOR}" != 0.000000 ]] ; then
+        _report -wt "WARNING - There are ${FLATFACTOR} flat sections of waveform that may indicate clipped audio."
+        _report -qn "Enter q to quit, any other key to continue: "
+        read A4
+        [ "${A4}" == "q" ] && exit 0
+        _writeingestlog "Flat sections of audio waveform" "${FLATFACTOR}"
+    fi
+}
+
 _blackframetest(){
     _black_at_ends "${INPUT}"
     if [ "${HEAD_BLACK}" -gt "30" ] ; then
@@ -368,6 +391,7 @@ AUDIODECISION.option =
 AUDIODECISION.option = Default audio mapping [first audio track used for stereo output]
 AUDIODECISION.option = Only use left of the first audio track [for 21st Century]
 AUDIODECISION.option = Only use right of the first audio track
+AUDIODECISION.option = Only use the first track
 AUDIODECISION.width = 400
 
 # cropping strategy
@@ -501,7 +525,7 @@ fi
     if [[ -z "${AUDIODECISION}" ]] ; then
         _report -q "Select an audio strategy?"
         PS3="Selection: "
-        select AUDIODECISION in "Default audio mapping [first audio track used for stereo output]" "Only use left of the first audio track [for 21st Century]" "Only use right of the first audio track"
+        select AUDIODECISION in "Default audio mapping [first audio track used for stereo output]" "Only use left of the first audio track [for 21st Century]" "Only use right of the first audio track" "Only use the first track"
         do
             break
         done
@@ -571,7 +595,9 @@ if [[ "${MODE}" == "preservation" ]] ; then
     if [[ "${FORMULA}" == "check interlacement" ]] ; then
         _get_interlacement_full "${INPUT}"
     fi
-elif [[ "${MODE}" != "audio" ]] ; then
+elif [[ "${MODE}" == "audio" ]] ; then
+    _audioreport
+else
     # presumes default, psa, or local mode
     _blackframetest
     _phasechecktest
@@ -629,11 +655,12 @@ else
     fi
 fi
 
+# set makederivopts
 unset MAKEDERIVOPTS
-[ "${AUDIODECISION}" == "Only use left of the first audio track [for 21st Century]" -o "${AUDIODECISION}" == "left" ] && MAKEDERIVOPTS+=(-l)
-[ "${AUDIODECISION}" == "Only use right of the first audio track" ] && MAKEDERIVOPTS+=(-r)
-[ "${CROPDECISION}" == "Examine the video and auto-crop out black borders (slower)" ] && MAKEDERIVOPTS+=(-c)
-if [ ! -z "${FORMULA}" ] ; then
+if [[ "${CROPDECISION}" == "Examine the video and auto-crop out black borders (slower)" ]] ; then
+    MAKEDERIVOPTS+=(-c)
+fi
+if [[ ! -z "${FORMULA}" ]] ; then
     MAKEDERIVOPTS+=(-F "${FORMULA}")
 fi