safunctions

#!/usr/bin/env bash

#*****************************************************************************************
# Parse stdin arguments from sasync command line
#*****************************************************************************************
process_cmd_args_and_clean_set_files()
{ . sasync.conf.default
  . sasync.conf
  if ${NOTIF_ALL,,};then notif_all='| tee >($NOTIF_COMM)';else notif_all="";fi
  if ${NOTIF_KEY,,};then notif_key='| tee >($NOTIF_COMM)';else notif_key="";fi
  . rccheck
  saconf="sasync.conf"
  rcmd="gclone"
  unset gflag
  par=false
  args="$@"
  while getopts 'c:p:P:n:tglf' o;do
    case "$o" in
      c) saconf="${OPTARG}";source $saconf ;;
      p|P|n) par=true;n=${OPTARG} ;;
      t) tmux new "$SASDIR/sasync ${args/-t/} $gflag" ;;
      g) rcmd="gclone";gflag="-g" ;;
      l) rcmd="lclone";gflag="-l" ;;
      f) rcmd="fclone";gflag="-f" ;;
    esac
  done
  shift "$(($OPTIND - 1))"
  args="$@"
  files="${args%% -*}"
  if [ -z "$files" ];then . msg no_files;exit;fi
  eval . msg start $notif_key
  tmp_files=$SETDIR/tmp_"${files// /_}"_
  cmd_flags="${args/$files/}"
  clean_set_files
}

#*****************************************************************************************
# Clean comments and blank lines from set files. Parse , and | delimited files
#*****************************************************************************************
clean_set_files()
{ for file in $files; do
    if [ -v "$file" ];then set_file+=$(echo "${!files}")
    elif [ -f "$SETDIR/$file" ]; then set_file+=$(<$SETDIR/$file)$'\n'
    else . msg no_file;exit;fi
  done

  set_clean=$(echo "$set_file" | sed $'/^#/d; /^$/d; s/[\t\r]/ /g; s/\"//g')
  #if echo "$set_clean" | grep -E '\||,' &>/dev/null;then IFS1=',|';fi
  echo -e "$set_clean" | column -s"$IFS1" -t
  if [ ${par,,} == "true" ];then cd $SASDIR;parallel_sasync;exit;fi
  init_log_n_bak
}

#*****************************************************************************************
# Initialize log and config backup files. Delete older files
#*****************************************************************************************
init_log_n_bak()
{ STD_LOG="$SASDIR"/logs/$(date +%F)_stdout.log
  ERR_LOG="$SASDIR"/logs/$(date +%F)_stderr.log
  FAIL_LOG="$SASDIR"/logs/$(date +%F)_"$files"_fail.log;>"$FAIL_LOG"

  if [ ! -f $SASDIR/json.count ];then echo $MINJS >$SASDIR/json.count;fi
  mkdir -p $SASDIR/backup $SASDIR/logs
  cp $SASDIR/sasync.conf "$SASDIR/backup/$(date +%F)_sasync.conf"
  find $SASDIR/backup/* -type f -mtime +14 -exec rm -f {} \;
  find $SASDIR/logs/* -type f -mtime +7 -exec rm -f {} \;
  mkdir -p ~/.parallel
  touch ~/.parallel/will-cite
  tm_ck=`tmux --help &>/dev/null;echo $?`
}

#*****************************************************************************************
# sizeup is a python app that check size and file count in remotes then uploads to a gsheet
#*****************************************************************************************
run_sizeup()
{ if "${SIZEUP,,}";then cd $SASDIR/utils; python3 sizeup "$1";fi ; }
# { if "${SIZEUP,,}";then cd $SASDIR/utils; python3 sizeup "$dest";fi ; }

#*****************************************************************************************
# Clean remotes - dedupe identical files, remove empty folders and permanently deletes trash
#*****************************************************************************************
cleanup ()
{ . msg dedupe "$1"
  rclone dedupe skip "$1" $3 -v --drive-use-trash=false --tpslimit=5 --max-duration=30m
  . msg perm_del "$1"
  rclone delete "$2": $3 -v --fast-list --drive-trashed-only --drive-use-trash=false --max-duration=10m
  rclone cleanup "$1" -v $3
  rclone rmdirs "$1" $3 -v --drive-use-trash=false --fast-list --leave-root --max-duration=10m
}

clean_remotes()
{ if ${CLEAN_DEST,,};then
    if "${rem_sa[1]}";then sa_dest="$dsaff";else unset sa_dest;fi
    cleanup "$dest" "${dest%:*}" "$sa_dest"
  fi
  if ${CLEAN_SRC,,};then
    if "${rem_sa[0]}";then sa_src="$dsaff";else unset sa_src;fi
    cleanup "$src" "${src%:*}" "$sa_src"
  fi
  # run_sizeup
}

#*****************************************************************************************
# Start multiple instances of sasync in tmux, starting new set pairs as old ones finish
#*****************************************************************************************
parallel_sasync()
{ . msg start_par
  rm -f $tmp_files*
  echo "$set_clean">"$tmp_files"
  split -l 1 "$tmp_files" "$tmp_files"
  basename -a $tmp_files[a-z]* | parallel -j $n --tmux ./sasync $gflag -c $saconf {} $cmd_flags
  rm -f $tmp_files*
  . msg fin_par
}

#*****************************************************************************************
# sacalc calculates the size and difference of source and destination remotes
#*****************************************************************************************
sacalc()
{ . msg ck_size
  srcsize=`eval rclone size "$src" --json | cut -d ":" -f3 | cut -d "}" -f1`
  if [[ $srcsize -eq 0 ]];then $CON_EX;fi
  srcsizeGB=$(( $srcsize/1024**3 ))

  destsize=`eval rclone size "$dest" --json | cut -d ":" -f3 | cut -d "}" -f1`
  destsizeGB=$(( $destsize/1024**3 ))

  diff=$(( $srcsize-$destsize ))
  diffGB=$(( $diff/1024**3 ))

  SRC_LIMIT=${SRC_LIMIT%%[A-Za-z]*}
  LIM=${SRC_LIMIT:-$diffGB}
  LIM=$((diffGB>LIM?LIM:diffGB))
  MAXT=750
  SAs=$(( ($MAXT+${LIM#-})/$MAXT ))
  . msg diff_size
}

#*****************************************************************************************
# Add 1 to counter, then use next service account in rclone remote ENV or config
#*****************************************************************************************
add_to_counter ()
{ COUNT=$(<$JSCOUNT)
  MOD=$(( $MAXJS - $MINJS + 1 ))
  echo $(( (($COUNT - $MINJS + $NEXTJS) % $MOD) + $MINJS ))>$JSCOUNT
  [ -f "$JSDIR"/"$COUNT".json ] || next_sa
}

export_js ()
{ set -x;export "RCLONE_CONFIG_"$1"_SERVICE_ACCOUNT_FILE="$JSDIR"/"$COUNT.json;set +x; }

next_sa ()
{ add_to_counter
  if "${rem_sa[0]}" && "${rem_sa[1]}" && "${CRYPT[1]}";then
    set -x;sa_flag="$dsaf$COUNT.json ";set +x
  else if "${rem_sa[0]}";then export_js "$SRC";fi
       if "${rem_sa[1]}";then export_js "$DEST";fi
  fi; ((SAs--))
}

#*****************************************************************************************
# Run rclone with flags from config file, set file and command line. Use SAs where authed
#*****************************************************************************************
run_rclone_w_flags ()
{ eval . msg run_rc $notif_key
  . msg print_flags
  while [[ $ret =~ ^(7|8)$ ]] && (($SAs));do
    next_sa
    set -x
    timeout $TIMEOUT $rcmd $sync "$src" "$dest" $sa_flag $FLAGS $bak $set_flags $cmd_flags
    ret=$?
    set +x
  done
  eval . msg fin_rc $notif_key
}

#*****************************************************************************************
# Process each line in set file. Check remotes, run rclone then *option* clean the remotes
#*****************************************************************************************
check_remotes_and_run_rclone ()
{ while IFS="$IFS1" read sync src dest set_flags;do
    check_remotes || continue
    run_rclone_w_flags
    if [ ${#TMUX} -gt 3 ];then clean_remotes;else (clean_remotes &) ;fi
    ((run_sizeup "$src" && run_sizeup "$dest") &)
  done <<<"$set_clean" > >(tee "$STD_LOG") 2> >(tee "$ERR_LOG" >&2)
}