texttodict

#!/bin/bash
version=0.1.2
############################################################################
##                                                                        ##
##  texttodict: A shell-script to create words dictionary from pdf file   ##
##              using pdftotext utility                                   ##
##                                                                        ##
##  Author: Pavel Loskot, loskot@gmail.com                                ##
##                                                                        ##
##  Usage: "texttodict -help"                                             ##
##                                                                        ##
##  License: GNU/GPL version 2 or later. No other warranty.               ##
##                                                                        ##
##  Configuration file may appear in future versions.                     ##
##                                                                        ## 
##  Developed and tested with GNU bash version 5.2.15                     ##
##                                                                        ##
############################################################################

trap "echo :exiting;exit 1" SIGINT SIGTERM

set -e

export LC_CTYPE="en_US.utf8"

help_file="$0.1"

this="${0/#*\/}"

# check BASH version used is at least 5.0
printf "5.0\n${BASH_VERSION%.*}" | sort -C || echo "Warning use BASH >= 5.0"

req_commands="pdftotext aspell" # external dependencies

# check dependencies on external commands
for c in $req_commands; do
    hash "$c" 2>/dev/null || err_exit "$c required but not found"
done

pdftotext_cmd='/usr/bin/pdftotext -eol unix -enc UTF-8 -nodiag -nopgbrk -q'

aspell_dict="en_US" # aspell dictionary to use

aspell_cmd="aspell --master=$aspell_dict list \
--ignore 3 --ignore-case --ignore-accents --sug-mode normal"

# check aspell English dictionary is available
$aspell_cmd <<< "" 2> /dev/null ||
    err_exit "English dictionary for aspell not found."

# message to stdout (for debuggin)
DM() { echo -e "$@" >&1; }

# messages to stdout
msg0() { echo -n ""; } # no messages
msg1() { echo -e "$@" >&1; }  # stdout
[[ "$@" =~ -nomsg ]] && MSG=msg0 || MSG=msg1

# exit ok
ok_exit() { $MSG "$1" ; exit 0; }

# exit with error
err_exit() { echo "Error: $1" >&2; exit 1; }

# show help and quit
[[ "$1" == @(-h|-help) ]] && { cat "$help_file" | sed '1,/^#/d'; exit 0; }

# show version number
[[ "$1" == "-version" ]] &&
    ok_exit "Text to word dictionary $this version $version"

# initial checks
[[ "$#" == "0" ]] && err_exit "Use $this -help for instructions."
[[ "$#" == "1" ]] && err_exit "Nothing to do. Try $this -help"


############################################################################
#                                                                          #
#  Functions                                                               #
#                                                                          #
############################################################################

# build dictionary for aspell
# Usage: buildaspelldict filename [-l {lang}] [-o filename]
buildaspelldict() {
    [[ -f "$1" ]] || err_exit "File $1 not found."
    local fin="$1" lang fout
    if [[ "$(/usr/bin/file $fin)" =~ (pdf|PDF) ]]; then
	$pdftotext_cmd "$fin" "$fin.txt"
	$MSG "++ Converted $fin to $fin.txt"
	fin="$fin.txt"
    fi
    shift
    while [[ ! -z "$@" ]]; do
	case "$1" in
	    -l)
		lang="$2"
		shift 2;;
	    -o)
		fout="$2"
		shift 2;;
	    -nomsg)
		shift;;
	    *)
		err_exit "Unknown option $2 in buildaspelldict."
		;;
	esac
    done
    [[ -z "$lang" ]] && lang="$aspell_dict"
    [[ -z "$fout" ]] && fout="${fin%.*}.tdict"
    [[ "$fout" =~ /.+ ]] || fout="./$fout"    
    aspell --lang="$lang" create master "$fout" < "$fin"
}

# dump content of aspell dictionary
# Usage: dumpaspelldict -l {lang} -o {filename}
dumpaspelldict() {
    local lang="$aspell_dict" fout
    while [[ ! -z "$@" ]]; do
	case "$1" in
	    -l)
		lang="$2"
		shift 2;;
	    -o)
		fout="$2"
		shift 2;;
	    -nomsg)
		shift;;
	    *)
		err_exit "Unknown option $2 in dumpaspelldict."
		;;
	esac
    done
    [[ -z "$fout" ]] && fout="$lang.tdict"
    [[ -f "$fout" ]] && err_exit "File $fout already exists."
    aspell -d "${lang}" dump master | aspell -l "${lang}" expand | \
	tr -s '[:space:]' '\n' | LC_ALL=C sort -u > "$fout"
}


# obtain bag of words from input file 
# Usage: bow inputfile OPTIONS
#   -ef {filename} exclude stop words in given file
#   -es {string}   exclude space separated words in given string
#   -ep {pat}      removes words matching the pattern {pat}
#   -w {nubmber}   exclude words shorter than {number} of characters
#   -{number}      truncate the output to {number} of lines
#   -gt|-eq|lt|-ge|-le {number}  keep only words with given occurrences   
#   -o filename    output filename
#   -so            sort output by number of occurrences
#   -sa            sort output alphanumerically
#   -sar	   sort output alphanumerically in reverse order
#   -sl		   sort by length of strings
#   -slr	   sort by length of strings in reverse order
bow() { 
    [[ -f "$1" ]] || err_exit "File $1 not found."
    local fin="$1" s=o
    if [[ "$(/usr/bin/file $fin)" =~ (pdf|PDF) ]]; then
	[[ -f "${fin%.*}.txt" ]] \
	    && err_exit "File ${fin%.*}.txt already exists."
	$pdftotext_cmd "$fin" "${fin%.*}.txt"
	$MSG "++ Converted $fin to ${fin%.*}.txt"
	fin="${fin%.*}.txt"
    fi
    shift
    ww="$(cat "$fin" | tr -dc '[:print:]\n\r' | \
    	      sed 's/[/+=<>{}|(),*-.;:0-9"?!]/ /g' | sed 's/[][]/ /g' | \
	      sed 's/ /\n/g' | sed '/^\s*$/d' | \
	      sed '/^[0-9A-Z \t]\+$/!s/.*/\L&/')"
    ww="$(echo "$ww" | sort | uniq -c | sort -rnk1)"
    while [[ ! -z "$@" ]]; do
	case "$1" in
	    -ef)
		[[ -f "$2" ]] || err_exit "File $2 not found."
		ww0="$(comm -13 <(cat "$2" | sort) \
			   <(echo "$ww" | awk '{print $2}' | sort))"
		ww="$(echo "$ww" | grep -i -w "$ww0")"
		shift 2;;
	    -es)
		ww0="$(echo "$2" | sed 's/ /|/g')"
		ww="$(echo "$ww" | grep -vE "$ww0")"
		shift 2;;
	    -w)
		[[ "$2" =~ [0-9][0-9]* ]] \
		    || err_exit "Parameter -w must be followed by a number."
		ww="$(echo "$ww" | awk -v n="$2" 'length($2)>n{print $0}')"
		shift 2;;
	    -ep)
		[[ "${2::1}" == "-" ]] && err_exit "Missing pattern after -ep"
		ww="$(echo "$ww" | grep -ve "$2")"
		shift 2;;
	    -[0-9]*)
		ww="$(echo "$ww" | head $1)"
		shift;;
	    -gt)
		ww="$(echo "$ww" | awk -v v="$2" '$1>v{print $0}')"
		shift 2;;
	    -eq)
		ww="$(echo "$ww" | awk -v v="$2" '$1==v{print $0}')"
		shift 2;;
	    -lt)
		ww="$(echo "$ww" | awk -v v="$2" '$1<v{print $0}')"
		shift 2;;
	    -ge)
		ww="$(echo "$ww" | awk -v v="$2" '$1>=v{print $0}')"
		shift 2;;
	    -le)
		ww="$(echo "$ww" | awk -v v="$2" '$1<=v{print $0}')"
		shift 2;;
	    -o)
		local fout="$2"
		[[ -f "$fout" ]] && err_exit "Output file $fout exists."
		shift 2;;
	    -s*)
		s=${1:2}
		shift;;
	    -nomsg)
		shift;;
	    *)
		err_exit "Unkonwn parameter $1 in bow() function."
		;;
	esac
    done
    [[ -z "$fout" ]] && fout="${fin%.*}.tdict"
    [[ -f "$fout" ]] && err_exit "Output file $fout exists."
    [[ -z "$s" ]] && s=a
    # output
    touch "$fout"
    case "$s" in
	o)
	    echo "$ww" | awk '{ print $2}' > "$fout" ;;
	a)
	    echo "$ww" | awk '{ print $2}' | sort > "$fout" ;;
	ar)
	    echo "$ww" | awk '{ print $2}' | sort -r > "$fout" ;;
	l)
	    echo "$ww" | awk '{ print $2}' | \
		awk '{ print length, $0 }' | sort -n | \
		cut -d" " -f2- > "$fout";;
	lr)
	    echo "$ww" | awk '{ print $2}' | \
		awk '{ print length, $0 }' | sort -n -r | \
		cut -d" " -f2- > "$fout";;
	*)
	    err_exit "Unknown sort option $s in texttodict/bow."
	    ;;
    esac
}

############################################################################
#                                                                          #
#  Main loop of command processor                                          #
#                                                                          #
############################################################################

case "$1" in
    -dict)
	# Command: -dict build [-l lang] [-o filename] inputfile
	#    Call: buildaspelldict inputfile [-l {lang}] [-o filename]
	case "$2" in
	    build)
		shift 2
		inf="${@: -1}"
		[[ -f "$inf" ]] || err_exit "Input file $inf does not exist."
		set -- "${@:1:$(($#-1))}"
		$MSG "<< buildaspelldict $inf $@"
		buildaspelldict "$inf" "$@"
		[[ "$?" == "0" ]] && $MSG ">> buildaspelldict ok" \
			|| $MSG "-- buildaspelldict??"
		;;
	    dump)
		shift 2
		$MSG "<< dumpaspelldict $@"
		dumpaspelldict "$@"
		[[ "$?" == "0" ]] && $MSG ">> dumpaspelldict ok" \
			|| $MSG "-- dumpaspelldict??"
		;;
	    *)
		err_exit "Unknown option $2 for -dict command."
		;;
	esac
	;;
    -bow)
	# Command: -bow [OPTIONS] inputfile
	#    Call: bow inputfile [OPTIONS]
	shift
	inf="${@: -1}"
	[[ -f "$inf" ]] || err_exit "Input file $inf does not exist."
	set -- "${@:1:$(($#-1))}"
	$MSG "<< bow $inf $@"
	bow "$inf" "$@"
	[[ "$?" == "0" ]] && $MSG ">> bow ok" \
		|| $MSG "-- bow??"
	;;
    *)
	err_exit "Unknown command $1"
	;;
esac
[[ "$MSG" == "msg1" ]] && ok_exit ">> Processing terminated normally."