update

#!/usr/bin/env bash

set -e

curl --silent https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json | jq -r .[] > ./data/disposable.txt
curl --silent https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/wildcard.json | jq -r .[] >> ./data/disposable.txt

tmp=$(mktemp -t freemailXXX)
cat ./data/disposable.txt \
    | sort \
    | uniq > $tmp
mv $tmp ./data/disposable.txt

curl --silent https://www.stopforumspam.com/downloads/toxic_domains_whole.txt > ./data/spammy.txt

tmp=$(mktemp -t freemailXXX)
cat ./data/spammy.txt \
    | sed "s/^[^[:alnum:]]//g" \
    | sort \
    | uniq > $tmp
mv $tmp ./data/spammy.txt

sources=$(cat ./data/sources.txt)
new_free=$(mktemp -t freemailXXX)
for source in $sources; do
    echo "$(curl --silent $source)" >> $new_free
done;

deduped=$(mktemp -t freemailXXX)
cat $new_free \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | sed "s/^[^[:alnum:]]//g" \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/free.txt \
    | comm -23 - ./data/disposable.txt \
    | comm -23 - ./data/spammy.txt > $deduped

# NOTE: Checking DNS records is pretty slow. Having dead domains doesn't seem
# like that bad of an issue.

# confirmed=$(mktemp -t freemailXXX)
# for domain in $(cat $deduped); do
#     echo "Checking $domain"
#     result=`dig +short mx $domain`
#     if [ -n "$result" ]; then
#         echo $domain >> $confirmed
#     fi
# done

tmp=$(mktemp -t freemailXXX)
cat $deduped ./data/free.txt \
    | sort \
    | uniq > $tmp
mv $tmp ./data/free.txt

echo 'Complete!'