getData.txt

#!/bin/bash

# Set Environment variables
DATARAW=/scratch/groups/jzeitzer/UKBB/Data/Raw/
DATACOMM=/scratch/groups/jzeitzer/UKBB/Data/Commands/
INPUT=/scratch/groups/jzeitzer/UKBB/Data/Commands/inputUKBB.txt

# Export Environment variables
export DATARAW
export DATACOMM
export INPUT

# Load modules
#module purge
#pip install --upgrade pip # Upgrades pip version if required
#pip3 install --upgrade -r requirements.txt # Installs a known working set of dependencies, other package versions may also work
ml python/3.6.1
ml py-numpy/1.18.1_py36
ml py-scipy/1.4.1_py36
ml py-scikit-learn/0.24.2_py36
ml py-pandas/1.0.3_py36
ml java

# Chunk input file into 10 jobs
python3 /scratch/groups/jzeitzer/UKBB/Code/ukbb/chunkInput.py

cd $DATARAW

for i in $(seq 0 $(( ${NUMINPUT} - 1 ))); do
    for FILE in input_$i.txt; do
        echo ${FILE}
        if [ $i \> 9 ];
        then
            Nm=$(($i - 10))
            sbatch -n 1 -t 18:00:00 --wrap="./ukbfetch -b${FILE} -ak63099r51627.key" --job-name=getDATA_${i} --dependency=afterany:getDATA_${Nm}
        else
            sbatch -n 1 -t 18:00:00 --wrap="./ukbfetch -b${FILE} -ak63099r51627.key" --job-name=getDATA_${i}
        fi;
    done
done