CWA2CSV.txt

#!/bin/bash
#SBATCH --time=01:00:00
#SBATCH --nodes=1
#SBATCH --job-name=processCmds

# Set file paths
DATARAW=/scratch/groups/jzeitzer/UKBB/Data/Raw/
DATAOUTPUTS=/scratch/groups/jzeitzer/UKBB/Data/Outputs/
DATACOMM=/scratch/groups/jzeitzer/UKBB/Data/Commands/
CODEBAA=/scratch/groups/jzeitzer/UKBB/Code/biobankAccelerometerAnalysis
DATACSV=/scratch/groups/jzeitzer/UKBB/Data/Outputs/timeSeries

# Make sure directories exist
mkdir -p $DATARAW
mkdir -p $DATAOUTPUTS
mkdir -p $DATACOMM
mkdir -p $DATACSV
mkdir -p $DATAOUTPUTS/summary  
mkdir -p $DATAOUTPUTS/epoch        
mkdir -p $DATAOUTPUTS/nonWear  
mkdir -p $DATAOUTPUTS/stationary  
mkdir -p $DATAOUTPUTS/raw  
mkdir -p $DATAOUTPUTS/npy  

# Export environment variables
export DATARAW
export DATAOUTPUTS
export DATACOMM
export CODEBAA
export DATACSV

# Load modules
module purge
cd $CODEBAA
bash utilities/downloadDataModels.sh
pip install --upgrade pip # Upgrades pip version if required
pip3 install --upgrade -r requirements.txt # Installs a known working set of dependencies, other package versions may also work
ml python/3.6.1
ml py-numpy/1.18.1_py36
ml py-scipy/1.4.1_py36
ml py-scikit-learn/0.24.2_py36
ml py-pandas/1.0.3_py36
ml java
javac -cp java/JTransforms-3.1-with-dependencies.jar java/*.java 


# Generate Process Commands 
cd $DATACOMM
python3 /scratch/groups/jzeitzer/UKBB/Code/ukbb/processCommandsMod.py 


# Submit Jobs for converting each file to .csv
cd $CODEBAA
for FILE in $DATACOMM/processCmds_*.txt; do
    echo ${FILE}
    sbatch -n 1 -t 1-00:00:00 --job-name=CWA2CSV --wrap="bash ${FILE}"
done