run_sf_nf_cse_cic_ids2018_v2

#!/usr/bin/env bash

cd `dirname $0`

tmp_dir=./tmp_ds_data
output_dir=./output
base_download_url="https://github.com/WickedElm/feature_residuals_with_pretraining/releases/download/v1.0.0"

# Create data directory if needed
if [ ! -d ${tmp_dir} ];
then
    echo "Data directory not found.  Executing ./download_data"
    ./download_data
fi

if [ ! -d ${output_dir} ];
then
    mkdir ${output_dir}
    chmod 755 ${output_dir}
fi

###
# NF-CSE-CIC-IDS2018-V2 SPARSE
###
training_file=conference_sf_nf_cse_cic_ids2018_v2_ae_tuning_train.pkl
validation_file=conference_sf_nf_cse_cic_ids2018_v2_ae_tuning_validation.pkl
test_file=conference_sf_nf_cse_cic_ids2018_v2_ae_tuning_test.pkl
save_prefix=sf_nf_cse_cic_ids2018_v2
dataset=sf_nf_cse_cic_ids2018_v2
dataset_path=`pwd`/datasets/nf-cse-cic-ids2018-v2/sf_nf_cse_cic_ids2018_v2/sf_nf_cse_cic_ids2018_v2.pkl

# Get date for results  dir
cdate=$(date +%Y%m%d)
cseconds=$(date +%s%N)
timestamp=${cdate}_${cseconds}
epochs="600"
pretraining_epochs="500"
clf_input_types="X L S XL XS LS XLS"
lambda_filter=0.0 # Not used for this model
tuning_lr=0.001 # Not used for this model
clf_adadelta_lr=0.001
main_experiment="nfv2_sparse"

for clf_input_type in `echo ${clf_input_types}`; do
    project=${timestamp}_${save_prefix}_${clf_input_type}
    experiment="${main_experiment}_${epochs}_${clf_input_type}"

    python lightning_train_validate.py \
        general.model=mtl_pretrain_learned_threshold_sarhan_clf.MtlPretrainLearnedThresholdSarhanClf \
        general.ae_model_input_dims=35 \
        general.feature_transformer=s_threshold_feature_transformer.SThresholdFeatureTransformer \
        general.lambda_filter=${lambda_filter} \
        general.project=${project} \
        general.experiment=${experiment} \
        general.dataset_path=${dataset_path} \
        general.save_prefix=${save_prefix} \
        general.num_epochs=${epochs} \
        general.pretraining_epochs=${pretraining_epochs} \
        general.lr=1 \
        general.clf_adadelta_lr=${clf_adadelta_lr} \
        general.tuning_lr=${tuning_lr} \
        general.data_module=netflow_conference_rescaling_data_module.NetflowConferenceRescalingDataModule \
        general.total_rows_threshold=500000 \
        general.reserve_type=ae_tuning \
        general.hidden_layer_size=12 \
        general.use_all_training_data=True \
        general.load_from_disk=True \
        general.save_data_prefix=conference \
        general.clf_input_type=${clf_input_type} \
        general.threshold_name=no_threshold \
        general.check_val_every_n_epoch=10 \
        general.batch_size=128
done