dvc.yaml

stages:
  data_collection:
    cmd: >
        papermill
        nbs/01_data_collection.ipynb
        scripts/01_data_collection.ipynb
        --cwd nbs/
    deps:
    - nbs/01_data_collection.ipynb
    - pdm.lock
    - pyproject.toml
    - data/raw-data
    - .pdm.toml
    outs:
    - scripts/01_data_collection.ipynb
    - data/Concatenated_Orig_data.csv
  data_cleaning:
    cmd: >
        papermill
        nbs/02_data_cleaning.ipynb
        scripts/02_data_cleaning.ipynb
        --cwd nbs/
    deps:
    - nbs/02_data_cleaning.ipynb
    - pdm.lock
    - pyproject.toml
    - data/Concatenated_Orig_data.csv
    - .pdm.toml
    outs:
    - scripts/02_data_cleaning.ipynb
    - data/Concatenated_Clean_data.csv
  feature_extraction:
    cmd: >
        papermill
        nbs/03_feature_extraction.ipynb
        scripts/03_feature_extraction.ipynb
        --cwd nbs/
    deps:
    - nbs/03_feature_extraction.ipynb
    - pdm.lock
    - pyproject.toml
    - data/Concatenated_Clean_data.csv
    - .pdm.toml
    outs:
    - scripts/03_feature_extraction.ipynb 
    - data/splits
  training:
    cmd: >
        papermill
        nbs/05_training.ipynb
        scripts/05_training.ipynb
        -p TOKENIZER ${tokenizer}
        -p LEARNING_RATE ${learning_rate}
        -p BATCH_SIZE ${batch_size}
        -p EPOCHS ${epochs}
        -p KFOLD ${kfold}
        --cwd nbs/
    deps:
    - nbs/05_training.ipynb
    - data/splits/train/FAA-${kfold}.csv
    - data/splits/test/FAA-${kfold}.csv
    - data/splits/val/FAA-${kfold}.csv
    - pdm.lock
    - pyproject.toml
    - .pdm.toml
    outs:
    - model/
    - scripts/05_training.ipynb
  eval:
    cmd: >
        papermill
        nbs/06_inference.ipynb
        scripts/06_inference.ipynb
        -p KFOLD ${kfold}
        --cwd nbs/
    deps:
    - nbs/06_inference.ipynb
    - pdm.lock
    - pyproject.toml
    - .pdm.toml
    - data/splits/val/FAA-${kfold}.csv
    outs:
    - scripts/06_inference.ipynb
    - eval/plots/sklearn:
        persist: true
    metrics:
    - eval/metrics.json:
        persist: true
        cache: true
plots:
  - Confusion matrix:
      template: confusion
      x: actual
      y:
        eval/plots/sklearn/confusion_matrix.json: predicted