dvc.yaml

stages:
  make_dataset:
    desc: Create groups of datasets of different sizes & number of classes.
    cmd: python src/data/make_dataset.py
    deps:
    - src/data/make_dataset.py
    params:
    - dataset_kwargs
    - seeds
    outs:
    - data/generated
  configure_train:
    desc: Configure the model training pipeline.
    cmd: python src/models/configure_train.py
    deps:
    - src/models/configure_train.py
    params:
    - train
    outs:
    - ${train.clf_dict}
    - ${train.cross_validator}
  train:
    desc: Train models and get out-of-sample predicted probabilities on the training
      sets.
    cmd: python src/models/train.py
    deps:
    - data/generated/
    - src/models/train.py
    - ${train.clf_dict}
    - ${train.cross_validator}
    outs:
    - data/pred_probs/
  plot_avg_trace:
    desc: Plot average traces of noise matrices used for noisy label generation.
    cmd: python src/data/plot_avg_trace.py
    deps:
    - src/data/plot_avg_trace.py
    params:
    - dataset_kwargs.small.gamma
    plots:
    - data/images/avg_trace.svg
  get_avg_accuracy:
    desc: Get model performance metrics on test sets, with and without label errors.
    cmd: python src/models/avg_accuracy.py
    deps:
    - data/generated
    - src/models/avg_accuracy.py
    outs:
    - data/accuracy/results.csv:
        persist: true
  group_stats:
    desc: Summarize model performance metrics for each group of datasets.
    cmd: python src/models/group_stats.py
    deps:
    - data/accuracy/results.csv
    - src/models/group_stats.py
    metrics:
    - data/accuracy/results_group.csv:
        cache: false
    outs:
    - data/accuracy/results_group.tex:
        cache: false
  score_classes:
    desc: Compute class label quality scores for each example in a dataset.
    cmd: python src/evaluation/class_score.py
    deps:
    - data/pred_probs/
    - src/evaluation/class_score.py
    outs:
    - data/scores/class_scores.pkl
  aggregate:
    desc: Aggregate class label quality scores for all classes into a single score.
    cmd: python src/evaluation/score.py
    deps:
    - data/scores/class_scores.pkl
    - src/evaluation/aggregate.py
    - src/evaluation/score.py
    params:
    - eval
    outs:
    - data/scores/scores.pkl
  rank_metrics:
    desc: Compute label error detection metrics for aggregated scores.
    cmd: python src/evaluation/eval_ranking_metrics.py
    deps:
    - data/scores/scores.pkl
    - src/evaluation/eval_ranking_metrics.py
    outs:
    - data/scores/results.csv
  plot_metrics:
    desc: Plot the label error detection and ranking metrics for the aggregated scores.
    cmd: python src/evaluation/plot_metrics.py
    deps:
    - data/scores/results.csv
    - src/evaluation/plot_metrics.py
    plots:
    - data/images/scores/
    metrics:
    - data/scores/metrics.csv:
        cache: false