From 7c8baca01d6ecb28a2eff99f7e0b3270e20cfae8 Mon Sep 17 00:00:00 2001 From: loveligo97 Date: Wed, 11 Nov 2026 16:03:21 +0800 Subject: [PATCH] chore: prepare official Meta-Radiology code release --- .DS_Store | Bin 8196 -> 0 bytes .github/workflows/ci.yml | 33 + .gitignore | 28 + CITATION.cff | 45 + README.md | 267 ++- REPRODUCIBILITY.md | 150 ++ analysis/__init__.py | 1 + analysis/aggregate_results.py | 94 + analysis/export_tabular_auc.py | 141 ++ analysis/finalize_revision_results.py | 360 ++++ analysis/merge_result_shards.py | 83 + analysis/plot_budget_trends.py | 279 +++ analysis/statistical_tests.py | 86 + analysis/tabular_legacy_statistics.py | 128 ++ attacks/__init__.py | 15 + attacks/_utils.py | 43 + attacks/augmentation.py | 55 + attacks/denoise.py | 20 + attacks/finetune.py | 59 + attacks/jpeg.py | 35 + attacks/smoothing.py | 26 + configs/.DS_Store | Bin 6148 -> 0 bytes configs/MedMNIST/.DS_Store | Bin 10244 -> 0 bytes configs/MedMNIST/ChestMNIST/resnet18.yaml | 29 + configs/pipelines/bloodmnist.yaml | 35 + configs/pipelines/breastmnist.yaml | 34 + configs/pipelines/breastmnist224.yaml | 35 + configs/pipelines/chestmnist.yaml | 50 + configs/pipelines/dermamnist.yaml | 34 + configs/pipelines/fracturemnist3d.yaml | 34 + configs/pipelines/medmnist3d_four.yaml | 6 + configs/pipelines/medmnist3d_smoke.yaml | 3 + configs/pipelines/medmnist_224_three.yaml | 5 + .../pipelines/medmnist_additional_two.yaml | 5 + configs/pipelines/medmnist_extra_three.yaml | 5 + configs/pipelines/medmnist_multilabel.yaml | 3 + configs/pipelines/medmnist_three.yaml | 5 + configs/pipelines/nodulemnist3d.yaml | 34 + configs/pipelines/organcmnist.yaml | 34 + configs/pipelines/organmnist3d.yaml | 34 + configs/pipelines/organmnist3d_smoke.yaml | 34 + configs/pipelines/pneumoniamnist.yaml | 34 + configs/pipelines/pneumoniamnist224.yaml | 35 + configs/pipelines/retinamnist.yaml | 34 + configs/pipelines/retinamnist224.yaml | 35 + configs/pipelines/synapsemnist3d.yaml | 34 + configs/revision.yaml | 9 + configs/revision/224/generated.yaml | 31 + configs/revision/224/generated/attacks.yaml | 443 ++++ .../224/generated/attacks_finetune.yaml | 445 ++++ .../generated/attacks_finetune_budget.yaml | 479 +++++ .../224/generated/final_analysis.yaml | 12 + .../revision/224/generated/image_quality.yaml | 119 ++ configs/revision/224/generated/pseudo.yaml | 139 ++ configs/revision/224/generated/runtime.yaml | 71 + configs/revision/3d/eps32/generated.yaml | 31 + .../revision/3d/eps32/generated/runtime.yaml | 95 + configs/revision/3d/generated.yaml | 31 + configs/revision/3d/generated/attacks.yaml | 2 + .../3d/generated/attacks_finetune.yaml | 4 + .../3d/generated/attacks_finetune_budget.yaml | 2 + .../revision/3d/generated/final_analysis.yaml | 9 + .../revision/3d/generated/image_quality.yaml | 470 +++++ configs/revision/3d/generated/pseudo.yaml | 4 + configs/revision/3d/generated/runtime.yaml | 2 + configs/revision/attacks.yaml | 12 + configs/revision/chestmnist/generated.yaml | 31 + .../chestmnist/generated/attacks.yaml | 149 ++ .../generated/attacks_finetune.yaml | 151 ++ .../generated/attacks_finetune_budget.yaml | 161 ++ .../chestmnist/generated/final_analysis.yaml | 10 + .../chestmnist/generated/image_quality.yaml | 41 + .../revision/chestmnist/generated/pseudo.yaml | 49 + .../chestmnist/generated/runtime.yaml | 25 + configs/revision/epsilon.yaml | 9 + configs/revision/exports.yaml | 12 + .../extensions/extra2d/generated.yaml | 31 + .../extensions/extra2d/generated/attacks.yaml | 1178 +++++++++++ .../extra2d/generated/attacks_finetune.yaml | 868 ++++++++ .../generated/attacks_finetune_budget.yaml | 1274 ++++++++++++ .../extra2d/generated/final_analysis.yaml | 12 + .../extra2d/generated/image_quality.yaml | 1562 ++++++++++++++ .../extensions/extra2d/generated/pseudo.yaml | 1804 +++++++++++++++++ .../extensions/extra2d/generated/runtime.yaml | 71 + .../extensions/retina-eps12/generated.yaml | 31 + .../retina-eps12/generated/attacks.yaml | 394 ++++ .../generated/attacks_finetune.yaml | 292 +++ .../generated/attacks_finetune_budget.yaml | 426 ++++ .../generated/final_analysis.yaml | 10 + .../retina-eps12/generated/image_quality.yaml | 106 + .../retina-eps12/generated/pseudo.yaml | 124 ++ .../retina-eps12/generated/runtime.yaml | 2 + .../extensions/seed3-7-all/generated.yaml | 31 + .../seed3-7-all/generated/attacks.yaml | 737 +++++++ .../generated/attacks_finetune.yaml | 739 +++++++ .../generated/attacks_finetune_budget.yaml | 797 ++++++++ .../seed3-7-all/generated/final_analysis.yaml | 12 + .../seed3-7-all/generated/image_quality.yaml | 977 +++++++++ .../seed3-7-all/generated/pseudo.yaml | 1129 +++++++++++ .../seed3-7-all/generated/runtime.yaml | 2 + .../extensions/seed3-7-eps8/generated.yaml | 31 + .../seed3-7-eps8/generated/attacks.yaml | 737 +++++++ .../generated/attacks_finetune.yaml | 544 +++++ .../generated/attacks_finetune_budget.yaml | 797 ++++++++ .../generated/final_analysis.yaml | 12 + .../seed3-7-eps8/generated/image_quality.yaml | 197 ++ .../seed3-7-eps8/generated/pseudo.yaml | 229 +++ .../seed3-7-eps8/generated/runtime.yaml | 2 + configs/revision/final_analysis.yaml | 24 + .../revision/final_analysis_2d_8seeds.yaml | 25 + .../final_analysis_2d_six_datasets.yaml | 29 + .../final_analysis_extra2d_8seeds.yaml | 23 + configs/revision/generated.yaml | 31 + configs/revision/generated/attacks.yaml | 443 ++++ .../revision/generated/attacks_finetune.yaml | 445 ++++ .../generated/attacks_finetune_budget.yaml | 479 +++++ .../revision/generated/final_analysis.yaml | 12 + configs/revision/generated/image_quality.yaml | 587 ++++++ configs/revision/generated/pseudo.yaml | 679 +++++++ configs/revision/generated/runtime.yaml | 71 + configs/revision/image_quality.yaml | 7 + configs/revision/merge_2d_8seeds.yaml | 23 + configs/revision/merge_2d_six_datasets.yaml | 36 + configs/revision/merge_tabular_epsilon.yaml | 25 + .../merge_tabular_legacy_extended.yaml | 25 + .../revision/merge_tabular_legacy_params.yaml | 25 + .../merge_tabular_legacy_params_7gpu.yaml | 45 + configs/revision/metrics.yaml | 21 + configs/revision/pseudo.yaml | 6 + configs/revision/runtime.yaml | 4 + configs/revision/statistics.yaml | 7 + .../revision/statistics_tabular_epsilon.yaml | 13 + .../statistics_tabular_legacy_extended.yaml | 20 + .../statistics_tabular_legacy_params.yaml | 13 + .../revision/tabular_epsilon_diabetes.yaml | 38 + .../revision/tabular_epsilon_hepatitis_c.yaml | 50 + .../tabular_epsilon_systemic_diseases.yaml | 64 + .../tabular_legacy_exact_diabetes.yaml | 44 + .../tabular_legacy_exact_hepatitis_c.yaml | 51 + ...abular_legacy_exact_systemic_diseases.yaml | 52 + .../tabular_legacy_extended_diabetes.yaml | 42 + .../tabular_legacy_extended_hepatitis_c.yaml | 49 + ...lar_legacy_extended_systemic_diseases.yaml | 49 + .../tabular_legacy_params_diabetes.yaml | 37 + .../tabular_legacy_params_hepatitis_c.yaml | 50 + ...bular_legacy_params_systemic_diseases.yaml | 63 + dataset.py | 40 +- evaluator.py | 140 +- export_revision_predictions.py | 58 + jobs/datacook-224-three-local.sh | 88 + jobs/datacook-2d-eight-seeds-local.sh | 120 ++ jobs/datacook-3d-four-local.sh | 47 + jobs/datacook-3d-pseudo-runtime-local.sh | 66 + jobs/datacook-3d-quality-local.sh | 74 + jobs/datacook-3d-smoke-local.sh | 40 + jobs/datacook-bloodmnist-revision-local.sh | 52 + jobs/datacook-chestmnist-local.sh | 86 + ...tacook-extra-three-2d-eight-seeds-local.sh | 87 + jobs/datacook-finetune-attacks-local.sh | 64 + ...cook-finetune-budget-and-analysis-local.sh | 71 + .../datacook-medmnist-additional-two-local.sh | 52 + jobs/datacook-merge-six-2d-results-local.sh | 32 + jobs/datacook-recompute-psnr-2d-local.sh | 67 + jobs/datacook-revision-posthoc-local.sh | 70 + jobs/datacook-synapse3d-rerun-local.sh | 77 + jobs/datacook-tabular-epsilon-local.sh | 93 + jobs/datacook-tabular-legacy-exact-local.sh | 68 + .../datacook-tabular-legacy-extended-local.sh | 98 + ...tacook-tabular-legacy-params-5gpu-local.sh | 137 ++ ...tacook-tabular-legacy-params-7gpu-local.sh | 131 ++ jobs/datacook-tabular-legacy-params-local.sh | 81 + main.py | 38 +- metrics/__init__.py | 12 + metrics/classification_metrics.py | 282 +++ metrics/cp_pp_metrics.py | 84 + metrics/image_quality.py | 223 ++ models/.DS_Store | Bin 6148 -> 0 bytes models/__init__.py | 2 +- perturbation.py | 308 ++- prepare_revision_posthoc.py | 910 +++++++++ pytest.ini | 3 + requirements-dev.txt | 2 + requirements.txt | 15 +- revision_experiments/__init__.py | 1 + revision_experiments/common.py | 244 +++ revision_experiments/modeling.py | 121 ++ run_adaptive_attacks.py | 203 ++ run_datacook_pipeline.py | 448 ++++ run_datacook_suite.py | 80 + run_epsilon_ablation.py | 50 + run_image_quality.py | 37 + run_pseudo_label_analysis.py | 152 ++ run_revision_experiments.py | 78 + run_runtime_analysis.py | 166 ++ run_tabular_epsilon_ablation.py | 1009 +++++++++ run_unified_evaluation.py | 77 + scripts/.DS_Store | Bin 6148 -> 0 bytes scripts/Datacook/.DS_Store | Bin 6148 -> 0 bytes scripts/README.md | 18 + summarize_tabular_legacy_reproduction.py | 82 + tests/test_3d_pipeline.py | 74 + tests/test_checkpoint_compatibility.py | 17 + tests/test_datacook_suite.py | 52 + tests/test_final_analysis.py | 76 + tests/test_high_resolution_pipeline.py | 68 + tests/test_merge_result_shards.py | 29 + tests/test_multilabel_pipeline.py | 100 + tests/test_perturbation_regressions.py | 17 + tests/test_prediction_export.py | 21 + tests/test_prepare_revision_posthoc.py | 282 +++ tests/test_revision_attacks.py | 51 + tests/test_revision_metrics.py | 134 ++ tests/test_runtime_analysis.py | 58 + tests/test_tabular_epsilon_ablation.py | 246 +++ tests/test_tabular_legacy_statistics.py | 25 + toolbox.py | 84 +- trainer.py | 37 +- util.py | 14 +- volume_io.py | 59 + 219 files changed, 32189 insertions(+), 291 deletions(-) delete mode 100644 .DS_Store create mode 100644 .github/workflows/ci.yml create mode 100644 CITATION.cff create mode 100644 REPRODUCIBILITY.md create mode 100644 analysis/__init__.py create mode 100644 analysis/aggregate_results.py create mode 100644 analysis/export_tabular_auc.py create mode 100644 analysis/finalize_revision_results.py create mode 100644 analysis/merge_result_shards.py create mode 100644 analysis/plot_budget_trends.py create mode 100644 analysis/statistical_tests.py create mode 100644 analysis/tabular_legacy_statistics.py create mode 100644 attacks/__init__.py create mode 100644 attacks/_utils.py create mode 100644 attacks/augmentation.py create mode 100644 attacks/denoise.py create mode 100644 attacks/finetune.py create mode 100644 attacks/jpeg.py create mode 100644 attacks/smoothing.py delete mode 100644 configs/.DS_Store delete mode 100644 configs/MedMNIST/.DS_Store create mode 100644 configs/MedMNIST/ChestMNIST/resnet18.yaml create mode 100644 configs/pipelines/bloodmnist.yaml create mode 100644 configs/pipelines/breastmnist.yaml create mode 100644 configs/pipelines/breastmnist224.yaml create mode 100644 configs/pipelines/chestmnist.yaml create mode 100644 configs/pipelines/dermamnist.yaml create mode 100644 configs/pipelines/fracturemnist3d.yaml create mode 100644 configs/pipelines/medmnist3d_four.yaml create mode 100644 configs/pipelines/medmnist3d_smoke.yaml create mode 100644 configs/pipelines/medmnist_224_three.yaml create mode 100644 configs/pipelines/medmnist_additional_two.yaml create mode 100644 configs/pipelines/medmnist_extra_three.yaml create mode 100644 configs/pipelines/medmnist_multilabel.yaml create mode 100644 configs/pipelines/medmnist_three.yaml create mode 100644 configs/pipelines/nodulemnist3d.yaml create mode 100644 configs/pipelines/organcmnist.yaml create mode 100644 configs/pipelines/organmnist3d.yaml create mode 100644 configs/pipelines/organmnist3d_smoke.yaml create mode 100644 configs/pipelines/pneumoniamnist.yaml create mode 100644 configs/pipelines/pneumoniamnist224.yaml create mode 100644 configs/pipelines/retinamnist.yaml create mode 100644 configs/pipelines/retinamnist224.yaml create mode 100644 configs/pipelines/synapsemnist3d.yaml create mode 100644 configs/revision.yaml create mode 100644 configs/revision/224/generated.yaml create mode 100644 configs/revision/224/generated/attacks.yaml create mode 100644 configs/revision/224/generated/attacks_finetune.yaml create mode 100644 configs/revision/224/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/224/generated/final_analysis.yaml create mode 100644 configs/revision/224/generated/image_quality.yaml create mode 100644 configs/revision/224/generated/pseudo.yaml create mode 100644 configs/revision/224/generated/runtime.yaml create mode 100644 configs/revision/3d/eps32/generated.yaml create mode 100644 configs/revision/3d/eps32/generated/runtime.yaml create mode 100644 configs/revision/3d/generated.yaml create mode 100644 configs/revision/3d/generated/attacks.yaml create mode 100644 configs/revision/3d/generated/attacks_finetune.yaml create mode 100644 configs/revision/3d/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/3d/generated/final_analysis.yaml create mode 100644 configs/revision/3d/generated/image_quality.yaml create mode 100644 configs/revision/3d/generated/pseudo.yaml create mode 100644 configs/revision/3d/generated/runtime.yaml create mode 100644 configs/revision/attacks.yaml create mode 100644 configs/revision/chestmnist/generated.yaml create mode 100644 configs/revision/chestmnist/generated/attacks.yaml create mode 100644 configs/revision/chestmnist/generated/attacks_finetune.yaml create mode 100644 configs/revision/chestmnist/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/chestmnist/generated/final_analysis.yaml create mode 100644 configs/revision/chestmnist/generated/image_quality.yaml create mode 100644 configs/revision/chestmnist/generated/pseudo.yaml create mode 100644 configs/revision/chestmnist/generated/runtime.yaml create mode 100644 configs/revision/epsilon.yaml create mode 100644 configs/revision/exports.yaml create mode 100644 configs/revision/extensions/extra2d/generated.yaml create mode 100644 configs/revision/extensions/extra2d/generated/attacks.yaml create mode 100644 configs/revision/extensions/extra2d/generated/attacks_finetune.yaml create mode 100644 configs/revision/extensions/extra2d/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/extensions/extra2d/generated/final_analysis.yaml create mode 100644 configs/revision/extensions/extra2d/generated/image_quality.yaml create mode 100644 configs/revision/extensions/extra2d/generated/pseudo.yaml create mode 100644 configs/revision/extensions/extra2d/generated/runtime.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/attacks.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/attacks_finetune.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/final_analysis.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/image_quality.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/pseudo.yaml create mode 100644 configs/revision/extensions/retina-eps12/generated/runtime.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/attacks.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/attacks_finetune.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/final_analysis.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/image_quality.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/pseudo.yaml create mode 100644 configs/revision/extensions/seed3-7-all/generated/runtime.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/attacks.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/final_analysis.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/image_quality.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/pseudo.yaml create mode 100644 configs/revision/extensions/seed3-7-eps8/generated/runtime.yaml create mode 100644 configs/revision/final_analysis.yaml create mode 100644 configs/revision/final_analysis_2d_8seeds.yaml create mode 100644 configs/revision/final_analysis_2d_six_datasets.yaml create mode 100644 configs/revision/final_analysis_extra2d_8seeds.yaml create mode 100644 configs/revision/generated.yaml create mode 100644 configs/revision/generated/attacks.yaml create mode 100644 configs/revision/generated/attacks_finetune.yaml create mode 100644 configs/revision/generated/attacks_finetune_budget.yaml create mode 100644 configs/revision/generated/final_analysis.yaml create mode 100644 configs/revision/generated/image_quality.yaml create mode 100644 configs/revision/generated/pseudo.yaml create mode 100644 configs/revision/generated/runtime.yaml create mode 100644 configs/revision/image_quality.yaml create mode 100644 configs/revision/merge_2d_8seeds.yaml create mode 100644 configs/revision/merge_2d_six_datasets.yaml create mode 100644 configs/revision/merge_tabular_epsilon.yaml create mode 100644 configs/revision/merge_tabular_legacy_extended.yaml create mode 100644 configs/revision/merge_tabular_legacy_params.yaml create mode 100644 configs/revision/merge_tabular_legacy_params_7gpu.yaml create mode 100644 configs/revision/metrics.yaml create mode 100644 configs/revision/pseudo.yaml create mode 100644 configs/revision/runtime.yaml create mode 100644 configs/revision/statistics.yaml create mode 100644 configs/revision/statistics_tabular_epsilon.yaml create mode 100644 configs/revision/statistics_tabular_legacy_extended.yaml create mode 100644 configs/revision/statistics_tabular_legacy_params.yaml create mode 100644 configs/revision/tabular_epsilon_diabetes.yaml create mode 100644 configs/revision/tabular_epsilon_hepatitis_c.yaml create mode 100644 configs/revision/tabular_epsilon_systemic_diseases.yaml create mode 100644 configs/revision/tabular_legacy_exact_diabetes.yaml create mode 100644 configs/revision/tabular_legacy_exact_hepatitis_c.yaml create mode 100644 configs/revision/tabular_legacy_exact_systemic_diseases.yaml create mode 100644 configs/revision/tabular_legacy_extended_diabetes.yaml create mode 100644 configs/revision/tabular_legacy_extended_hepatitis_c.yaml create mode 100644 configs/revision/tabular_legacy_extended_systemic_diseases.yaml create mode 100644 configs/revision/tabular_legacy_params_diabetes.yaml create mode 100644 configs/revision/tabular_legacy_params_hepatitis_c.yaml create mode 100644 configs/revision/tabular_legacy_params_systemic_diseases.yaml create mode 100644 export_revision_predictions.py create mode 100755 jobs/datacook-224-three-local.sh create mode 100755 jobs/datacook-2d-eight-seeds-local.sh create mode 100644 jobs/datacook-3d-four-local.sh create mode 100644 jobs/datacook-3d-pseudo-runtime-local.sh create mode 100755 jobs/datacook-3d-quality-local.sh create mode 100644 jobs/datacook-3d-smoke-local.sh create mode 100755 jobs/datacook-bloodmnist-revision-local.sh create mode 100755 jobs/datacook-chestmnist-local.sh create mode 100755 jobs/datacook-extra-three-2d-eight-seeds-local.sh create mode 100755 jobs/datacook-finetune-attacks-local.sh create mode 100755 jobs/datacook-finetune-budget-and-analysis-local.sh create mode 100755 jobs/datacook-medmnist-additional-two-local.sh create mode 100755 jobs/datacook-merge-six-2d-results-local.sh create mode 100755 jobs/datacook-recompute-psnr-2d-local.sh create mode 100755 jobs/datacook-revision-posthoc-local.sh create mode 100755 jobs/datacook-synapse3d-rerun-local.sh create mode 100755 jobs/datacook-tabular-epsilon-local.sh create mode 100755 jobs/datacook-tabular-legacy-exact-local.sh create mode 100755 jobs/datacook-tabular-legacy-extended-local.sh create mode 100644 jobs/datacook-tabular-legacy-params-5gpu-local.sh create mode 100644 jobs/datacook-tabular-legacy-params-7gpu-local.sh create mode 100644 jobs/datacook-tabular-legacy-params-local.sh create mode 100644 metrics/__init__.py create mode 100644 metrics/classification_metrics.py create mode 100644 metrics/cp_pp_metrics.py create mode 100644 metrics/image_quality.py delete mode 100644 models/.DS_Store create mode 100755 prepare_revision_posthoc.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 revision_experiments/__init__.py create mode 100644 revision_experiments/common.py create mode 100644 revision_experiments/modeling.py create mode 100644 run_adaptive_attacks.py create mode 100644 run_datacook_pipeline.py create mode 100755 run_datacook_suite.py create mode 100644 run_epsilon_ablation.py create mode 100644 run_image_quality.py create mode 100644 run_pseudo_label_analysis.py create mode 100644 run_revision_experiments.py create mode 100644 run_runtime_analysis.py create mode 100644 run_tabular_epsilon_ablation.py create mode 100644 run_unified_evaluation.py delete mode 100644 scripts/.DS_Store delete mode 100644 scripts/Datacook/.DS_Store create mode 100644 scripts/README.md create mode 100755 summarize_tabular_legacy_reproduction.py create mode 100644 tests/test_3d_pipeline.py create mode 100644 tests/test_checkpoint_compatibility.py create mode 100644 tests/test_datacook_suite.py create mode 100644 tests/test_final_analysis.py create mode 100644 tests/test_high_resolution_pipeline.py create mode 100644 tests/test_merge_result_shards.py create mode 100644 tests/test_multilabel_pipeline.py create mode 100644 tests/test_perturbation_regressions.py create mode 100644 tests/test_prediction_export.py create mode 100644 tests/test_prepare_revision_posthoc.py create mode 100644 tests/test_revision_attacks.py create mode 100644 tests/test_revision_metrics.py create mode 100644 tests/test_runtime_analysis.py create mode 100644 tests/test_tabular_epsilon_ablation.py create mode 100644 tests/test_tabular_legacy_statistics.py create mode 100644 volume_io.py diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 875292f1870b4e8e397eb7bf084fb640c7dd3e8e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeI1zityj5XQfaqc{mDqyQUbg&hy)^mNFWl31pWmAT(hOD*R1=tHP(?pByc4O z$n&AaLu|*{!diWFu+l96?J7@e;b-mx)TiZa$JxSKb!DG+_h6i>aa#;y-Pzs}Ic>+; z!dmMNW8Gog$j0qZjE+vfmdar|*2X##hy<1i$lkq)Au=4{WqJRe4cNk)$$X$~|I(uF z6?HP%?GLkbrfqotowlM#vu<~gXn(v(i)Q=o*S(Nx^arZhtI}wVA2R4BMtFgo#{y&8 zHfA`n?;y@%SdoWCIfm<0zM)5gWA)825BXuAejiJ*uaQtnj|u(Vdyca)@cAOGN9pFz zPtd;)^WD5mMtuQ{Y3zG3UU2c^s&0a1bzedET4P; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c6dc79b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.8" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install torch==2.0.0 torchvision==0.15.1 --index-url https://download.pytorch.org/whl/cpu + pip install -r requirements-dev.txt + + - name: Check Python syntax + run: python -m compileall -q . + + - name: Check shell syntax + run: find jobs scripts -type f -name '*.sh' -print0 | xargs -0 -n1 bash -n + + - name: Run tests + run: pytest -q diff --git a/.gitignore b/.gitignore index 83972fa..3b77b90 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,31 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# macOS / editors +.DS_Store +Thumbs.db +*.swp + +# Local datasets and generated research artifacts +data/ +datasets/ +experiments/ +results/ +outputs/ +artifacts/ +revision_logs/ +runs/ +wandb/ + +# Model, prediction, and perturbation artifacts +*.ckpt +*.pt +*.pth +*.pkl +*.npz + +# Local tabular data and logs +TableData/dataset/ +*.csv +*.log diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..c1cf787 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,45 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite the DataCook paper." +type: software +title: "DataCook: Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control" +authors: + - family-names: Shang + given-names: Sihan + - family-names: Yan + given-names: Jun + - family-names: Shi + given-names: Danli + - family-names: Zheng + given-names: Haizhong + - family-names: Zheng + given-names: Shuangjia + - family-names: Zhao + given-names: Wei + - family-names: Yang + given-names: Jiancheng +year: 2026 +license: Apache-2.0 +repository-code: "https://github.com/HINTLab/DataCook" +url: "https://github.com/HINTLab/DataCook" +preferred-citation: + type: article + title: "DataCook: Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control" + authors: + - family-names: Shang + given-names: Sihan + - family-names: Yan + given-names: Jun + - family-names: Shi + given-names: Danli + - family-names: Zheng + given-names: Haizhong + - family-names: Zheng + given-names: Shuangjia + - family-names: Zhao + given-names: Wei + - family-names: Yang + given-names: Jiancheng + journal: Meta-Radiology + year: 2026 + url: "https://arxiv.org/abs/2403.17755" + notes: "Accepted for publication; journal DOI pending." diff --git a/README.md b/README.md index cf02cc5..8ca3c70 100644 --- a/README.md +++ b/README.md @@ -1,166 +1,225 @@ # DataCook -## Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control -An official implementation of the paper _DataCook_. +[![arXiv](https://img.shields.io/badge/arXiv-2403.17755-b31b1b.svg)](https://arxiv.org/abs/2403.17755) +[![License: Apache-2.0](https://img.shields.io/badge/License-Apache--2.0-yellow.svg)](LICENSE) -**Accepted by Meta-Radiology, 2026!** The open-access version of the paper can be downloaded from [Arxiv](https://arxiv.org/abs/2403.17755). **Feel free to clone our full project, test, and even make improvements on your own.** +Official implementation of **DataCook: Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control**, accepted by *Meta-Radiology* (2026). -### Overview -

-DataCook explores deployment-stage healthcare data usage control using anti-adversarial fingerprints. +Sihan Shang, Jun Yan, Danli Shi, Haizhong Zheng, Shuangjia Zheng, Wei Zhao, and Jiancheng Yang. -The framework applies a private cooking procedure to healthcare data before distribution. Models trained on cooked data can preserve predictive utility when evaluated on similarly cooked inputs, while their performance on raw inputs may be reduced. +> The journal DOI has not yet been assigned. The accepted paper and current metadata are available from [arXiv:2403.17755](https://arxiv.org/abs/2403.17755). -We evaluate DataCook on 2D, 3D, and high-resolution medical images, as well as structured healthcare tabular data. The results demonstrate the feasibility of authorization-dependent model use in several settings, while highlighting important task- and metric-dependent limitations. +## Overview -DataCook is presented as a proof-of-concept framework rather than a universally effective mechanism for preventing unauthorized data use. -

+DataCook explores deployment-stage healthcare data usage control using anti-adversarial fingerprints. A private cooking procedure is applied before data distribution. Models trained on cooked data may preserve predictive utility on similarly cooked inputs while performing less effectively on raw inputs. -![image](images/Overview.png) +The method is a **task-dependent proof of concept**, not a universal mechanism for preventing unauthorized data use. Its behavior depends on the dataset, modality, task, metric, model, and perturbation budget. +![DataCook overview](images/Overview.png) +## Included experiments ---- +| Component | Included scope | +| --- | --- | +| MedMNIST 2D | BloodMNIST, BreastMNIST, RetinaMNIST, PneumoniaMNIST, DermaMNIST, and OrganCMNIST | +| Multi-label imaging | ChestMNIST | +| MedMNIST3D | FractureMNIST3D, NoduleMNIST3D, OrganMNIST3D, and SynapseMNIST3D | +| MedMNIST+ 224x224 | BreastMNIST, PneumoniaMNIST, and RetinaMNIST | +| Structured medical data | Pima Diabetes, Hepatitis C, and Systemic Diseases | +| Evaluation | Accuracy, task-aware AUC, CP, PP, calibration, image quality, runtime, pseudo-label analysis, and paired exploratory statistics | +| Adaptive attacks | JPEG, smoothing, median filtering, augmentation, and limited-data fine-tuning | -## Table of Contents +The unified configurations in `configs/pipelines/` reproduce DataCook experiments. `attack_type: datacook_adv` selects the adversarial variant. Historical per-dataset launchers remain under `scripts/` for traceability during migration and are not used by the unified quick start below. -- [Environment Setup](#environment-setup) -- [Dataset Preparation](#dataset-preparation) -- [How to Run Our Codes)](#how-to-run-our-codes) -- [Transfer Testing](#transfer-testing) -- [High-Resolution & 3D](#high-resolution--3d) -- [Baselines](#baselines) -- [Command-Line Arguments](#command-line-arguments) -- [Citation](#citation) -- [License](#license) -- [Contact](#contact) +## Installation ---- +The experiments were developed with Python 3.8 and PyTorch 2.0. A CUDA-capable GPU is recommended for training and attack evaluation. -## Environment Setup ```bash -# Create conda environment conda create -n datacook python=3.8 -y conda activate datacook - -# Install required dependencies pip install -r requirements.txt ``` ---- -## Dataset Preparation -### 1. MedMNIST Datasets -We primarily use datasets from [MedMNIST](https://medmnist.com/ "https://medmnist.com/"). -Please download them via the official Python API or manually place them under: +LPIPS is optional and is only required for 2D perceptual image-quality evaluation: + ```bash -./data/MedMNIST/ +pip install lpips ``` -Example: -```python -from medmnist import BloodMNIST -from torchvision import transforms - -transform = transforms.Compose([transforms.ToTensor()]) -train_dataset = BloodMNIST(split='train', transform=transform, download=True, root='./data/MedMNIST') -test_dataset = BloodMNIST(split='test', transform=transform, download=True, root='./data/MedMNIST') + +## Data preparation + +### MedMNIST + +Install `medmnist` through `requirements.txt`. The pipeline downloads an official split when it is absent from the configured root. Set `dataset.data_path` in each pipeline YAML to the desired MedMNIST directory. The bundled configurations default to: + +```text +../datasets/ +``` + +For example, edit `configs/pipelines/bloodmnist.yaml` when using a different location. Both 28x28/28x28x28 and 224x224 variants are obtained through the official [MedMNIST API](https://medmnist.com/). + +### Structured medical data + +The datasets are not redistributed. After obtaining them from their respective providers, use the following filenames or override `--data-path`: + +```text +TableData/dataset/diabetes.csv +TableData/dataset/HepatitisCdata.csv +TableData/dataset/systemic diseases.csv ``` ---- -### 2. Table Data -We also support **tabular datasets**. Please place them under: +The Systemic Diseases experiment uses restricted-access cohort data; users are responsible for obtaining the required authorization and complying with the provider's terms. + +## Quick start + +Inspect the commands for the primary 2D suite without running training: ```bash -./data/TableData/ +python run_datacook_suite.py \ + --config configs/pipelines/medmnist_three.yaml \ + --dry-run ``` -| Dataset Name | Download Link | -| ------------- | ----------------------------------- | -| Pima Indians Diabetes dataset | [Using the ADAP Learning Algorithm to Forecast the Onset of Diabetes Mellitus](https://pmc.ncbi.nlm.nih.gov/articles/PMC2245318/) | -| Hepatitis C Prediction dataset | [UCI Machine Learning Repository](https://archive.ics.uci.edu/datasets) | -| Systemic Diseases dataset | [UK biobank](https://pubmed.ncbi.nlm.nih.gov/25826379/) | ---- +Run the suite on one visible GPU: -## How to Run Our Codes -**Example: Run DataCook on BloodMNIST (2D)** ```bash -# Step 1: Search perturbation noise -cd scripts/Datacook/2d/BloodMNIST -./search_perturbation_noise.sh +CUDA_VISIBLE_DEVICES=0 python run_datacook_suite.py \ + --config configs/pipelines/medmnist_three.yaml +``` -# Step 2: Train model with generated perturbations -cd scripts/Datacook/2d/BloodMNIST -./train.sh +The pipeline is resumable. Existing complete artifacts are reused. A single dataset, seed, and perturbation budget can be selected for a smoke test: + +```bash +CUDA_VISIBLE_DEVICES=0 python run_datacook_pipeline.py \ + --config configs/pipelines/bloodmnist.yaml \ + --seeds 0 --epsilons 8 ``` -You can follow the same format to add other datasets. ---- -**Example: Run tabular datasets - systemic diseases.csv.** +Additional suites: + ```bash -# Step 1: pretrain model -cd TableData -python MLP.py +# Additional 2D datasets +python run_datacook_suite.py --config configs/pipelines/medmnist_extra_three.yaml -# Step 2: Generated perturbations and get result such as using datcook -python PrivacyAttacker.py --attack_type datacook +# Multi-label ChestMNIST +python run_datacook_pipeline.py --config configs/pipelines/chestmnist.yaml + +# Four volumetric datasets +python run_datacook_suite.py --config configs/pipelines/medmnist3d_four.yaml + +# Three 224x224 datasets +python run_datacook_suite.py --config configs/pipelines/medmnist_224_three.yaml ``` -## Transfer Testing -To test transferability on another model (e.g., from ResNet-18 to ResNet-50), -modify the `model_name` argument in `train.sh`: +## Structured-data experiments + +The final tabular protocol uses Adam, batch size 32, `epsilon=5`, absolute step size `0.05`, and 300 cooking steps at the representative budget. The same scripts also support the configured perturbation-budget grid. + ```bash -model_name=resnet50 +python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_diabetes.yaml \ + --device cuda + +python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_hepatitis_c.yaml \ + --device cuda + +python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_systemic_diseases.yaml \ + --device cuda +``` + +Use `--data-path /path/to/file.csv` to override the dataset location. + +## Evaluation and post-hoc experiments + +Each completed imaging run exports three prediction artifacts: + +```text +clean.npz # clean model evaluated on raw test data +authorized_cooked.npz # protected model evaluated on cooked test data +unauthorized_raw.npz # protected model evaluated on raw test data ``` ---- +Generate post-hoc configurations from completed runs and execute the enabled analyses: -## High-Resolution & 3D -For high-resolution and 3D datasets, see: ```bash -scripts/Datacook/high-resolution/ -scripts/Datacook/3d/ +python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --strict + +python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --device cuda ``` ---- -## Baselines +CP and PP are signed changes relative to the clean reference: + +```text +CP = unauthorized_raw - clean +PP = authorized_cooked - clean +``` + +More negative CP indicates stronger unauthorized degradation. PP closer to zero indicates better preservation of authorized utility. Binary tasks use AUC; multiclass and multi-label tasks use macro-AUC when task-aware AUC results are reported. + +Detailed commands, artifact contracts, and experiment notes are provided in [REPRODUCIBILITY.md](REPRODUCIBILITY.md). -We provide the following baselines in `scripts/`: +## Repository structure -* Error-Minimum - [**EM**](https://arxiv.org/abs/2101.04898 "Unlearnable Examples: Making Personal Data Unexploitable") And modified verison of **EM** in our paper, called **EM-Pseudo** -* Adversarial Poison - [**ADV**](https://arxiv.org/abs/2106.10807 "Adversarial Examples Make Strong Poisons") -* Unlearnable Clusters - [**UC**](https://arxiv.org/abs/2301.01217 "Unlearnable Clusters: Towards Label-agnostic Unlearnable Examples") -* Synthetic Perturbations - [**LSP**](https://arxiv.org/abs/2111.00898 "https://arxiv.org/abs/2111.00898") +```text +configs/pipelines/ End-to-end dataset and budget configurations +configs/revision/ Evaluation, attack, ablation, and analysis configurations +metrics/ Classification, CP/PP, and image-quality metrics +attacks/ Adaptive attack implementations +analysis/ Aggregation and exploratory statistical analysis +revision_experiments/ Shared configuration, I/O, and model utilities +jobs/ Optional multi-stage job wrappers +tests/ Regression and smoke tests +``` + +Generated data, checkpoints, predictions, and results are intentionally excluded from version control. ---- +## Testing -## Command-Line Arguments -All major hyperparameters can be set via command-line arguments in the provided `.sh` scripts. -Please refer to each `train.sh` or `search_perturbation_noise.sh` file for details. +```bash +pip install -r requirements-dev.txt +python -m compileall -q . +pytest -q +``` ---- +GPU training is not performed by the unit tests. ## Citation -If you use this code, please cite the following: +```bibtex @misc{shang2026datacook, - title = {DataCook: Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control}, - author = {Shang, Sihan and Yan, Jun and Shi, Danli and Zheng, Haizhong and Zheng, Shuangjia and Zhao, Wei and Yang, Jiancheng}, - year = {2026}, - eprint = {2403.17755}, + title = {DataCook: Exploring Anti-Adversarial Fingerprints for Healthcare Data Usage Control}, + author = {Shang, Sihan and Yan, Jun and Shi, Danli and Zheng, Haizhong and Zheng, Shuangjia and Zhao, Wei and Yang, Jiancheng}, + year = {2026}, + eprint = {2403.17755}, archivePrefix = {arXiv}, - primaryClass = {cs.AI}, - url = {https://arxiv.org/abs/2403.17755}, - note = {Accepted for publication in Meta-Radiology} + primaryClass = {cs.AI}, + url = {https://arxiv.org/abs/2403.17755}, + note = {Accepted for publication in Meta-Radiology} } +``` + +The citation will be updated when the journal DOI and final bibliographic record become available. -> We adapted some codes from: -> * [Unlearnable Examples](https://github.com/HanxunH/Unlearnable-Examples) -> * [Unlearnable Clusters](https://github.com/jiamingzhang94/Unlearnable-Clusters) +## Acknowledgements + +Parts of the implementation were adapted from: + +- [Unlearnable Examples](https://github.com/HanxunH/Unlearnable-Examples) +- [Unlearnable Clusters](https://github.com/jiamingzhang94/Unlearnable-Clusters) + +Please also cite the original MedMNIST datasets and the baseline methods used in an experiment. ## License -This project is licensed under the MIT License. -## Contact -For any questions or collaborations, please contact: -**Max Shang** – [loveligo97@gmail.com](loveligo97@gmail.com) +This repository is released under the [Apache License 2.0](LICENSE). Dataset access and redistribution remain subject to the licenses and terms of the original data providers. +## Contact +For questions about the code, contact Sihan Shang at [loveligo97@gmail.com](mailto:loveligo97@gmail.com). diff --git a/REPRODUCIBILITY.md b/REPRODUCIBILITY.md new file mode 100644 index 0000000..c8af1fd --- /dev/null +++ b/REPRODUCIBILITY.md @@ -0,0 +1,150 @@ +# Reproducing the DataCook experiments + +This document describes the public experiment interfaces. YAML files are the source of truth for dataset sizes, seeds, perturbation budgets, models, and output locations. + +## 1. Experiment contract + +An imaging run contains four stages: + +1. Train or restore the surrogate and generate cooked train/test data. +2. Export clean reference predictions. +3. Train or restore the protected victim and export authorized and unauthorized predictions. +4. Compute task-aware metrics and CP/PP. + +`run_datacook_pipeline.py` records stage state and reuses complete artifacts. Use `--force` only when an intentional rerun is required. + +```bash +python run_datacook_pipeline.py --config CONFIG.yaml --dry-run +python run_datacook_pipeline.py --config CONFIG.yaml +python run_datacook_pipeline.py --config CONFIG.yaml --only metrics +``` + +Command-line `--seeds` and `--epsilons` values override the YAML grid. + +## 2. Imaging suites + +| Scope | Configuration | +| --- | --- | +| Primary 2D | `configs/pipelines/medmnist_three.yaml` | +| Additional 2D | `configs/pipelines/medmnist_extra_three.yaml` | +| ChestMNIST multi-label | `configs/pipelines/chestmnist.yaml` | +| MedMNIST3D | `configs/pipelines/medmnist3d_four.yaml` | +| MedMNIST+ 224x224 | `configs/pipelines/medmnist_224_three.yaml` | + +Suite files contain relative paths to individual dataset configurations. Run them from the repository root: + +```bash +python run_datacook_suite.py --config configs/pipelines/medmnist3d_four.yaml +``` + +## 3. Prediction artifacts + +Prediction files are compressed NPZ files containing: + +```text +y_true Reference labels +y_prob Softmax or sigmoid probabilities in [0, 1] +``` + +Accepted shapes are: + +- binary: `(N,)`, `(N,1)`, or `(N,2)` probabilities; +- multiclass: `(N,C)` probabilities; +- multi-label: `(N,L)` labels and probabilities. + +The three required settings are `clean`, `authorized_cooked`, and `unauthorized_raw`. Existing checkpoints can be exported with: + +```bash +python export_revision_predictions.py --config path/to/exports.yaml --device cuda +``` + +## 4. Unified metrics + +```bash +python run_unified_evaluation.py --config configs/revision/metrics.yaml +``` + +The evaluator supports task accuracy, AUC/macro-AUC, sensitivity, specificity, precision, F1, and ECE where mathematically defined. CP and PP are computed only from metrics whose direction is explicitly supported. + +Results are written as long-form CSV files. Missing probabilities, classes, checkpoints, or paired observations produce explicit errors or NaN values; the scripts do not create synthetic result rows. + +## 5. Adaptive attacks + +```bash +python run_adaptive_attacks.py --config configs/revision/attacks.yaml --device cuda +``` + +Supported analyses include JPEG compression, Gaussian smoothing, median filtering, strong augmentation, ordinary fine-tuning, and augmentation-based robust fine-tuning. Fine-tuning copies a protected checkpoint and does not overwrite the original model. + +## 6. Image quality, pseudo labels, and runtime + +```bash +python run_image_quality.py --config configs/revision/image_quality.yaml +python run_pseudo_label_analysis.py --config configs/revision/pseudo.yaml +python run_runtime_analysis.py --config configs/revision/runtime.yaml --device cuda +``` + +LPIPS is optional and applies only to compatible 2D images. Volumetric quality reports SSIM, PSNR, L2, and Linf without LPIPS. Runtime configurations record the device, warm-up, repetitions, sample count, batch size, and aggregation rule. + +## 7. Perturbation-budget ablations + +Imaging ablations consume completed prediction artifacts: + +```bash +python run_epsilon_ablation.py --config configs/revision/epsilon.yaml +``` + +Structured-data ablations train and evaluate directly from the configured CSV: + +```bash +python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_diabetes.yaml \ + --device cuda +``` + +At the representative tabular budget, the final configuration is `epsilon=5`, absolute step size `0.05`, and 300 steps. The data are standardized features, and only the explicitly configured feature indices are cookable. + +## 8. Statistical analysis + +```bash +python -m analysis.aggregate_results --config configs/revision/statistics.yaml +python -m analysis.statistical_tests --config configs/revision/statistics.yaml +``` + +Paired tests match observations by experiment identifiers such as dataset, model, seed, and perturbation budget. Holm correction is applied within the configured families. These tests are exploratory; conclusions should be reported together with means, standard deviations, confidence intervals, effect sizes, and the number of paired observations. + +## 9. Generated post-hoc configuration + +After completing a suite, generate configs that point to its actual checkpoints and artifacts: + +```bash +python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --strict +``` + +Then run the enabled tasks: + +```bash +python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --device cuda +``` + +`--strict` is recommended for final analyses because it rejects incomplete experiment sets. + +## 10. Outputs excluded from Git + +The following directories are generated locally and must not be committed: + +```text +data/ +datasets/ +experiments/ +artifacts/ +results/ +outputs/ +revision_logs/ +``` + +Do not commit patient-level data, UK Biobank-derived tables, checkpoints, prediction NPZ files, or server-specific synchronization scripts. diff --git a/analysis/__init__.py b/analysis/__init__.py new file mode 100644 index 0000000..85c1d92 --- /dev/null +++ b/analysis/__init__.py @@ -0,0 +1 @@ +"""Result aggregation and statistical analysis utilities.""" diff --git a/analysis/aggregate_results.py b/analysis/aggregate_results.py new file mode 100644 index 0000000..4479cd0 --- /dev/null +++ b/analysis/aggregate_results.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Task 6: aggregate long-format revision results across random seeds.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.stats import t + +from revision_experiments.common import load_config, resolve_path, setup_logging, write_rows + + +GROUP_COLUMNS = [ + "experiment_type", "dataset", "modality", "task_type", "method", + "surrogate_model", "victim_model", "epsilon", "setting", + "attack_type", "attack_strength", "metric", + "raw_subset_size", +] + + +def load_long_results(config, base_dir: Path) -> pd.DataFrame: + frames = [] + inputs = config.get("inputs", []) + if not inputs: + raise ValueError("Statistics config requires a non-empty `inputs` list.") + for item in inputs: + path = resolve_path(item["path"], base_dir) + if not path.is_file(): + raise FileNotFoundError(f"Statistics input does not exist: {path}") + frame = pd.read_csv(path) + frame["experiment_type"] = item.get("experiment_type", path.stem) + if "metric" not in frame: + raise ValueError(f"Long-format input {path} is missing `metric`.") + value_columns = item.get("value_columns") + if value_columns is None: + value_columns = ["value"] if "value" in frame else [name for name in ("cp", "pp", "residual_utility", "recovery_rate", "mean") if name in frame] + if not value_columns: + raise ValueError(f"Could not infer numeric value columns in {path}.") + for value_column in value_columns: + if value_column not in frame: + raise KeyError(f"Configured value column {value_column!r} is missing from {path}.") + selected = frame.copy() + if value_column != "value": + selected["metric"] = value_column + ":" + selected["metric"].astype(str) + selected["value"] = pd.to_numeric(selected[value_column], errors="coerce") + frames.append(selected) + combined = pd.concat(frames, ignore_index=True) + for column in ("dataset", "modality", "task_type", "method", "seed"): + if column not in combined: + raise ValueError(f"Statistics inputs are missing required column {column!r}.") + for column in ("surrogate_model", "victim_model", "epsilon", "setting", "attack_type", "attack_strength", "raw_subset_size"): + if column not in combined: + combined[column] = "" + else: + combined[column] = combined[column].fillna("") + return combined + + +def aggregate(frame: pd.DataFrame) -> list[dict]: + rows = [] + for keys, group in frame.groupby(GROUP_COLUMNS, dropna=False): + values = group["value"].dropna().to_numpy(dtype=float) + if values.size == 0: + continue + with np.errstate(invalid="ignore"): + mean = float(np.mean(values)) + std = float(np.std(values, ddof=1)) if values.size > 1 else 0.0 + margin = float(t.ppf(0.975, values.size - 1) * std / np.sqrt(values.size)) if values.size > 1 else 0.0 + rows.append({ + **dict(zip(GROUP_COLUMNS, keys)), "seed": "aggregate", + "num_seeds": int(group.loc[group["value"].notna(), "seed"].nunique()), + "num_observations": int(values.size), "mean": mean, "std": std, + "ci95_low": mean - margin, "ci95_high": mean + margin, + }) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("aggregate-results", args.verbose) + config, base_dir = load_config(args.config) + output = config.get("aggregated_output", "results/statistics/aggregated_results.csv") + path = write_rows(aggregate(load_long_results(config, base_dir)), output) + logger.info("Wrote %s", path) + + +if __name__ == "__main__": + main() diff --git a/analysis/export_tabular_auc.py b/analysis/export_tabular_auc.py new file mode 100644 index 0000000..0d0d988 --- /dev/null +++ b/analysis/export_tabular_auc.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Export paper-ready AUC summaries for the tabular DataCook experiments.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import pandas as pd + + +DEFAULT_INPUT = Path( + "downloaded_tabular_legacy_extended/results/" + "revision_pipeline_tabular_legacy_extended/summary/cp_pp_summary.csv" +) +DEFAULT_OUTPUT_DIR = Path( + "downloaded_tabular_legacy_extended/results/" + "revision_pipeline_tabular_legacy_extended/paper_tables" +) + +# AUC must be selected according to task type. Binary tasks expose ``auc``; +# multiclass and multilabel tasks expose macro-averaged one-vs-rest AUC. +AUC_METRIC_BY_TASK = { + "binary": "auc", + "multiclass": "macro_auc", + "multilabel": "macro_auc", +} + +VALUE_COLUMNS = ["clean", "authorized_cooked", "unauthorized_raw", "cp", "pp"] + + +def select_auc_rows(frame: pd.DataFrame) -> pd.DataFrame: + """Return task-appropriate AUC rows and fail on missing task coverage.""" + required = {"dataset", "task_type", "seed", "epsilon", "metric", *VALUE_COLUMNS} + missing = required.difference(frame.columns) + if missing: + raise ValueError(f"AUC input is missing columns: {sorted(missing)}") + + selected_frames = [] + for task_type, metric in AUC_METRIC_BY_TASK.items(): + selected_frames.append( + frame[frame["task_type"].eq(task_type) & frame["metric"].eq(metric)].copy() + ) + selected = pd.concat(selected_frames, ignore_index=True) + if selected.empty: + raise ValueError("No task-appropriate AUC rows were found.") + + expected_datasets = set(frame["dataset"].dropna().astype(str)) + selected_datasets = set(selected["dataset"].dropna().astype(str)) + missing_datasets = expected_datasets.difference(selected_datasets) + if missing_datasets: + raise ValueError( + "No task-appropriate AUC rows for datasets: " + f"{sorted(missing_datasets)}" + ) + if selected[VALUE_COLUMNS].isna().any().any(): + counts = selected[VALUE_COLUMNS].isna().sum() + raise ValueError(f"Selected AUC rows contain missing values:\n{counts}") + return selected + + +def summarize_auc(frame: pd.DataFrame) -> pd.DataFrame: + """Aggregate AUC, CP and PP over seeds for each dataset and budget.""" + grouped = frame.groupby( + ["dataset", "task_type", "metric", "epsilon"], dropna=False + )[VALUE_COLUMNS].agg(["mean", "std", "count"]) + grouped.columns = [f"{column}_{stat}" for column, stat in grouped.columns] + result = grouped.reset_index().rename(columns={"metric": "auc_metric"}) + count_columns = [f"{column}_count" for column in VALUE_COLUMNS] + if not (result[count_columns].nunique(axis=1) == 1).all(): + raise ValueError("AUC value columns have inconsistent seed counts.") + result.insert(4, "num_seeds", result[count_columns[0]].astype(int)) + return result.drop(columns=count_columns) + + +def _latex_number(mean: float, std: float, *, difference: bool = False) -> str: + scale = 100.0 + return f"${mean * scale:.2f}\\pm{std * scale:.2f}$" + + +def render_latex_table(dataset: str, frame: pd.DataFrame) -> str: + """Render one self-contained LaTeX table for a single dataset.""" + selected = frame[frame["dataset"].eq(dataset)].sort_values("epsilon") + if selected.empty: + raise ValueError(f"No AUC summary rows for {dataset}.") + metric = str(selected["auc_metric"].iloc[0]).replace("_", "\\_") + rows = [] + for row in selected.itertuples(index=False): + cells = [f"{row.epsilon:g}"] + for value in VALUE_COLUMNS: + cells.append( + _latex_number( + getattr(row, f"{value}_mean"), + getattr(row, f"{value}_std"), + difference=value in {"cp", "pp"}, + ) + ) + rows.append(" & ".join(cells) + r" \\") + body = "\n".join(rows) + label = dataset.lower().replace("_", "-") + return rf"""\begin{{table*}}[t] +\small +\caption{{\textbf{{Tabular AUC Results on {dataset}.}} Values are mean$\pm$standard deviation over eight seeds. The task-appropriate AUC metric is \texttt{{{metric}}}; CP and PP are reported as percentage-point changes.}} +\label{{tab:tabular-auc-{label}}} +\centering +\begin{{tabular}}{{rrrrrr}} +\toprule +$\epsilon$ & Clean & Authorized & Unauthorized & CP$\downarrow$ & PP$\uparrow$ \\ +\midrule +{body} +\bottomrule +\end{{tabular}} +\end{{table*}} +""" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, default=DEFAULT_INPUT) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + args = parser.parse_args() + + source = args.input.expanduser().resolve() + output_dir = args.output_dir.expanduser().resolve() + frame = pd.read_csv(source) + selected = select_auc_rows(frame) + summary = summarize_auc(selected) + + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = output_dir / "table_tabular_auc_all.csv" + summary.to_csv(csv_path, index=False) + print(f"Wrote {csv_path}") + + for dataset in summary["dataset"].drop_duplicates(): + path = output_dir / f"table_tabular_auc_{dataset.lower()}.tex" + path.write_text(render_latex_table(dataset, summary), encoding="utf-8") + print(f"Wrote {path}") + + +if __name__ == "__main__": + main() diff --git a/analysis/finalize_revision_results.py b/analysis/finalize_revision_results.py new file mode 100644 index 0000000..7f6d217 --- /dev/null +++ b/analysis/finalize_revision_results.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""Build final DataCook revision statistics and paper-ready CSV tables.""" + +from __future__ import annotations + +import argparse +import warnings +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +from scipy.stats import wilcoxon + +from analysis.aggregate_results import GROUP_COLUMNS, aggregate +from revision_experiments.common import load_config, resolve_path, setup_logging + + +IDENTITY = [ + "dataset", "modality", "task_type", "method", "surrogate_model", + "victim_model", "seed", "epsilon", +] + + +def _read_input(config: dict[str, Any], base_dir: Path, name: str, *, optional: bool = False) -> pd.DataFrame: + value = config.get("inputs", {}).get(name) + if not value: + if optional: + return pd.DataFrame() + raise ValueError(f"Final-analysis config is missing inputs.{name}.") + path = resolve_path(value, base_dir) + if not path.is_file(): + if optional: + return pd.DataFrame() + raise FileNotFoundError(f"Required final-analysis input does not exist: {path}") + return pd.read_csv(path) + + +def _write(frame: pd.DataFrame, path: Path) -> Path: + if frame.empty: + raise ValueError(f"Refusing to write an empty final-analysis table: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + frame.to_csv(temporary, index=False) + temporary.replace(path) + return path + + +def build_epsilon_ablation(cp_pp: pd.DataFrame, quality: pd.DataFrame) -> pd.DataFrame: + quality_columns = [ + column for column in quality.columns + if column.endswith("_mean") or column.endswith("_std") or column == "fid" + ] + keys = [column for column in IDENTITY if column in cp_pp and column in quality] + quality_unique = quality[keys + quality_columns].drop_duplicates(keys) + result = cp_pp.merge(quality_unique, on=keys, how="left", validate="many_to_one") + result["epsilon_pixel"] = (pd.to_numeric(result["epsilon"]) * 255).round(6) + return result + + +def build_budget_all( + attacks: pd.DataFrame, + budget: pd.DataFrame, + full_subset_sizes: dict[str, int], +) -> pd.DataFrame: + full = attacks[ + attacks["attack_type"].isin(["finetune", "robust_finetune"]) + & attacks["attack_strength"].astype(str).str.contains("epochs=10", regex=False) + & ~attacks["attack_strength"].astype(str).str.contains(",n=", regex=False) + ].copy() + full["raw_subset_size"] = full["dataset"].map(full_subset_sizes) + full["raw_subset_budget"] = "full" + frames = [full] + if not budget.empty: + selected = budget.copy() + selected["raw_subset_size"] = pd.to_numeric(selected["raw_subset_size"], errors="raise").astype(int) + selected["raw_subset_budget"] = selected["raw_subset_size"].astype(str) + frames.append(selected) + result = pd.concat(frames, ignore_index=True, sort=False).drop_duplicates() + sort_columns = [column for column in ["dataset", "seed", "raw_subset_size", "attack_type", "metric"] if column in result] + return result.sort_values(sort_columns).reset_index(drop=True) + + +def _long_values(frame: pd.DataFrame, experiment_type: str, value_columns: list[str]) -> list[pd.DataFrame]: + rows = [] + for value_column in value_columns: + if value_column not in frame: + continue + selected = frame.copy() + if "metric" in selected: + selected["metric"] = value_column + ":" + selected["metric"].astype(str) + else: + selected["metric"] = value_column + selected["value"] = pd.to_numeric(selected[value_column], errors="coerce") + selected["experiment_type"] = experiment_type + rows.append(selected) + return rows + + +def build_aggregates( + cp_pp: pd.DataFrame, + attacks: pd.DataFrame, + budget_all: pd.DataFrame, + quality: pd.DataFrame, + runtime: pd.DataFrame, + pseudo: pd.DataFrame, +) -> pd.DataFrame: + frames = [] + frames += _long_values(cp_pp, "main", ["cp", "pp", "residual_utility"]) + frames += _long_values(attacks, "adaptive_attack", [ + "attacked_unauthorized_raw", "cp_after_attack", "recovery_rate", + "recovery_rate_unclipped", + ]) + frames += _long_values(budget_all, "finetune_budget", [ + "attacked_unauthorized_raw", "cp_after_attack", "recovery_rate", + "recovery_rate_unclipped", + ]) + frames += _long_values(quality, "image_quality", [ + "ssim_mean", "psnr_mean", "lpips_mean", "l2_norm_mean", "linf_norm_mean", + ]) + frames += _long_values(runtime, "runtime", [ + "cooking_time_per_sample", "raw_inference_time_per_sample", + "cooked_inference_time_per_sample", "overhead_ratio", "gpu_memory_mb", + ]) + frames += _long_values(pseudo, "pseudo_labels", [ + "pseudo_acc", "mean_confidence", "cp_acc", "pp_acc", "cp_auc", "pp_auc", + ]) + combined = pd.concat(frames, ignore_index=True, sort=False) + for column in GROUP_COLUMNS: + if column not in combined: + combined[column] = "" + elif column not in {"epsilon", "raw_subset_size"}: + combined[column] = combined[column].fillna("") + combined["raw_subset_size"] = combined["raw_subset_size"].fillna("") + return pd.DataFrame(aggregate(combined)) + + +def _paired_test(a: pd.Series, b: pd.Series) -> tuple[float, float, float]: + paired = pd.concat([pd.to_numeric(a, errors="coerce"), pd.to_numeric(b, errors="coerce")], axis=1).dropna() + if len(paired) < 2: + return float("nan"), float("nan"), float("nan") + difference = paired.iloc[:, 1].to_numpy() - paired.iloc[:, 0].to_numpy() + if np.allclose(difference, 0): + return 0.0, 1.0, 0.0 + with warnings.catch_warnings(): + # With three seeds, zero differences make SciPy use its documented + # normal approximation. We expose num_pairs and effect size explicitly. + warnings.filterwarnings("ignore", category=UserWarning, module="scipy.stats") + result = wilcoxon(paired.iloc[:, 0], paired.iloc[:, 1], alternative="two-sided") + return float(result.statistic), float(result.pvalue), float(np.mean(difference)) + + +def _holm_adjust(frame: pd.DataFrame) -> pd.DataFrame: + frame = frame.copy() + frame["p_value_holm"] = np.nan + for _, indices in frame.groupby("comparison", dropna=False).groups.items(): + valid = frame.loc[indices, "p_value"].dropna().sort_values() + running = 0.0 + m = len(valid) + for rank, (index, value) in enumerate(valid.items()): + running = max(running, min(1.0, float(value) * (m - rank))) + frame.loc[index, "p_value_holm"] = running + frame["significant"] = frame["p_value"] < 0.05 + frame["significant_holm"] = frame["p_value_holm"] < 0.05 + return frame + + +def _primary_subset(frame: pd.DataFrame, settings: dict[str, Any], *, attacks: bool = False) -> pd.DataFrame: + selected = frame.copy() + metrics = settings.get("metrics") + if metrics: + selected = selected[selected["metric"].isin(metrics)] + epsilon_map = settings.get("epsilons_pixel_by_dataset", {}) + if epsilon_map: + mask = pd.Series(False, index=selected.index) + for dataset, epsilon_pixels in epsilon_map.items(): + epsilon_values = np.asarray(epsilon_pixels, dtype=float) / 255.0 + dataset_values = pd.to_numeric(selected["epsilon"], errors="coerce").to_numpy() + matches = np.any(np.isclose(dataset_values[:, None], epsilon_values[None, :]), axis=1) + mask |= selected["dataset"].eq(dataset) & matches + selected = selected[mask] + if attacks: + attack_types = settings.get("attack_types") + if attack_types: + selected = selected[selected["attack_type"].isin(attack_types)] + epochs = settings.get("epochs") + if epochs: + pattern = "|".join(f"epochs={int(epoch)}(?:,|$)" for epoch in epochs) + selected = selected[selected["attack_strength"].astype(str).str.contains(pattern, regex=True)] + return selected + + +def build_significance( + cp_pp: pd.DataFrame, + attacks: pd.DataFrame, + budget_all: pd.DataFrame, + settings: dict[str, Any] | None = None, +) -> pd.DataFrame: + settings = settings or {} + cp_pp = _primary_subset(cp_pp, settings) + attacks = _primary_subset(attacks, settings, attacks=True) + budget_all = _primary_subset(budget_all, settings, attacks=True) + rows: list[dict[str, Any]] = [] + for keys, group in cp_pp.groupby(["dataset", "epsilon", "metric"], dropna=False): + statistic, p_value, difference = _paired_test(group["clean"], group["unauthorized_raw"]) + rows.append({ + "comparison": "clean_vs_unauthorized", "dataset": keys[0], "epsilon": keys[1], + "attack_type": "", "attack_strength": "", "raw_subset_size": "", + "metric": keys[2], "condition_a": "clean", "condition_b": "unauthorized_raw", + "num_pairs": int(group[["clean", "unauthorized_raw"]].dropna().shape[0]), + "test_type": "wilcoxon", "statistic": statistic, "p_value": p_value, + "mean_difference_b_minus_a": difference, + }) + attack_sources = [("attack_before_vs_after", attacks), ("budget_before_vs_after", budget_all)] + for comparison, source in attack_sources: + group_columns = ["dataset", "epsilon", "attack_type", "attack_strength", "metric"] + if comparison.startswith("budget"): + group_columns.insert(-1, "raw_subset_size") + for keys, group in source.groupby(group_columns, dropna=False): + values = dict(zip(group_columns, keys if isinstance(keys, tuple) else (keys,))) + statistic, p_value, difference = _paired_test( + group["unauthorized_raw"], group["attacked_unauthorized_raw"] + ) + rows.append({ + "comparison": comparison, "dataset": values["dataset"], + "epsilon": values["epsilon"], "attack_type": values["attack_type"], + "attack_strength": values["attack_strength"], + "raw_subset_size": values.get("raw_subset_size", ""), "metric": values["metric"], + "condition_a": "unauthorized_raw", "condition_b": "after_attack", + "num_pairs": int(group[["unauthorized_raw", "attacked_unauthorized_raw"]].dropna().shape[0]), + "test_type": "wilcoxon", "statistic": statistic, "p_value": p_value, + "mean_difference_b_minus_a": difference, + }) + paired_attack = budget_all.pivot_table( + index=["dataset", "epsilon", "seed", "raw_subset_size", "metric"], + columns="attack_type", values="attacked_unauthorized_raw", aggfunc="mean", + ).reset_index() + if {"finetune", "robust_finetune"}.issubset(paired_attack.columns): + for keys, group in paired_attack.groupby(["dataset", "epsilon", "raw_subset_size", "metric"], dropna=False): + statistic, p_value, difference = _paired_test(group["finetune"], group["robust_finetune"]) + rows.append({ + "comparison": "finetune_vs_robust", "dataset": keys[0], "epsilon": keys[1], + "attack_type": "finetune_vs_robust_finetune", "attack_strength": "epochs=10", + "raw_subset_size": keys[2], "metric": keys[3], "condition_a": "finetune", + "condition_b": "robust_finetune", "num_pairs": len(group), "test_type": "wilcoxon", + "statistic": statistic, "p_value": p_value, + "mean_difference_b_minus_a": difference, + }) + return _holm_adjust(pd.DataFrame(rows)) + + +def _summary_table(frame: pd.DataFrame, group_columns: list[str], value_columns: list[str]) -> pd.DataFrame: + def safe_mean(values: pd.Series) -> float: + array = pd.to_numeric(values, errors="coerce").dropna().to_numpy(dtype=float) + if not len(array): + return float("nan") + if np.isposinf(array).any() and not np.isneginf(array).any(): + return float("inf") + if np.isneginf(array).any() and not np.isposinf(array).any(): + return float("-inf") + if np.isinf(array).any(): + return float("nan") + return float(np.mean(array)) + + def safe_std(values: pd.Series) -> float: + array = pd.to_numeric(values, errors="coerce").dropna().to_numpy(dtype=float) + if len(array) <= 1: + return 0.0 + if np.isinf(array).any(): + return 0.0 if np.all(array == array[0]) else float("nan") + return float(np.std(array, ddof=1)) + + safe_mean.__name__ = "mean" + safe_std.__name__ = "std" + selected = [column for column in value_columns if column in frame] + grouped = frame.groupby(group_columns, dropna=False)[selected].agg([safe_mean, safe_std, "count"]) + grouped.columns = [f"{value}_{stat}" for value, stat in grouped.columns] + return grouped.reset_index() + + +def build_tables( + cp_pp: pd.DataFrame, + attacks: pd.DataFrame, + budget_all: pd.DataFrame, + quality: pd.DataFrame, + runtime: pd.DataFrame, + pseudo: pd.DataFrame, +) -> dict[str, pd.DataFrame]: + main = cp_pp[cp_pp["metric"].eq("acc")].copy() + main["epsilon_pixel"] = (main["epsilon"] * 255).round(6) + attack_acc = attacks[attacks["metric"].eq("acc")] + attack_acc = attack_acc.copy() + attack_acc["epsilon_pixel"] = (attack_acc["epsilon"] * 255).round(6) + budget_acc = budget_all[budget_all["metric"].eq("acc")].copy() + budget_acc["epsilon_pixel"] = (budget_acc["epsilon"] * 255).round(6) + quality = quality.copy() + quality["epsilon_pixel"] = (quality["epsilon"] * 255).round(6) + return { + "table_main_acc.csv": _summary_table( + main, ["dataset", "epsilon", "epsilon_pixel"], + ["clean", "authorized_cooked", "unauthorized_raw", "cp", "pp"], + ), + "table_adaptive_attacks_acc.csv": _summary_table( + attack_acc, ["dataset", "epsilon", "epsilon_pixel", "attack_type", "attack_strength"], + ["unauthorized_raw", "attacked_unauthorized_raw", "cp_before_attack", + "cp_after_attack", "recovery_rate", "recovery_rate_unclipped"], + ), + "table_finetune_budget_acc.csv": _summary_table( + budget_acc, ["dataset", "epsilon", "epsilon_pixel", "attack_type", "raw_subset_size", "raw_subset_budget"], + ["unauthorized_raw", "attacked_unauthorized_raw", "cp_after_attack", + "recovery_rate", "recovery_rate_unclipped"], + ), + "table_image_quality.csv": _summary_table( + quality, ["dataset", "epsilon", "epsilon_pixel"], + ["ssim_mean", "psnr_mean", "lpips_mean", "l2_norm_mean", "linf_norm_mean"], + ), + "table_runtime.csv": _summary_table( + runtime, ["dataset", "resolution", "device", "batch_size"], + ["cooking_time_per_sample", "raw_inference_time_per_sample", + "cooked_inference_time_per_sample", "overhead_ratio", "gpu_memory_mb"], + ), + "table_pseudo_labels.csv": _summary_table( + pseudo, ["dataset"], ["pseudo_acc", "mean_confidence", "cp_acc", "pp_acc", "cp_auc", "pp_auc"], + ), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("final-analysis", args.verbose) + config, base_dir = load_config(args.config) + cp_pp = _read_input(config, base_dir, "cp_pp") + quality = _read_input(config, base_dir, "quality") + attacks = _read_input(config, base_dir, "attacks") + budget = _read_input(config, base_dir, "budget_attacks", optional=True) + runtime = _read_input(config, base_dir, "runtime") + pseudo = _read_input(config, base_dir, "pseudo") + budget_all = build_budget_all(attacks, budget, config.get("full_subset_sizes", {})) + output_dir = Path(config.get("output_dir", "results/revision_pipeline/final")).expanduser() + if not output_dir.is_absolute(): + output_dir = (Path.cwd() / output_dir).resolve() + + outputs = { + "epsilon_ablation.csv": build_epsilon_ablation(cp_pp, quality), + "adaptive_finetune_budget_all.csv": budget_all, + "aggregated_results.csv": build_aggregates(cp_pp, attacks, budget_all, quality, runtime, pseudo), + "significance_tests.csv": build_significance( + cp_pp, attacks, budget_all, config.get("significance", {}) + ), + **build_tables(cp_pp, attacks, budget_all, quality, runtime, pseudo), + } + for filename, frame in outputs.items(): + logger.info("Wrote %s", _write(frame, output_dir / filename)) + + +if __name__ == "__main__": + main() diff --git a/analysis/merge_result_shards.py b/analysis/merge_result_shards.py new file mode 100644 index 0000000..a8f6b10 --- /dev/null +++ b/analysis/merge_result_shards.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Safely merge non-overlapping revision-result shards into extended CSVs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any + +import pandas as pd + +from revision_experiments.common import load_config, resolve_path, setup_logging + + +IDENTITY_COLUMNS = [ + "dataset", "modality", "task_type", "method", "surrogate_model", + "victim_model", "seed", "epsilon", "setting", "attack_type", + "attack_strength", "metric", "raw_subset_size", +] + + +def merge_shards(paths: list[Path]) -> pd.DataFrame: + if not paths: + raise ValueError("At least one input shard is required.") + frames = [] + columns: list[str] = [] + for path in paths: + if not path.is_file(): + raise FileNotFoundError(f"Result shard does not exist: {path}") + frame = pd.read_csv(path) + if frame.empty: + raise ValueError(f"Result shard is empty: {path}") + frame["_source_shard"] = str(path) + frames.append(frame) + columns.extend(column for column in frame.columns if column not in columns) + combined = pd.concat( + [frame.reindex(columns=columns) for frame in frames], ignore_index=True + ) + value_columns = [column for column in columns if column != "_source_shard"] + combined = combined.drop_duplicates(subset=value_columns).reset_index(drop=True) + keys = [column for column in IDENTITY_COLUMNS if column in combined] + normalized = combined[keys].copy() + for column in keys: + normalized[column] = normalized[column].fillna("").astype(str) + conflicts = normalized.duplicated(keys, keep=False) + if conflicts.any(): + examples = combined.loc[conflicts, keys + ["_source_shard"]].head(10) + raise ValueError( + "Conflicting result rows share the same experiment identity:\n" + + examples.to_string(index=False) + ) + return combined.drop(columns="_source_shard") + + +def _write(frame: pd.DataFrame, path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + frame.to_csv(temporary, index=False) + temporary.replace(path) + return path + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("merge-result-shards", args.verbose) + config, base_dir = load_config(args.config) + groups: dict[str, Any] = config.get("groups", {}) + if not groups: + raise ValueError("Merge config requires a non-empty `groups` mapping.") + for name, spec in groups.items(): + inputs = [resolve_path(value, base_dir) for value in spec.get("inputs", [])] + output = Path(spec["output"]).expanduser() + if not output.is_absolute(): + output = (Path.cwd() / output).resolve() + merged = merge_shards(inputs) + logger.info("Wrote %s rows for %s to %s", len(merged), name, _write(merged, output)) + + +if __name__ == "__main__": + main() diff --git a/analysis/plot_budget_trends.py b/analysis/plot_budget_trends.py new file mode 100644 index 0000000..79c299d --- /dev/null +++ b/analysis/plot_budget_trends.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Plot task-aware perturbation-budget trends across modalities. + +Binary tasks use AUC and multiclass/multilabel tasks use macro-AUC. CP and PP +are plotted as signed percentage-point changes. Thin lines show the median +within each dataset, while thick lines and bands show the median and IQR across +dataset-level medians. Collapsed and single-budget settings are excluded. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +import numpy as np +import pandas as pd + +# Avoid a non-writable user-level Matplotlib cache on managed machines. +os.environ.setdefault("MPLCONFIGDIR", "/tmp/datacook-matplotlib") +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.lines import Line2D # noqa: E402 + + +DEFAULT_2D = [ + Path("results/revision_pipeline/summary/cp_pp_summary.csv"), + Path("results/revision_pipeline_extra2d/summary/cp_pp_summary.csv"), +] +DEFAULT_3D = Path("results/revision_pipeline_3d/summary/cp_pp_summary.csv") +DEFAULT_TABULAR = Path( + "downloaded_tabular_legacy_extended/results/" + "revision_pipeline_tabular_legacy_extended/summary/cp_pp_summary.csv" +) +DEFAULT_OUTPUT_DIR = Path("results/revision_pipeline/final_figures") + +MODALITY_ORDER = ["2D", "3D", "Tabular"] +METRIC_LABELS = { + "cp": r"CP (unauthorized $-$ clean; $\downarrow$)", + "pp": r"PP (authorized $-$ clean; $\uparrow$)", +} +COLORS = { + "cp": "#0072B2", # Okabe-Ito blue + "pp": "#D55E00", # Okabe-Ito vermillion +} + + +def _read_many(paths: list[Path]) -> pd.DataFrame: + frames = [] + for path in paths: + resolved = path.expanduser().resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"Missing budget-trend input: {resolved}") + frames.append(pd.read_csv(resolved)) + return pd.concat(frames, ignore_index=True) + + +def _primary_rows(frame: pd.DataFrame, modality: str) -> pd.DataFrame: + task_type = ( + frame["task_type"].astype(str).str.lower() + .str.replace("-", "", regex=False) + .str.replace("_", "", regex=False) + ) + primary_metric = task_type.map({ + "binary": "auc", + "multiclass": "macro_auc", + "multilabel": "macro_auc", + }) + selected = frame[frame["metric"].eq(primary_metric)].copy() + + if modality in {"2D", "3D"}: + selected["budget"] = (pd.to_numeric(selected["epsilon"]) * 255.0).round(6) + selected["budget_unit"] = "pixel" + else: + selected["budget"] = pd.to_numeric(selected["epsilon"]) + selected["budget_unit"] = "standardized_feature_linf" + + if modality == "3D": + selected = selected[~selected["dataset"].eq("SynapseMNIST3D")].copy() + + budget_counts = selected.groupby("dataset")["budget"].transform("nunique") + selected = selected[budget_counts.ge(2)].copy() + + if selected.empty: + raise ValueError(f"No primary metric rows found for {modality}.") + for column in ["clean", "authorized_cooked", "unauthorized_raw"]: + selected[column] = pd.to_numeric(selected[column], errors="raise") + selected["modality_group"] = modality + selected["cp"] = ( + selected["unauthorized_raw"] - selected["clean"] + ) * 100.0 + selected["pp"] = ( + selected["authorized_cooked"] - selected["clean"] + ) * 100.0 + return selected + + +def load_budget_data( + paths_2d: list[Path], path_3d: Path, path_tabular: Path +) -> pd.DataFrame: + frames = [ + _primary_rows(_read_many(paths_2d), "2D"), + _primary_rows(_read_many([path_tabular]), "Tabular"), + _primary_rows(_read_many([path_3d]), "3D"), + ] + return pd.concat(frames, ignore_index=True) + + +def summarize_budget_data(frame: pd.DataFrame) -> pd.DataFrame: + rows: list[dict[str, object]] = [] + for modality in MODALITY_ORDER: + selected = frame[frame["modality_group"].eq(modality)] + for metric in METRIC_LABELS: + dataset_summaries: list[dict[str, object]] = [] + for (dataset, budget), group in selected.groupby(["dataset", "budget"]): + values = group[metric].dropna() + record = { + "modality": modality, + "dataset": dataset, + "budget": budget, + "budget_unit": group["budget_unit"].iloc[0], + "metric": metric, + "median": values.median(), + "q1": values.quantile(0.25), + "q3": values.quantile(0.75), + "num_observations": len(values), + } + rows.append(record) + dataset_summaries.append(record) + + dataset_frame = pd.DataFrame(dataset_summaries) + for budget, group in dataset_frame.groupby("budget"): + values = group["median"].dropna() + rows.append({ + "modality": modality, + "dataset": "Across-dataset summary", + "budget": budget, + "budget_unit": group["budget_unit"].iloc[0], + "metric": metric, + "median": values.median(), + "q1": values.quantile(0.25), + "q3": values.quantile(0.75), + "num_observations": len(values), + }) + return pd.DataFrame(rows).sort_values( + ["metric", "modality", "dataset", "budget"] + ).reset_index(drop=True) + + +def _plot_panel(ax: plt.Axes, summary: pd.DataFrame, modality: str, metric: str) -> None: + selected = summary[ + summary["modality"].eq(modality) & summary["metric"].eq(metric) + ] + pooled = selected[selected["dataset"].eq("Across-dataset summary")].sort_values("budget") + individual = selected[~selected["dataset"].eq("Across-dataset summary")] + color = COLORS[metric] + budgets = pooled["budget"].to_numpy(dtype=float) + positions = np.arange(len(budgets), dtype=float) + position_by_budget = dict(zip(budgets, positions)) + + for dataset, dataset_rows in individual.groupby("dataset"): + dataset_rows = dataset_rows.sort_values("budget") + dataset_positions = [position_by_budget[float(value)] for value in dataset_rows["budget"]] + ax.plot( + dataset_positions, dataset_rows["median"], + color="#A6A6A6", linewidth=0.75, alpha=0.75, + marker="o", markersize=1.8, markeredgewidth=0, zorder=1, + ) + ax.plot( + positions, pooled["median"], + color=color, linewidth=1.8, + linestyle="-" if metric == "cp" else "--", + marker="o" if metric == "cp" else "s", + markersize=4.0, markeredgecolor="white", markeredgewidth=0.4, + zorder=3, + ) + ax.axhline( + 0.0, color="#777777", linewidth=0.7, linestyle=(0, (3, 2)), + alpha=0.8, zorder=0, + ) + labels = [f"{value:g}" for value in budgets] + ax.set_xticks(positions, labels) + if modality == "Tabular": + ax.set_xlabel(r"$\epsilon$ (standardized-feature $L_\infty$)") + else: + ax.set_xlabel(r"$\epsilon$ (pixel /255)") + ax.grid(axis="y", color="#D9D9D9", linewidth=0.55, alpha=0.85) + ax.spines[["top", "right"]].set_visible(False) + + +def draw_figure(summary: pd.DataFrame, output_dir: Path) -> list[Path]: + plt.rcParams.update({ + "font.family": "serif", + "font.size": 8, + "axes.titlesize": 9, + "axes.labelsize": 8, + "xtick.labelsize": 7, + "ytick.labelsize": 7, + "legend.fontsize": 7.5, + "pdf.fonttype": 42, + "ps.fonttype": 42, + }) + figure, axes = plt.subplots(3, 2, figsize=(7.15, 6.05), sharey=True) + metrics = ["cp", "pp"] + panel_labels = ["(a)", "(b)", "(c)", "(d)", "(e)", "(f)"] + row_labels = { + "2D": r"2D $28\!\times\!28$" + "\n" + r"$\Delta$ primary metric (pp)", + "3D": r"3D $28^3$" + "\n" + r"$\Delta$ primary metric (pp)", + "Tabular": "Tabular\n" + r"$\Delta$ primary metric (pp)", + } + panel_index = 0 + for row, modality in enumerate(MODALITY_ORDER): + for column, metric in enumerate(metrics): + ax = axes[row, column] + _plot_panel(ax, summary, modality, metric) + if row == 0: + ax.set_title(METRIC_LABELS[metric], pad=8) + if column == 0: + ax.set_ylabel(row_labels[modality]) + ax.set_ylim(-18.0, 3.0) + ax.text( + -0.12, 1.04, panel_labels[panel_index], transform=ax.transAxes, + fontsize=8, fontweight="bold", va="bottom", ha="left", + ) + panel_index += 1 + + legend_handles = [ + Line2D( + [0], [0], color="#A6A6A6", linewidth=0.75, marker="o", + markersize=2.5, label="Individual dataset", + ), + Line2D( + [0], [0], color=COLORS["cp"], linewidth=2, + marker="o", label="Across-dataset median CP", + ), + Line2D( + [0], [0], color=COLORS["pp"], linewidth=2, + linestyle="--", marker="s", label="Across-dataset median PP", + ), + ] + figure.legend( + handles=legend_handles, loc="lower center", ncol=3, + frameon=False, bbox_to_anchor=(0.5, 0.012), + ) + figure.subplots_adjust( + left=0.13, right=0.98, top=0.94, bottom=0.13, + wspace=0.18, hspace=0.46, + ) + + output_dir.mkdir(parents=True, exist_ok=True) + outputs = [] + for suffix in ("pdf", "svg", "png"): + path = output_dir / f"perturbation_budget_overview.{suffix}" + figure.savefig(path, dpi=300 if suffix == "png" else None, bbox_inches="tight") + outputs.append(path) + plt.close(figure) + return outputs + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-2d", nargs="+", type=Path, default=DEFAULT_2D) + parser.add_argument("--input-3d", type=Path, default=DEFAULT_3D) + parser.add_argument("--input-tabular", type=Path, default=DEFAULT_TABULAR) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + args = parser.parse_args() + + data = load_budget_data(args.input_2d, args.input_3d, args.input_tabular) + summary = summarize_budget_data(data) + output_dir = args.output_dir.expanduser().resolve() + output_dir.mkdir(parents=True, exist_ok=True) + summary_path = output_dir / "perturbation_budget_overview_data.csv" + summary.to_csv(summary_path, index=False) + print(f"Wrote {summary_path}") + for path in draw_figure(summary, output_dir): + print(f"Wrote {path}") + + +if __name__ == "__main__": + main() diff --git a/analysis/statistical_tests.py b/analysis/statistical_tests.py new file mode 100644 index 0000000..2cf810d --- /dev/null +++ b/analysis/statistical_tests.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Task 6: paired t-tests and Wilcoxon signed-rank comparisons.""" + +from __future__ import annotations + +import argparse + +import numpy as np +from scipy.stats import ttest_rel, wilcoxon + +from analysis.aggregate_results import load_long_results +from revision_experiments.common import load_config, setup_logging, write_rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("statistical-tests", args.verbose) + config, base_dir = load_config(args.config) + frame = load_long_results(config, base_dir) + comparisons = config.get("comparisons", []) + if not comparisons: + raise ValueError("Statistics config requires at least one method comparison.") + rows = [] + default_pair = [ + "experiment_type", "dataset", "surrogate_model", "victim_model", + "seed", "epsilon", "setting", "attack_type", "attack_strength", "metric", + ] + default_group = [ + "experiment_type", "dataset", "modality", "task_type", + "surrogate_model", "victim_model", "epsilon", "setting", + "attack_type", "attack_strength", "metric", + ] + for comparison in comparisons: + method_a, method_b = comparison["method_a"], comparison["method_b"] + subset = frame[frame["method"].isin([method_a, method_b])].copy() + if "experiment_type" in comparison: + subset = subset[subset["experiment_type"] == comparison["experiment_type"]] + if "metrics" in comparison: + subset = subset[subset["metric"].isin(comparison["metrics"])] + pair_by = [column for column in comparison.get("pair_by", default_pair) if column in subset] + group_by = [column for column in comparison.get("group_by", default_group) if column in subset] + for keys, group in subset.groupby(group_by, dropna=False): + pivot = group.pivot_table(index=pair_by, columns="method", values="value", aggfunc="mean").dropna() + if method_a not in pivot or method_b not in pivot or len(pivot) < 2: + logger.warning("Skipping %s vs %s for %s: fewer than two pairs.", method_a, method_b, keys) + continue + a, b = pivot[method_a].to_numpy(), pivot[method_b].to_numpy() + requested = comparison.get("test", "auto") + test_type = "wilcoxon" if requested == "auto" and len(a) < 30 else ("paired_t" if requested == "auto" else requested) + if test_type == "wilcoxon": + if np.allclose(a, b): + statistic, p_value = 0.0, 1.0 + else: + result = wilcoxon(a, b, alternative=comparison.get("alternative", "two-sided")) + statistic, p_value = float(result.statistic), float(result.pvalue) + elif test_type == "paired_t": + result = ttest_rel(a, b, alternative=comparison.get("alternative", "two-sided")) + statistic, p_value = float(result.statistic), float(result.pvalue) + else: + raise ValueError(f"Unsupported test {test_type!r}.") + key_values = keys if isinstance(keys, tuple) else (keys,) + group_map = dict(zip(group_by, key_values)) + alpha = float(comparison.get("alpha", 0.05)) + rows.append({ + "experiment_type": group_map.get("experiment_type", comparison.get("experiment_type", "all")), + "dataset": group_map.get("dataset", "all"), "dataset_group": comparison.get("dataset_group", group_map.get("dataset", "all")), + "modality": group_map.get("modality", "all"), "task_type": group_map.get("task_type", "all"), + "method": f"{method_a}_vs_{method_b}", "surrogate_model": group_map.get("surrogate_model", "all"), + "victim_model": group_map.get("victim_model", "all"), "seed": "paired", + "epsilon": group_map.get("epsilon", "all"), "setting": group_map.get("setting", "all"), + "attack_type": group_map.get("attack_type", ""), "attack_strength": group_map.get("attack_strength", ""), + "method_a": method_a, "method_b": method_b, + "metric": group_map.get("metric", "all"), "test_type": test_type, "num_pairs": len(a), + "statistic": statistic, "p_value": p_value, "alpha": alpha, "significant": bool(p_value < alpha), + }) + if not rows: + raise ValueError("No valid paired comparisons were available; no result file was written.") + output = config.get("significance_output", "results/statistics/significance_tests.csv") + logger.info("Wrote %s", write_rows(rows, output)) + + +if __name__ == "__main__": + main() diff --git a/analysis/tabular_legacy_statistics.py b/analysis/tabular_legacy_statistics.py new file mode 100644 index 0000000..bdb1872 --- /dev/null +++ b/analysis/tabular_legacy_statistics.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Paired significance tests for the extended historical tabular protocol.""" + +from __future__ import annotations + +import argparse +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.stats import wilcoxon + +from revision_experiments.common import load_config, resolve_path, setup_logging + + +DEFAULT_PRIMARY_METRICS = { + "binary": "acc", + "multiclass": "acc", + "multilabel": "subset_acc", +} + + +def _paired_wilcoxon(a: pd.Series, b: pd.Series) -> dict[str, float | int]: + paired = pd.concat( + [pd.to_numeric(a, errors="coerce"), pd.to_numeric(b, errors="coerce")], + axis=1, + ).dropna() + difference = paired.iloc[:, 1].to_numpy() - paired.iloc[:, 0].to_numpy() + nonzero = difference[~np.isclose(difference, 0.0)] + if len(paired) < 2: + return { + "num_pairs": len(paired), "num_nonzero": len(nonzero), + "statistic": float("nan"), "p_value": float("nan"), + "mean_difference_b_minus_a": float("nan"), + "cohens_dz": float("nan"), "rank_biserial": float("nan"), + } + if len(nonzero) == 0: + statistic, p_value, rank_biserial = 0.0, 1.0, 0.0 + else: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, module="scipy.stats") + result = wilcoxon(difference, alternative="two-sided") + statistic, p_value = float(result.statistic), float(result.pvalue) + ranks = pd.Series(np.abs(nonzero)).rank(method="average").to_numpy() + positive = float(ranks[nonzero > 0].sum()) + negative = float(ranks[nonzero < 0].sum()) + rank_biserial = (positive - negative) / (positive + negative) + std = float(np.std(difference, ddof=1)) + return { + "num_pairs": len(paired), "num_nonzero": len(nonzero), + "statistic": statistic, "p_value": p_value, + "mean_difference_b_minus_a": float(np.mean(difference)), + "cohens_dz": float(np.mean(difference) / std) if std > 0 else float("nan"), + "rank_biserial": rank_biserial, + } + + +def _holm_by_family(frame: pd.DataFrame, family_columns: list[str]) -> pd.DataFrame: + output = frame.copy() + output["p_value_holm"] = np.nan + for _, indices in output.groupby(family_columns, dropna=False).groups.items(): + valid = output.loc[indices, "p_value"].dropna().sort_values() + running = 0.0 + total = len(valid) + for rank, (index, value) in enumerate(valid.items()): + running = max(running, min(1.0, float(value) * (total - rank))) + output.loc[index, "p_value_holm"] = running + output["significant_raw"] = output["p_value"] < 0.05 + output["significant_holm"] = output["p_value_holm"] < 0.05 + return output + + +def build_tests(frame: pd.DataFrame, config: dict) -> pd.DataFrame: + primary = { + **DEFAULT_PRIMARY_METRICS, + **config.get("primary_metrics_by_task", {}), + } + selected = frame[ + frame.apply( + lambda row: row["metric"] == primary.get(str(row["task_type"])), + axis=1, + ) + ].copy() + rows = [] + comparisons = ( + ("clean_vs_unauthorized", "clean", "unauthorized_raw"), + ("clean_vs_authorized", "clean", "authorized_cooked"), + ) + for keys, group in selected.groupby( + ["dataset", "task_type", "epsilon", "metric"], dropna=False + ): + for comparison, condition_a, condition_b in comparisons: + rows.append({ + "dataset": keys[0], "task_type": keys[1], + "epsilon": keys[2], "metric": keys[3], + "comparison": comparison, "condition_a": condition_a, + "condition_b": condition_b, "test_type": "wilcoxon_two_sided", + **_paired_wilcoxon(group[condition_a], group[condition_b]), + }) + if not rows: + raise ValueError("No primary-metric rows were available for significance tests.") + family_columns = list(config.get("holm_family", ["dataset", "comparison"])) + return _holm_by_family(pd.DataFrame(rows), family_columns) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("tabular-legacy-statistics", args.verbose) + config, base_dir = load_config(args.config) + input_path = resolve_path(config["significance_input"], base_dir) + if not input_path.is_file(): + raise FileNotFoundError(f"Significance input does not exist: {input_path}") + output_path = Path(config["significance_output"]).expanduser() + if not output_path.is_absolute(): + output_path = (Path.cwd() / output_path).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary = output_path.with_suffix(output_path.suffix + ".tmp") + build_tests(pd.read_csv(input_path), config).to_csv(temporary, index=False) + temporary.replace(output_path) + logger.info("Wrote %s", output_path) + + +if __name__ == "__main__": + main() diff --git a/attacks/__init__.py b/attacks/__init__.py new file mode 100644 index 0000000..6bc6223 --- /dev/null +++ b/attacks/__init__.py @@ -0,0 +1,15 @@ +"""Adaptive attacks/purification operators used in revision experiments.""" + +from .augmentation import strong_augment +from .denoise import median_filter +from .finetune import fine_tune_model +from .jpeg import jpeg_compress +from .smoothing import gaussian_smooth + +__all__ = [ + "jpeg_compress", + "gaussian_smooth", + "strong_augment", + "median_filter", + "fine_tune_model", +] diff --git a/attacks/_utils.py b/attacks/_utils.py new file mode 100644 index 0000000..3bbf7c3 --- /dev/null +++ b/attacks/_utils.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch + + +def as_nchw_tensor(images: Any) -> tuple[torch.Tensor, tuple[str, Any]]: + """Convert supported image batches to NCHW while retaining restoration info.""" + is_tensor = torch.is_tensor(images) + original_device = images.device if is_tensor else None + original_dtype = images.dtype if is_tensor else np.asarray(images).dtype + tensor = images.detach().clone() if is_tensor else torch.as_tensor(np.asarray(images)) + layout = "nchw" + if tensor.ndim == 3: + tensor = tensor.unsqueeze(1) + layout = "nhw" + elif tensor.ndim == 4 and tensor.shape[1] not in (1, 3) and tensor.shape[-1] in (1, 3): + tensor = tensor.permute(0, 3, 1, 2) + layout = "nhwc" + elif tensor.ndim != 4 or tensor.shape[1] not in (1, 3): + raise ValueError("Expected image batch with grayscale or RGB channels.") + scale = 255.0 if not torch.is_floating_point(tensor) or float(tensor.max()) > 1.5 else 1.0 + tensor = tensor.float() / scale + return tensor, ("tensor" if is_tensor else "numpy", original_device, original_dtype, layout, scale) + + +def restore_images(tensor: torch.Tensor, info: tuple[str, Any]) -> Any: + source, device, dtype, layout, scale = info + tensor = tensor.clamp(0, 1) * scale + if layout == "nhw": + tensor = tensor[:, 0] + elif layout == "nhwc": + tensor = tensor.permute(0, 2, 3, 1) + if source == "tensor": + if not torch.empty((), dtype=dtype).is_floating_point(): + tensor = tensor.round() + return tensor.to(device=device, dtype=dtype) + array = tensor.cpu().numpy() + if np.issubdtype(dtype, np.integer): + array = np.rint(array) + return array.astype(dtype) diff --git a/attacks/augmentation.py b/attacks/augmentation.py new file mode 100644 index 0000000..da2923d --- /dev/null +++ b/attacks/augmentation.py @@ -0,0 +1,55 @@ +"""Strong stochastic augmentation used during unauthorized training.""" + +from __future__ import annotations + +from typing import Any + +import torch +from torchvision import transforms +from torchvision.transforms import InterpolationMode + +from ._utils import as_nchw_tensor, restore_images + + +def strong_augment(images: Any, strength: int = 2, seed: int | None = None) -> Any: + """Apply crop/flip/rotation/jitter/blur/RandAugment/erasing to a batch.""" + if strength < 1: + raise ValueError("strength must be at least 1.") + tensor, info = as_nchw_tensor(images) + height, width = tensor.shape[-2:] + channels = tensor.shape[1] + operators = [ + transforms.RandomResizedCrop((height, width), scale=(max(0.5, 1.0 - 0.15 * strength), 1.0), interpolation=InterpolationMode.BILINEAR), + transforms.RandomHorizontalFlip(), + transforms.RandomRotation(degrees=10 * strength, interpolation=InterpolationMode.BILINEAR), + ] + if channels == 3: + operators.append(transforms.ColorJitter(0.1 * strength, 0.1 * strength, 0.1 * strength, 0.03 * strength)) + operators.append(transforms.GaussianBlur(kernel_size=3 if min(height, width) < 64 else 5, sigma=(0.1, 0.5 * strength))) + pipeline = transforms.Compose(operators) + rand_augment = transforms.RandAugment( + num_ops=min(4, strength + 1), magnitude=min(30, 5 * strength) + ) if hasattr(transforms, "RandAugment") else None + erasing = transforms.RandomErasing( + p=min(0.8, 0.25 * strength), scale=(0.01, min(0.2, 0.05 * strength)) + ) + def apply_augmentation() -> torch.Tensor: + augmented_images = [] + for image in tensor: + image = pipeline(image) + # Older torchvision RandAugment expects uint8 for solarize/posterize. + if rand_augment is not None: + image = rand_augment((image.clamp(0, 1) * 255).round().to(torch.uint8)).float() / 255.0 + augmented_images.append(erasing(image)) + return torch.stack(augmented_images) + + if seed is None: + # During robust fine-tuning, consume the caller's seeded RNG normally so + # successive mini-batches receive different (but reproducible) views. + augmented = apply_augmentation() + else: + devices = [tensor.device] if tensor.is_cuda else [] + with torch.random.fork_rng(devices=devices): + torch.manual_seed(seed) + augmented = apply_augmentation() + return restore_images(augmented, info) diff --git a/attacks/denoise.py b/attacks/denoise.py new file mode 100644 index 0000000..0fa0e66 --- /dev/null +++ b/attacks/denoise.py @@ -0,0 +1,20 @@ +"""Simple, dependency-free median-filter purification baseline.""" + +from __future__ import annotations + +from typing import Any + +import torch.nn.functional as functional + +from ._utils import as_nchw_tensor, restore_images + + +def median_filter(images: Any, kernel_size: int = 3) -> Any: + if kernel_size <= 0 or kernel_size % 2 == 0: + raise ValueError("kernel_size must be a positive odd integer.") + tensor, info = as_nchw_tensor(images) + padding = kernel_size // 2 + padded = functional.pad(tensor, [padding] * 4, mode="reflect") + patches = padded.unfold(2, kernel_size, 1).unfold(3, kernel_size, 1) + filtered = patches.contiguous().view(*patches.shape[:4], -1).median(dim=-1).values + return restore_images(filtered, info) diff --git a/attacks/finetune.py b/attacks/finetune.py new file mode 100644 index 0000000..f3f7991 --- /dev/null +++ b/attacks/finetune.py @@ -0,0 +1,59 @@ +"""Small raw-subset fine-tuning attack for PyTorch classifiers.""" + +from __future__ import annotations + +import copy +from collections.abc import Callable +from typing import Any + +import torch +from torch.utils.data import DataLoader + + +def fine_tune_model( + model: torch.nn.Module, + data_loader: DataLoader, + *, + task_type: str, + epochs: int = 5, + learning_rate: float = 1e-4, + device: str | torch.device | None = None, + input_transform: Callable[[torch.Tensor], torch.Tensor] | None = None, + seed: int = 0, +) -> torch.nn.Module: + """Return a fine-tuned copy; the input model and checkpoint stay untouched.""" + if epochs <= 0 or learning_rate <= 0: + raise ValueError("epochs and learning_rate must be positive.") + selected_device = torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu")) + attacked_model = copy.deepcopy(model).to(selected_device) + optimizer = torch.optim.Adam(attacked_model.parameters(), lr=learning_rate) + normalized_task = task_type.replace("-", "").replace("_", "").lower() + if normalized_task not in {"binary", "multiclass", "multilabel"}: + raise ValueError(f"Unsupported task_type {task_type!r}.") + if normalized_task in {"multilabel", "binary"}: + binary_criterion: Any = torch.nn.BCEWithLogitsLoss() + if normalized_task in {"binary", "multiclass"}: + class_criterion: Any = torch.nn.CrossEntropyLoss() + + devices = [selected_device] if selected_device.type == "cuda" else [] + with torch.random.fork_rng(devices=devices): + torch.manual_seed(seed) + attacked_model.train() + for _ in range(epochs): + for batch in data_loader: + if len(batch) < 2: + raise ValueError("Fine-tuning loader batches must contain (inputs, labels).") + inputs, labels = batch[0].to(selected_device), batch[1].to(selected_device) + if input_transform is not None: + inputs = input_transform(inputs) + optimizer.zero_grad(set_to_none=True) + logits = attacked_model(inputs) + if normalized_task == "multilabel": + loss = binary_criterion(logits, labels.float()) + elif normalized_task == "binary" and (logits.ndim == 1 or logits.shape[-1] == 1): + loss = binary_criterion(logits.reshape(-1), labels.reshape(-1).float()) + else: + loss = class_criterion(logits, labels.reshape(-1).long()) + loss.backward() + optimizer.step() + return attacked_model.eval() diff --git a/attacks/jpeg.py b/attacks/jpeg.py new file mode 100644 index 0000000..2889e0c --- /dev/null +++ b/attacks/jpeg.py @@ -0,0 +1,35 @@ +"""JPEG compression purification attack.""" + +from __future__ import annotations + +from io import BytesIO +from typing import Any + +import numpy as np +import torch +from PIL import Image + +from ._utils import as_nchw_tensor, restore_images + + +def jpeg_compress(images: Any, quality: int = 75) -> Any: + if not 1 <= quality <= 100: + raise ValueError("JPEG quality must be in [1, 100].") + tensor, info = as_nchw_tensor(images) + output = [] + for image in tensor.cpu(): + uint8 = (image.clamp(0, 1) * 255).round().byte() + if uint8.shape[0] == 1: + pil = Image.fromarray(uint8[0].numpy(), mode="L") + else: + pil = Image.fromarray(uint8.permute(1, 2, 0).numpy(), mode="RGB") + buffer = BytesIO() + pil.save(buffer, format="JPEG", quality=int(quality)) + buffer.seek(0) + decoded = np.asarray(Image.open(buffer)).copy() + if decoded.ndim == 2: + decoded = decoded[None] + else: + decoded = np.transpose(decoded, (2, 0, 1)) + output.append(torch.from_numpy(decoded).float() / 255.0) + return restore_images(torch.stack(output).to(tensor.device), info) diff --git a/attacks/smoothing.py b/attacks/smoothing.py new file mode 100644 index 0000000..3c2574a --- /dev/null +++ b/attacks/smoothing.py @@ -0,0 +1,26 @@ +"""Gaussian smoothing attack implemented without optional SciPy dependencies.""" + +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn.functional as functional + +from ._utils import as_nchw_tensor, restore_images + + +def gaussian_smooth(images: Any, sigma: float = 1.0, kernel_size: int = 5) -> Any: + if sigma <= 0: + raise ValueError("sigma must be positive.") + if kernel_size <= 0 or kernel_size % 2 == 0: + raise ValueError("kernel_size must be a positive odd integer.") + tensor, info = as_nchw_tensor(images) + coordinates = torch.arange(kernel_size, device=tensor.device, dtype=tensor.dtype) - kernel_size // 2 + kernel_1d = torch.exp(-(coordinates**2) / (2 * sigma**2)) + kernel_1d /= kernel_1d.sum() + kernel_2d = torch.outer(kernel_1d, kernel_1d) + kernel = kernel_2d.expand(tensor.shape[1], 1, kernel_size, kernel_size) + padded = functional.pad(tensor, [kernel_size // 2] * 4, mode="reflect") + smoothed = functional.conv2d(padded, kernel, groups=tensor.shape[1]) + return restore_images(smoothed, info) diff --git a/configs/.DS_Store b/configs/.DS_Store deleted file mode 100644 index 725e4590d9d2e8174ddfc1015e6077a7db96ebfe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJ5Izf5S?Kc5s79?N;kAfdxNkF1qC$W04N&-MOKM~=zTm&j=?c_^8szNE)WGm zXh!lp+w*b!q}U!J;>}CHCR!0ug(k?-DTquLO-Js03}nr5OV`xWUH`lt3W@&Wlze+d zH}pU~^xpS(P=}00wf7oT<_5N&4`s W9kZcik$DXV#zQ~}2^9?d0s|j*%rF%I diff --git a/configs/MedMNIST/.DS_Store b/configs/MedMNIST/.DS_Store deleted file mode 100644 index 30a4f25d03026f37d7d7047bfba3e57ec1a7488c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10244 zcmeHMO>fgc5S@ie2~8>d1Yas~LR4Hiq^UvTP+F1x0NSQ%k&-xS;{!J%Kpc8P{3(7` z-+qv;?U{ANA>uF_t;e)TYWJcv=pC}`uKB~3PcW_{ey>Dx)JPKFdz#8}y zP3VX|(0dxwX$!3vQ~^~$6;K6K0af6CPyls-;Mj~+~LH6g?>-W~Td3x^#BTY2l<$#{1%p=T2! z6yth_U*_OshTiH?6;K7*3fS0vn?^LH0>3DJpI+^cFN)d1yY#+ExLJ|s=hNaG_pt=4 z-pjo|2QqfDHg<6x_kUt(48Ze-F6oSJY59<8XfXXMBL%KPyR(92E1(&^+mwUt4b8xK zl^1ES^|~@Rfh*glz?8)@JfG1!usov+s9HLOS8VAu<#oo!>f7NVaPRbD_3rShkH@|J zpCSik=W>ZzefglgFqQ;b7)R2B9Yex{;oqlYIz$HT=x;8q{sekI@dR2Jfh&V!8MbhB zUSa(mV|H9&rp)O;WcS;xKNcx)9ALamyTYs%&>egaMZUipd?Vxn$GPSk$>WFCwJCS!j2Sc@ec;=j-P26FR{hUedM5 z^#>GYPy$aD$*RgA8GgIqH^-Z+#LiLR>~muO_ztE?yui7`hh?AiVa73_j~K-g{O5@g zSgi%5z+FVMtOYc~cNgC+zBsN6{D;xKb0qNs_YNO+1WBLzIKp~7h32>TmGLMVyU3l> zoka@V2iTLCcE$C`qU(I!dIS+Is(>n>3aA1bRN%4iE3^6kxAOP@4KmWMQ~_0B(+Zfa ze4LNaYsjs)`nS3E4D}PLEf!aK>s)Yf>3CR|j>jK19sdH=VqaNn3(kkZR^H+Y%Emti k#F protected training -> NPZ -> metrics. +dataset: + name: BloodMNIST + poison_name: PoisonBloodMNIST + modality: 2d_image + task_type: multiclass + num_classes: 8 + config_path: configs/MedMNIST/BloodMNIST + data_path: ../datasets + noise_shape: [11959, 3, 28, 28] + test_noise_shape: [3421, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [2, 4, 8, 12, 16] + +runtime: + # Change to python/path/in/your/environment when needed. + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline +result_root: results/revision_pipeline diff --git a/configs/pipelines/breastmnist.yaml b/configs/pipelines/breastmnist.yaml new file mode 100644 index 0000000..6e15bb2 --- /dev/null +++ b/configs/pipelines/breastmnist.yaml @@ -0,0 +1,34 @@ +# One-click DataCook pipeline for the binary BreastMNIST dataset. +dataset: + name: BreastMNIST + poison_name: PoisonBreastMNIST + modality: 2d_image + task_type: binary + num_classes: 2 + config_path: configs/MedMNIST/BreastMNIST + data_path: ../datasets + noise_shape: [546, 3, 28, 28] + test_noise_shape: [156, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [2, 4, 8, 12, 16] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline +result_root: results/revision_pipeline diff --git a/configs/pipelines/breastmnist224.yaml b/configs/pipelines/breastmnist224.yaml new file mode 100644 index 0000000..0e28f8a --- /dev/null +++ b/configs/pipelines/breastmnist224.yaml @@ -0,0 +1,35 @@ +# DataCook 224x224 pipeline for the binary BreastMNIST+ dataset. +dataset: + name: BreastMNIST + poison_name: PoisonBreastMNIST + modality: high_resolution + task_type: binary + num_classes: 2 + high_resolution: true + config_path: configs/MedMNIST/BreastMNIST + data_path: ../datasets + noise_shape: [546, 3, 224, 224] + test_noise_shape: [156, 3, 224, 224] + +models: + surrogate: resnet18_224 + victim: resnet18_224 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 32 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_224 +result_root: results/revision_pipeline_224 diff --git a/configs/pipelines/chestmnist.yaml b/configs/pipelines/chestmnist.yaml new file mode 100644 index 0000000..223d0fa --- /dev/null +++ b/configs/pipelines/chestmnist.yaml @@ -0,0 +1,50 @@ +# ChestMNIST multi-label DataCook pipeline. The default is one validation run; +# use --seeds/--epsilons overrides only after it completes successfully. +dataset: + name: ChestMNIST + poison_name: PoisonChestMNIST + modality: 2d_image + task_type: multilabel + num_classes: 14 + config_path: configs/MedMNIST/ChestMNIST + data_path: ../datasets + noise_shape: [78468, 3, 28, 28] + test_noise_shape: [22433, 3, 28, 28] + # CP/PP uses utility metrics whose larger values are better. Hamming loss and + # ECE are still exported in main_metrics.csv, but excluded from CP/PP signs. + metric_names: + - subset_acc + - micro_auc + - macro_auc + - micro_f1 + - macro_f1 + - micro_precision + - macro_precision + - micro_sensitivity + - macro_sensitivity + - micro_specificity + - macro_specificity + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + threshold: 0.5 + +seeds: [0] +epsilons: [8] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline_multilabel +result_root: results/revision_pipeline_multilabel diff --git a/configs/pipelines/dermamnist.yaml b/configs/pipelines/dermamnist.yaml new file mode 100644 index 0000000..d11c05b --- /dev/null +++ b/configs/pipelines/dermamnist.yaml @@ -0,0 +1,34 @@ +# Extra-server 2D pipeline: RGB seven-class DermaMNIST. +dataset: + name: DermaMNIST + poison_name: PoisonDermaMNIST + modality: 2d_image + task_type: multiclass + num_classes: 7 + config_path: configs/MedMNIST/DermaMNIST + data_path: ../datasets + noise_shape: [7007, 3, 28, 28] + test_noise_shape: [2005, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2, 3, 4, 5, 6, 7] +epsilons: [2, 4, 8, 12, 16] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline_extra2d +result_root: results/revision_pipeline_extra2d diff --git a/configs/pipelines/fracturemnist3d.yaml b/configs/pipelines/fracturemnist3d.yaml new file mode 100644 index 0000000..9ce7a86 --- /dev/null +++ b/configs/pipelines/fracturemnist3d.yaml @@ -0,0 +1,34 @@ +# Formal 3D DataCook experiment for FractureMNIST3D. +dataset: + name: FractureMNIST3D + poison_name: PoisonFractureMNIST3D + modality: 3d_volume + task_type: multiclass + num_classes: 3 + config_path: configs/MedMNIST/FractureMNIST3D + data_path: ../datasets + noise_shape: [1027, 1, 28, 28, 28] + test_noise_shape: [240, 1, 28, 28, 28] + +models: + surrogate: resnet18_3d + victim: resnet18_3d + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8, 16, 32] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_3d +result_root: results/revision_pipeline_3d diff --git a/configs/pipelines/medmnist3d_four.yaml b/configs/pipelines/medmnist3d_four.yaml new file mode 100644 index 0000000..4961922 --- /dev/null +++ b/configs/pipelines/medmnist3d_four.yaml @@ -0,0 +1,6 @@ +# Four configured MedMNIST3D datasets with identical seeds and budgets. +pipelines: + - organmnist3d.yaml + - nodulemnist3d.yaml + - fracturemnist3d.yaml + - synapsemnist3d.yaml diff --git a/configs/pipelines/medmnist3d_smoke.yaml b/configs/pipelines/medmnist3d_smoke.yaml new file mode 100644 index 0000000..a840a8b --- /dev/null +++ b/configs/pipelines/medmnist3d_smoke.yaml @@ -0,0 +1,3 @@ +# Initial 3D suite. Keep this at one run until the complete artifact contract passes. +pipelines: + - organmnist3d_smoke.yaml diff --git a/configs/pipelines/medmnist_224_three.yaml b/configs/pipelines/medmnist_224_three.yaml new file mode 100644 index 0000000..c3f0c4c --- /dev/null +++ b/configs/pipelines/medmnist_224_three.yaml @@ -0,0 +1,5 @@ +# Representative MedMNIST+ 224x224 suite. +pipelines: + - breastmnist224.yaml + - pneumoniamnist224.yaml + - retinamnist224.yaml diff --git a/configs/pipelines/medmnist_additional_two.yaml b/configs/pipelines/medmnist_additional_two.yaml new file mode 100644 index 0000000..a531e2c --- /dev/null +++ b/configs/pipelines/medmnist_additional_two.yaml @@ -0,0 +1,5 @@ +# Run only the two datasets added after the BloodMNIST experiment. +# Their metrics are still merged with existing per-run results in the shared summary. +pipelines: + - breastmnist.yaml + - retinamnist.yaml diff --git a/configs/pipelines/medmnist_extra_three.yaml b/configs/pipelines/medmnist_extra_three.yaml new file mode 100644 index 0000000..d0708df --- /dev/null +++ b/configs/pipelines/medmnist_extra_three.yaml @@ -0,0 +1,5 @@ +# Three representative 2D datasets assigned to the second server. +pipelines: + - pneumoniamnist.yaml + - dermamnist.yaml + - organcmnist.yaml diff --git a/configs/pipelines/medmnist_multilabel.yaml b/configs/pipelines/medmnist_multilabel.yaml new file mode 100644 index 0000000..9c790c2 --- /dev/null +++ b/configs/pipelines/medmnist_multilabel.yaml @@ -0,0 +1,3 @@ +# Multi-label suite (currently ChestMNIST is MedMNIST's multi-label dataset). +pipelines: + - chestmnist.yaml diff --git a/configs/pipelines/medmnist_three.yaml b/configs/pipelines/medmnist_three.yaml new file mode 100644 index 0000000..7669c1a --- /dev/null +++ b/configs/pipelines/medmnist_three.yaml @@ -0,0 +1,5 @@ +# Paths are relative to this suite file. +pipelines: + - bloodmnist.yaml + - breastmnist.yaml + - retinamnist.yaml diff --git a/configs/pipelines/nodulemnist3d.yaml b/configs/pipelines/nodulemnist3d.yaml new file mode 100644 index 0000000..ca0e310 --- /dev/null +++ b/configs/pipelines/nodulemnist3d.yaml @@ -0,0 +1,34 @@ +# Formal 3D DataCook experiment for NoduleMNIST3D. +dataset: + name: NoduleMNIST3D + poison_name: PoisonNoduleMNIST3D + modality: 3d_volume + task_type: binary + num_classes: 2 + config_path: configs/MedMNIST/NoduleMNIST3D + data_path: ../datasets + noise_shape: [1158, 1, 28, 28, 28] + test_noise_shape: [310, 1, 28, 28, 28] + +models: + surrogate: resnet18_3d + victim: resnet18_3d + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8, 16, 32] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_3d +result_root: results/revision_pipeline_3d diff --git a/configs/pipelines/organcmnist.yaml b/configs/pipelines/organcmnist.yaml new file mode 100644 index 0000000..0fbcf3e --- /dev/null +++ b/configs/pipelines/organcmnist.yaml @@ -0,0 +1,34 @@ +# Extra-server 2D pipeline: grayscale eleven-class OrganCMNIST (converted to RGB). +dataset: + name: OrganCMNIST + poison_name: PoisonOrganCMNIST + modality: 2d_image + task_type: multiclass + num_classes: 11 + config_path: configs/MedMNIST/OrganCMNIST + data_path: ../datasets + noise_shape: [12975, 3, 28, 28] + test_noise_shape: [8216, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2, 3, 4, 5, 6, 7] +epsilons: [2, 4, 8, 12, 16] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline_extra2d +result_root: results/revision_pipeline_extra2d diff --git a/configs/pipelines/organmnist3d.yaml b/configs/pipelines/organmnist3d.yaml new file mode 100644 index 0000000..07f7dcd --- /dev/null +++ b/configs/pipelines/organmnist3d.yaml @@ -0,0 +1,34 @@ +# Formal 3D DataCook experiment for OrganMNIST3D. +dataset: + name: OrganMNIST3D + poison_name: PoisonOrganMNIST3D + modality: 3d_volume + task_type: multiclass + num_classes: 11 + config_path: configs/MedMNIST/OrganMNIST3D + data_path: ../datasets + noise_shape: [971, 1, 28, 28, 28] + test_noise_shape: [610, 1, 28, 28, 28] + +models: + surrogate: resnet18_3d + victim: resnet18_3d + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8, 16, 32] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_3d +result_root: results/revision_pipeline_3d diff --git a/configs/pipelines/organmnist3d_smoke.yaml b/configs/pipelines/organmnist3d_smoke.yaml new file mode 100644 index 0000000..f1635c8 --- /dev/null +++ b/configs/pipelines/organmnist3d_smoke.yaml @@ -0,0 +1,34 @@ +# Single-GPU end-to-end smoke test for the 3D DataCook pipeline. +dataset: + name: OrganMNIST3D + poison_name: PoisonOrganMNIST3D + modality: 3d_volume + task_type: multiclass + num_classes: 11 + config_path: configs/MedMNIST/OrganMNIST3D + data_path: ../datasets + noise_shape: [971, 1, 28, 28, 28] + test_noise_shape: [610, 1, 28, 28, 28] + +models: + surrogate: resnet18_3d + victim: resnet18_3d + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0] +epsilons: [8] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_3d +result_root: results/revision_pipeline_3d diff --git a/configs/pipelines/pneumoniamnist.yaml b/configs/pipelines/pneumoniamnist.yaml new file mode 100644 index 0000000..c50765c --- /dev/null +++ b/configs/pipelines/pneumoniamnist.yaml @@ -0,0 +1,34 @@ +# Extra-server 2D pipeline: grayscale binary PneumoniaMNIST (converted to RGB). +dataset: + name: PneumoniaMNIST + poison_name: PoisonPneumoniaMNIST + modality: 2d_image + task_type: binary + num_classes: 2 + config_path: configs/MedMNIST/PneumoniaMNIST + data_path: ../datasets + noise_shape: [4708, 3, 28, 28] + test_noise_shape: [624, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2, 3, 4, 5, 6, 7] +epsilons: [2, 4, 8, 12, 16] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline_extra2d +result_root: results/revision_pipeline_extra2d diff --git a/configs/pipelines/pneumoniamnist224.yaml b/configs/pipelines/pneumoniamnist224.yaml new file mode 100644 index 0000000..380ba04 --- /dev/null +++ b/configs/pipelines/pneumoniamnist224.yaml @@ -0,0 +1,35 @@ +# DataCook 224x224 pipeline for the binary PneumoniaMNIST+ dataset. +dataset: + name: PneumoniaMNIST + poison_name: PoisonPneumoniaMNIST + modality: high_resolution + task_type: binary + num_classes: 2 + high_resolution: true + config_path: configs/MedMNIST/PneumoniaMNIST + data_path: ../datasets + noise_shape: [4708, 3, 224, 224] + test_noise_shape: [624, 3, 224, 224] + +models: + surrogate: resnet18_224 + victim: resnet18_224 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8] + +runtime: + python: python + train_batch_size: 8 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_224 +result_root: results/revision_pipeline_224 diff --git a/configs/pipelines/retinamnist.yaml b/configs/pipelines/retinamnist.yaml new file mode 100644 index 0000000..4309948 --- /dev/null +++ b/configs/pipelines/retinamnist.yaml @@ -0,0 +1,34 @@ +# One-click DataCook pipeline for the five-class RetinaMNIST dataset. +dataset: + name: RetinaMNIST + poison_name: PoisonRetinaMNIST + modality: 2d_image + task_type: multiclass + num_classes: 5 + config_path: configs/MedMNIST/RetinaMNIST + data_path: ../datasets + noise_shape: [1080, 3, 28, 28] + test_noise_shape: [400, 3, 28, 28] + +models: + surrogate: resnet18 + victim: resnet18 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [2, 4, 8, 12, 16] + +runtime: + python: python + train_batch_size: 128 + eval_batch_size: 128 + num_workers: 8 + +experiment_root: experiments/revision_pipeline +result_root: results/revision_pipeline diff --git a/configs/pipelines/retinamnist224.yaml b/configs/pipelines/retinamnist224.yaml new file mode 100644 index 0000000..f91c471 --- /dev/null +++ b/configs/pipelines/retinamnist224.yaml @@ -0,0 +1,35 @@ +# DataCook 224x224 pipeline for the five-class RetinaMNIST+ dataset. +dataset: + name: RetinaMNIST + poison_name: PoisonRetinaMNIST + modality: high_resolution + task_type: multiclass + num_classes: 5 + high_resolution: true + config_path: configs/MedMNIST/RetinaMNIST + data_path: ../datasets + noise_shape: [1080, 3, 224, 224] + test_noise_shape: [400, 3, 224, 224] + +models: + surrogate: resnet18_224 + victim: resnet18_224 + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 32 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_224 +result_root: results/revision_pipeline_224 diff --git a/configs/pipelines/synapsemnist3d.yaml b/configs/pipelines/synapsemnist3d.yaml new file mode 100644 index 0000000..f8e838c --- /dev/null +++ b/configs/pipelines/synapsemnist3d.yaml @@ -0,0 +1,34 @@ +# Formal 3D DataCook experiment for SynapseMNIST3D. +dataset: + name: SynapseMNIST3D + poison_name: PoisonSynapseMNIST3D + modality: 3d_volume + task_type: binary + num_classes: 2 + config_path: configs/MedMNIST/SynapseMNIST3D + data_path: ../datasets + noise_shape: [1230, 1, 28, 28, 28] + test_noise_shape: [352, 1, 28, 28, 28] + +models: + surrogate: resnet18_3d + victim: resnet18_3d + +experiment: + attack_type: datacook + epsilon_scale: pixel + step_size_ratio: 0.2 + num_steps: 20 + universal_stop_error: 0.01 + +seeds: [0, 1, 2] +epsilons: [8, 16, 32] + +runtime: + python: python + train_batch_size: 16 + eval_batch_size: 16 + num_workers: 4 + +experiment_root: experiments/revision_pipeline_3d +result_root: results/revision_pipeline_3d diff --git a/configs/revision.yaml b/configs/revision.yaml new file mode 100644 index 0000000..e5a746b --- /dev/null +++ b/configs/revision.yaml @@ -0,0 +1,9 @@ +# Master switchboard. Enable a task only after filling its task-specific config. +tasks: + metrics: {enabled: false, config: revision/metrics.yaml} + attacks: {enabled: false, config: revision/attacks.yaml} + epsilon: {enabled: false, config: revision/epsilon.yaml} + pseudo: {enabled: false, config: revision/pseudo.yaml} + statistics: {enabled: false, config: revision/statistics.yaml} + runtime: {enabled: false, config: revision/runtime.yaml} + image_quality: {enabled: false, config: revision/image_quality.yaml} diff --git a/configs/revision/224/generated.yaml b/configs/revision/224/generated.yaml new file mode 100644 index 0000000..eb0f4c3 --- /dev/null +++ b/configs/revision/224/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: true + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/224/generated/attacks.yaml b/configs/revision/224/generated/attacks.yaml new file mode 100644 index 0000000..3724d28 --- /dev/null +++ b/configs/revision/224/generated/attacks.yaml @@ -0,0 +1,443 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_224.csv diff --git a/configs/revision/224/generated/attacks_finetune.yaml b/configs/revision/224/generated/attacks_finetune.yaml new file mode 100644 index 0000000..8c62bdc --- /dev/null +++ b/configs/revision/224/generated/attacks_finetune.yaml @@ -0,0 +1,445 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id001 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id002 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id001 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id003 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id004 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id003 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id005 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id006 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id005 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id007 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id008 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id009 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id010 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id011 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id012 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id013 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id014 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id015 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id016 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id017 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: &id018 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_224.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_224.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_224.csv diff --git a/configs/revision/224/generated/attacks_finetune_budget.yaml b/configs/revision/224/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..2e8d28e --- /dev/null +++ b/configs/revision/224/generated/attacks_finetune_budget.yaml @@ -0,0 +1,479 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id001 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id003 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id005 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id007 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id009 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id011 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id013 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id015 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + eval_data: &id017 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: &id018 + inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 25 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 16 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_224.csv diff --git a/configs/revision/224/generated/final_analysis.yaml b/configs/revision/224/generated/final_analysis.yaml new file mode 100644 index 0000000..474f5ab --- /dev/null +++ b/configs/revision/224/generated/final_analysis.yaml @@ -0,0 +1,12 @@ +inputs: + cp_pp: ../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../results/revision_pipeline/posthoc/quality_metrics_224.csv + attacks: ../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_224.csv + budget_attacks: ../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_224.csv + runtime: ../../../../results/revision_pipeline/posthoc/runtime_summary_224.csv + pseudo: ../../../../results/revision_pipeline/posthoc/pseudo_label_quality_224.csv +full_subset_sizes: + BreastMNIST: 78 + PneumoniaMNIST: 524 + RetinaMNIST: 120 +output_dir: results/revision_pipeline/final_224 diff --git a/configs/revision/224/generated/image_quality.yaml b/configs/revision/224/generated/image_quality.yaml new file mode 100644 index 0000000..85f343d --- /dev/null +++ b/configs/revision/224/generated/image_quality.yaml @@ -0,0 +1,119 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_224.csv diff --git a/configs/revision/224/generated/pseudo.yaml b/configs/revision/224/generated/pseudo.yaml new file mode 100644 index 0000000..330fa03 --- /dev/null +++ b/configs/revision/224/generated/pseudo.yaml @@ -0,0 +1,139 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=1-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=2-eps=8/resnet18_224/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_224.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_224.csv diff --git a/configs/revision/224/generated/runtime.yaml b/configs/revision/224/generated/runtime.yaml new file mode 100644 index 0000000..80cf9c1 --- /dev/null +++ b/configs/revision/224/generated/runtime.yaml @@ -0,0 +1,71 @@ +experiments: +- metadata: + dataset: BreastMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../MedMNIST/BreastMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + data: + raw_inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST/raw_test.npz::images + cooked_inputs: ../../../../artifacts/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../experiments/revision_pipeline_224/BreastMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +- metadata: + dataset: PneumoniaMNIST + modality: high_resolution + task_type: binary + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../MedMNIST/PneumoniaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + data: + raw_inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST/raw_test.npz::images + cooked_inputs: ../../../../artifacts/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../experiments/revision_pipeline_224/PneumoniaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +- metadata: + dataset: RetinaMNIST + modality: high_resolution + task_type: multiclass + method: datacook + surrogate_model: resnet18_224 + victim_model: resnet18_224 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../MedMNIST/RetinaMNIST/resnet18_224.yaml + checkpoint: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/protected/resnet18_224/checkpoints/resnet18_224.pth + data: + raw_inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST/raw_test.npz::images + cooked_inputs: ../../../../artifacts/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../experiments/revision_pipeline_224/RetinaMNIST-datacook-sur=resnet18_224-vic=resnet18_224-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +output: results/revision_pipeline/posthoc/runtime_summary_224.csv diff --git a/configs/revision/3d/eps32/generated.yaml b/configs/revision/3d/eps32/generated.yaml new file mode 100644 index 0000000..f5aa56d --- /dev/null +++ b/configs/revision/3d/eps32/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: false + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: false + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: true + config: generated/runtime.yaml + image_quality: + enabled: false + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/3d/eps32/generated/runtime.yaml b/configs/revision/3d/eps32/generated/runtime.yaml new file mode 100644 index 0000000..835f052 --- /dev/null +++ b/configs/revision/3d/eps32/generated/runtime.yaml @@ -0,0 +1,95 @@ +experiments: +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + model: + config_path: ../../../../MedMNIST/OrganMNIST3D/resnet18_3d.yaml + checkpoint: ../../../../../experiments/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/protected/resnet18_3d/checkpoints/resnet18_3d.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + model: + config_path: ../../../../MedMNIST/NoduleMNIST3D/resnet18_3d.yaml + checkpoint: ../../../../../experiments/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/protected/resnet18_3d/checkpoints/resnet18_3d.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + model: + config_path: ../../../../MedMNIST/FractureMNIST3D/resnet18_3d.yaml + checkpoint: ../../../../../experiments/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/protected/resnet18_3d/checkpoints/resnet18_3d.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + model: + config_path: ../../../../MedMNIST/SynapseMNIST3D/resnet18_3d.yaml + checkpoint: ../../../../../experiments/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/protected/resnet18_3d/checkpoints/resnet18_3d.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 16 + repeats: 5 + warmup: 1 +include_average: true +output: results/revision_pipeline_3d/posthoc/runtime_summary_eps32.csv diff --git a/configs/revision/3d/generated.yaml b/configs/revision/3d/generated.yaml new file mode 100644 index 0000000..08316c1 --- /dev/null +++ b/configs/revision/3d/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: false + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: false + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: false + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/3d/generated/attacks.yaml b/configs/revision/3d/generated/attacks.yaml new file mode 100644 index 0000000..6cce768 --- /dev/null +++ b/configs/revision/3d/generated/attacks.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline_3d/posthoc/adaptive_attack_results.csv diff --git a/configs/revision/3d/generated/attacks_finetune.yaml b/configs/revision/3d/generated/attacks_finetune.yaml new file mode 100644 index 0000000..3b2b89c --- /dev/null +++ b/configs/revision/3d/generated/attacks_finetune.yaml @@ -0,0 +1,4 @@ +experiments: [] +output: results/revision_pipeline_3d/posthoc/adaptive_finetune_results.csv +merge_with: results/revision_pipeline_3d/posthoc/adaptive_attack_results.csv +combined_output: results/revision_pipeline_3d/posthoc/adaptive_attack_results_all.csv diff --git a/configs/revision/3d/generated/attacks_finetune_budget.yaml b/configs/revision/3d/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..02cc709 --- /dev/null +++ b/configs/revision/3d/generated/attacks_finetune_budget.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline_3d/posthoc/adaptive_finetune_budget_results.csv diff --git a/configs/revision/3d/generated/final_analysis.yaml b/configs/revision/3d/generated/final_analysis.yaml new file mode 100644 index 0000000..1ba99a7 --- /dev/null +++ b/configs/revision/3d/generated/final_analysis.yaml @@ -0,0 +1,9 @@ +inputs: + cp_pp: ../../../../results/revision_pipeline_3d/summary/cp_pp_summary.csv + quality: ../../../../results/revision_pipeline_3d/posthoc/quality_metrics.csv + attacks: ../../../../results/revision_pipeline_3d/posthoc/adaptive_attack_results_all.csv + budget_attacks: ../../../../results/revision_pipeline_3d/posthoc/adaptive_finetune_budget_results.csv + runtime: ../../../../results/revision_pipeline_3d/posthoc/runtime_summary.csv + pseudo: ../../../../results/revision_pipeline_3d/posthoc/pseudo_label_quality.csv +full_subset_sizes: {} +output_dir: results/revision_pipeline_3d/final diff --git a/configs/revision/3d/generated/image_quality.yaml b/configs/revision/3d/generated/image_quality.yaml new file mode 100644 index 0000000..2857c7d --- /dev/null +++ b/configs/revision/3d/generated/image_quality.yaml @@ -0,0 +1,470 @@ +experiments: +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: OrganMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/OrganMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: NoduleMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/NoduleMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: FractureMNIST3D + modality: 3d_volume + task_type: multiclass + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/FractureMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 0 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=0-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 1 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=1-eps=32/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=8/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=16/cooked_test.npz::images + compute_lpips: false +- metadata: + dataset: SynapseMNIST3D + modality: 3d_volume + task_type: binary + method: datacook + surrogate_model: resnet18_3d + victim_model: resnet18_3d + seed: 2 + epsilon: 0.12549019607843137 + images: + raw: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_3d/SynapseMNIST3D-datacook-sur=resnet18_3d-vic=resnet18_3d-seed=2-eps=32/cooked_test.npz::images + compute_lpips: false +output: results/revision_pipeline_3d/posthoc/quality_metrics.csv diff --git a/configs/revision/3d/generated/pseudo.yaml b/configs/revision/3d/generated/pseudo.yaml new file mode 100644 index 0000000..c47757e --- /dev/null +++ b/configs/revision/3d/generated/pseudo.yaml @@ -0,0 +1,4 @@ +experiments: [] +outputs: + samples: results/revision_pipeline_3d/posthoc/pseudo_label_samples.csv + quality: results/revision_pipeline_3d/posthoc/pseudo_label_quality.csv diff --git a/configs/revision/3d/generated/runtime.yaml b/configs/revision/3d/generated/runtime.yaml new file mode 100644 index 0000000..acc41f7 --- /dev/null +++ b/configs/revision/3d/generated/runtime.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline_3d/posthoc/runtime_summary.csv diff --git a/configs/revision/attacks.yaml b/configs/revision/attacks.yaml new file mode 100644 index 0000000..3f25842 --- /dev/null +++ b/configs/revision/attacks.yaml @@ -0,0 +1,12 @@ +experiments: [] +# Each record uses the metadata/predictions schema from metrics.yaml and: +# attacks: +# - {type: jpeg, strength: 95, predictions: ../../artifacts/jpeg95.npz} +# - {type: gaussian_smoothing, strength: {sigma: 1.0, kernel_size: 5}, predictions: ...} +# - {type: strong_augmentation, strength: 2, predictions: ...} +# - type: finetune +# epochs: 5 +# model: {config_path: ../MedMNIST/BloodMNIST/resnet18.yaml, checkpoint: ...} +# raw_subset: {inputs: raw_subset.npz::images, labels: raw_subset.npz::labels} +# eval_data: {inputs: raw_test.npz::images, labels: raw_test.npz::labels} +output: results/attacks/adaptive_attack_results.csv diff --git a/configs/revision/chestmnist/generated.yaml b/configs/revision/chestmnist/generated.yaml new file mode 100644 index 0000000..eb0f4c3 --- /dev/null +++ b/configs/revision/chestmnist/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: true + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/chestmnist/generated/attacks.yaml b/configs/revision/chestmnist/generated/attacks.yaml new file mode 100644 index 0000000..acd6e79 --- /dev/null +++ b/configs/revision/chestmnist/generated/attacks.yaml @@ -0,0 +1,149 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_chestmnist.csv diff --git a/configs/revision/chestmnist/generated/attacks_finetune.yaml b/configs/revision/chestmnist/generated/attacks_finetune.yaml new file mode 100644 index 0000000..9fb5fba --- /dev/null +++ b/configs/revision/chestmnist/generated/attacks_finetune.yaml @@ -0,0 +1,151 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id001 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id003 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id005 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_chestmnist.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_chestmnist.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_chestmnist.csv diff --git a/configs/revision/chestmnist/generated/attacks_finetune_budget.yaml b/configs/revision/chestmnist/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..c0bb742 --- /dev/null +++ b/configs/revision/chestmnist/generated/attacks_finetune_budget.yaml @@ -0,0 +1,161 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::images + labels: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_chestmnist.csv diff --git a/configs/revision/chestmnist/generated/final_analysis.yaml b/configs/revision/chestmnist/generated/final_analysis.yaml new file mode 100644 index 0000000..da8f765 --- /dev/null +++ b/configs/revision/chestmnist/generated/final_analysis.yaml @@ -0,0 +1,10 @@ +inputs: + cp_pp: ../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../results/revision_pipeline/posthoc/quality_metrics_chestmnist.csv + attacks: ../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_chestmnist.csv + budget_attacks: ../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_chestmnist.csv + runtime: ../../../../results/revision_pipeline/posthoc/runtime_summary_chestmnist.csv + pseudo: ../../../../results/revision_pipeline/posthoc/pseudo_label_quality_chestmnist.csv +full_subset_sizes: + ChestMNIST: 11219 +output_dir: results/revision_pipeline/final_chestmnist diff --git a/configs/revision/chestmnist/generated/image_quality.yaml b/configs/revision/chestmnist/generated/image_quality.yaml new file mode 100644 index 0000000..6659488 --- /dev/null +++ b/configs/revision/chestmnist/generated/image_quality.yaml @@ -0,0 +1,41 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + cooked: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_chestmnist.csv diff --git a/configs/revision/chestmnist/generated/pseudo.yaml b/configs/revision/chestmnist/generated/pseudo.yaml new file mode 100644 index 0000000..a5cf89a --- /dev/null +++ b/configs/revision/chestmnist/generated/pseudo.yaml @@ -0,0 +1,49 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 14 + predictions: + clean: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_chestmnist.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_chestmnist.csv diff --git a/configs/revision/chestmnist/generated/runtime.yaml b/configs/revision/chestmnist/generated/runtime.yaml new file mode 100644 index 0000000..b5eba21 --- /dev/null +++ b/configs/revision/chestmnist/generated/runtime.yaml @@ -0,0 +1,25 @@ +experiments: +- metadata: + dataset: ChestMNIST + modality: 2d_image + task_type: multilabel + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../MedMNIST/ChestMNIST/resnet18.yaml + checkpoint: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST/raw_test.npz::images + cooked_inputs: ../../../../artifacts/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../experiments/revision_pipeline_multilabel/ChestMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +output: results/revision_pipeline/posthoc/runtime_summary_chestmnist.csv diff --git a/configs/revision/epsilon.yaml b/configs/revision/epsilon.yaml new file mode 100644 index 0000000..b7261e7 --- /dev/null +++ b/configs/revision/epsilon.yaml @@ -0,0 +1,9 @@ +metadata_defaults: + method: datacook +seeds: [0, 1, 2] +experiments: [] +# A record may set epsilons: [0.0078431373, 0.0156862745, 0.0313725490, +# 0.0470588235, 0.0627450980] and use {epsilon} +# in artifact paths. For legacy perturbation.py commands, pass the corresponding +# 2/4/8/12/16 with --epsilon_scale pixel; or pass these values with unit. +output: results/ablations/epsilon_ablation.csv diff --git a/configs/revision/exports.yaml b/configs/revision/exports.yaml new file mode 100644 index 0000000..65a3d0f --- /dev/null +++ b/configs/revision/exports.yaml @@ -0,0 +1,12 @@ +exports: [] +# - metadata: {dataset: BloodMNIST, modality: 2d_image, task_type: multiclass, +# method: datacook, surrogate_model: resnet18, victim_model: resnet18, +# seed: 0, epsilon: 0.031372549} +# model: +# config_path: ../MedMNIST/BloodMNIST/resnet18.yaml +# checkpoint: ../../checkpoints/resnet18_100.pth +# data: +# inputs: ../../artifacts/test.npz::images +# labels: ../../artifacts/test.npz::labels +# output: ../../artifacts/predictions/clean.npz +# batch_size: 128 diff --git a/configs/revision/extensions/extra2d/generated.yaml b/configs/revision/extensions/extra2d/generated.yaml new file mode 100644 index 0000000..eb0f4c3 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: true + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/extensions/extra2d/generated/attacks.yaml b/configs/revision/extensions/extra2d/generated/attacks.yaml new file mode 100644 index 0000000..ed620b1 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/attacks.yaml @@ -0,0 +1,1178 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_extra2d.csv diff --git a/configs/revision/extensions/extra2d/generated/attacks_finetune.yaml b/configs/revision/extensions/extra2d/generated/attacks_finetune.yaml new file mode 100644 index 0000000..5a2b1b2 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/attacks_finetune.yaml @@ -0,0 +1,868 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id020 + eval_data: *id019 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id022 + eval_data: *id021 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id024 + eval_data: *id023 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id026 + eval_data: *id025 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id028 + eval_data: *id027 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id030 + eval_data: *id029 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id031 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id032 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id031 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id032 + eval_data: *id031 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id033 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id034 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id033 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id034 + eval_data: *id033 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id035 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id036 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id035 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id036 + eval_data: *id035 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id037 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id038 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id037 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id038 + eval_data: *id037 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id039 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id040 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id039 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id040 + eval_data: *id039 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id041 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id042 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id041 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id042 + eval_data: *id041 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id043 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id044 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id043 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id044 + eval_data: *id043 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id045 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id046 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id045 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id046 + eval_data: *id045 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id047 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id048 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id047 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id048 + eval_data: *id047 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_extra2d.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_extra2d.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_extra2d.csv diff --git a/configs/revision/extensions/extra2d/generated/attacks_finetune_budget.yaml b/configs/revision/extensions/extra2d/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..89200ee --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/attacks_finetune_budget.yaml @@ -0,0 +1,1274 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id031 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id032 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_val.npz::labels + eval_data: *id031 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id032 + eval_data: *id031 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id032 + eval_data: *id031 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id032 + eval_data: *id031 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id033 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id034 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id033 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id034 + eval_data: *id033 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id034 + eval_data: *id033 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id034 + eval_data: *id033 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id035 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id036 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id035 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id036 + eval_data: *id035 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id036 + eval_data: *id035 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id036 + eval_data: *id035 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id037 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id038 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id037 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id038 + eval_data: *id037 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id038 + eval_data: *id037 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id038 + eval_data: *id037 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id039 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id040 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id039 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id040 + eval_data: *id039 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id040 + eval_data: *id039 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id040 + eval_data: *id039 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id041 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id042 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id041 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id042 + eval_data: *id041 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id042 + eval_data: *id041 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id042 + eval_data: *id041 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id043 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id044 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id043 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id044 + eval_data: *id043 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id044 + eval_data: *id043 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id044 + eval_data: *id043 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id045 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id046 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id045 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id046 + eval_data: *id045 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id046 + eval_data: *id045 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id046 + eval_data: *id045 + augmentation_strength: 2 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id047 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id048 + inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_val.npz::labels + eval_data: *id047 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id048 + eval_data: *id047 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id048 + eval_data: *id047 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id048 + eval_data: *id047 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extra2d.csv diff --git a/configs/revision/extensions/extra2d/generated/final_analysis.yaml b/configs/revision/extensions/extra2d/generated/final_analysis.yaml new file mode 100644 index 0000000..f5a406d --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/final_analysis.yaml @@ -0,0 +1,12 @@ +inputs: + cp_pp: ../../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../../results/revision_pipeline/posthoc/quality_metrics_extra2d.csv + attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_extra2d.csv + budget_attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extra2d.csv + runtime: ../../../../../results/revision_pipeline/posthoc/runtime_summary_extra2d.csv + pseudo: ../../../../../results/revision_pipeline/posthoc/pseudo_label_quality_extra2d.csv +full_subset_sizes: + PneumoniaMNIST: 524 + DermaMNIST: 1003 + OrganCMNIST: 2392 +output_dir: results/revision_pipeline/final_extra2d diff --git a/configs/revision/extensions/extra2d/generated/image_quality.yaml b/configs/revision/extensions/extra2d/generated/image_quality.yaml new file mode 100644 index 0000000..c58bea8 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/image_quality.yaml @@ -0,0 +1,1562 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_extra2d.csv diff --git a/configs/revision/extensions/extra2d/generated/pseudo.yaml b/configs/revision/extensions/extra2d/generated/pseudo.yaml new file mode 100644 index 0000000..d42dbe7 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/pseudo.yaml @@ -0,0 +1,1804 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 7 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 11 + predictions: + clean: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_extra2d.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_extra2d.csv diff --git a/configs/revision/extensions/extra2d/generated/runtime.yaml b/configs/revision/extensions/extra2d/generated/runtime.yaml new file mode 100644 index 0000000..57de056 --- /dev/null +++ b/configs/revision/extensions/extra2d/generated/runtime.yaml @@ -0,0 +1,71 @@ +experiments: +- metadata: + dataset: PneumoniaMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../../MedMNIST/PneumoniaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_extra2d/PneumoniaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +- metadata: + dataset: DermaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../../MedMNIST/DermaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_extra2d/DermaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +- metadata: + dataset: OrganCMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../../../MedMNIST/OrganCMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST/raw_test.npz::images + cooked_inputs: ../../../../../artifacts/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../../../experiments/revision_pipeline_extra2d/OrganCMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +output: results/revision_pipeline/posthoc/runtime_summary_extra2d.csv diff --git a/configs/revision/extensions/retina-eps12/generated.yaml b/configs/revision/extensions/retina-eps12/generated.yaml new file mode 100644 index 0000000..f5936ce --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: false + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/extensions/retina-eps12/generated/attacks.yaml b/configs/revision/extensions/retina-eps12/generated/attacks.yaml new file mode 100644 index 0000000..6ce3716 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/attacks.yaml @@ -0,0 +1,394 @@ +experiments: +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_retina-eps12.csv diff --git a/configs/revision/extensions/retina-eps12/generated/attacks_finetune.yaml b/configs/revision/extensions/retina-eps12/generated/attacks_finetune.yaml new file mode 100644 index 0000000..8f8aa87 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/attacks_finetune.yaml @@ -0,0 +1,292 @@ +experiments: +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_retina-eps12.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_retina-eps12.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_retina-eps12.csv diff --git a/configs/revision/extensions/retina-eps12/generated/attacks_finetune_budget.yaml b/configs/revision/extensions/retina-eps12/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..df7add6 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/attacks_finetune_budget.yaml @@ -0,0 +1,426 @@ +experiments: +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_retina-eps12.csv diff --git a/configs/revision/extensions/retina-eps12/generated/final_analysis.yaml b/configs/revision/extensions/retina-eps12/generated/final_analysis.yaml new file mode 100644 index 0000000..7da88d2 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/final_analysis.yaml @@ -0,0 +1,10 @@ +inputs: + cp_pp: ../../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../../results/revision_pipeline/posthoc/quality_metrics_retina-eps12.csv + attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_retina-eps12.csv + budget_attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_retina-eps12.csv + runtime: ../../../../../results/revision_pipeline/posthoc/runtime_summary_retina-eps12.csv + pseudo: ../../../../../results/revision_pipeline/posthoc/pseudo_label_quality_retina-eps12.csv +full_subset_sizes: + RetinaMNIST: 120 +output_dir: results/revision_pipeline/final_retina-eps12 diff --git a/configs/revision/extensions/retina-eps12/generated/image_quality.yaml b/configs/revision/extensions/retina-eps12/generated/image_quality.yaml new file mode 100644 index 0000000..a0e6cf7 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/image_quality.yaml @@ -0,0 +1,106 @@ +experiments: +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_retina-eps12.csv diff --git a/configs/revision/extensions/retina-eps12/generated/pseudo.yaml b/configs/revision/extensions/retina-eps12/generated/pseudo.yaml new file mode 100644 index 0000000..5efb8e8 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/pseudo.yaml @@ -0,0 +1,124 @@ +experiments: +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_retina-eps12.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_retina-eps12.csv diff --git a/configs/revision/extensions/retina-eps12/generated/runtime.yaml b/configs/revision/extensions/retina-eps12/generated/runtime.yaml new file mode 100644 index 0000000..25af3f3 --- /dev/null +++ b/configs/revision/extensions/retina-eps12/generated/runtime.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline/posthoc/runtime_summary_retina-eps12.csv diff --git a/configs/revision/extensions/seed3-7-all/generated.yaml b/configs/revision/extensions/seed3-7-all/generated.yaml new file mode 100644 index 0000000..f5936ce --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: false + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/extensions/seed3-7-all/generated/attacks.yaml b/configs/revision/extensions/seed3-7-all/generated/attacks.yaml new file mode 100644 index 0000000..24156c9 --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/attacks.yaml @@ -0,0 +1,737 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-all/generated/attacks_finetune.yaml b/configs/revision/extensions/seed3-7-all/generated/attacks_finetune.yaml new file mode 100644 index 0000000..4fec30b --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/attacks_finetune.yaml @@ -0,0 +1,739 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id007 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id009 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id013 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id015 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id017 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id019 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id020 + eval_data: *id019 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id020 + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id020 + eval_data: *id019 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id022 + eval_data: *id021 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id022 + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id022 + eval_data: *id021 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id024 + eval_data: *id023 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id024 + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id024 + eval_data: *id023 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id026 + eval_data: *id025 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id026 + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id026 + eval_data: *id025 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id028 + eval_data: *id027 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id028 + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id028 + eval_data: *id027 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id030 + eval_data: *id029 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id030 + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id030 + eval_data: *id029 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_seed3-7-all.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_seed3-7-all.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-all/generated/attacks_finetune_budget.yaml b/configs/revision/extensions/seed3-7-all/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..a363818 --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/attacks_finetune_budget.yaml @@ -0,0 +1,797 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-all/generated/final_analysis.yaml b/configs/revision/extensions/seed3-7-all/generated/final_analysis.yaml new file mode 100644 index 0000000..01cb332 --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/final_analysis.yaml @@ -0,0 +1,12 @@ +inputs: + cp_pp: ../../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../../results/revision_pipeline/posthoc/quality_metrics_seed3-7-all.csv + attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_seed3-7-all.csv + budget_attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_seed3-7-all.csv + runtime: ../../../../../results/revision_pipeline/posthoc/runtime_summary_seed3-7-all.csv + pseudo: ../../../../../results/revision_pipeline/posthoc/pseudo_label_quality_seed3-7-all.csv +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 +output_dir: results/revision_pipeline/final_seed3-7-all diff --git a/configs/revision/extensions/seed3-7-all/generated/image_quality.yaml b/configs/revision/extensions/seed3-7-all/generated/image_quality.yaml new file mode 100644 index 0000000..31167ef --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/image_quality.yaml @@ -0,0 +1,977 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-all/generated/pseudo.yaml b/configs/revision/extensions/seed3-7-all/generated/pseudo.yaml new file mode 100644 index 0000000..eeaf9dd --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/pseudo.yaml @@ -0,0 +1,1129 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=16/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_seed3-7-all.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-all/generated/runtime.yaml b/configs/revision/extensions/seed3-7-all/generated/runtime.yaml new file mode 100644 index 0000000..8f23265 --- /dev/null +++ b/configs/revision/extensions/seed3-7-all/generated/runtime.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline/posthoc/runtime_summary_seed3-7-all.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated.yaml b/configs/revision/extensions/seed3-7-eps8/generated.yaml new file mode 100644 index 0000000..f5936ce --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: false + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/extensions/seed3-7-eps8/generated/attacks.yaml b/configs/revision/extensions/seed3-7-eps8/generated/attacks.yaml new file mode 100644 index 0000000..e7d40dc --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/attacks.yaml @@ -0,0 +1,737 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results_seed3-7-eps8.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune.yaml b/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune.yaml new file mode 100644 index 0000000..2af6cba --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune.yaml @@ -0,0 +1,544 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id020 + eval_data: *id019 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id022 + eval_data: *id021 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id024 + eval_data: *id023 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id026 + eval_data: *id025 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id028 + eval_data: *id027 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id030 + eval_data: *id029 +output: results/revision_pipeline/posthoc/adaptive_finetune_results_seed3-7-eps8.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results_seed3-7-eps8.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all_seed3-7-eps8.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune_budget.yaml b/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..906ab63 --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/attacks_finetune_budget.yaml @@ -0,0 +1,797 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id018 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id019 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id020 + inputs: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id020 + eval_data: *id019 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id021 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id022 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id022 + eval_data: *id021 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id023 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id024 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id024 + eval_data: *id023 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id025 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id026 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id026 + eval_data: *id025 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id027 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id028 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id028 + eval_data: *id027 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id029 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id030 + inputs: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id030 + eval_data: *id029 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_seed3-7-eps8.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated/final_analysis.yaml b/configs/revision/extensions/seed3-7-eps8/generated/final_analysis.yaml new file mode 100644 index 0000000..6b78b59 --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/final_analysis.yaml @@ -0,0 +1,12 @@ +inputs: + cp_pp: ../../../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../../../results/revision_pipeline/posthoc/quality_metrics_seed3-7-eps8.csv + attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_attack_results_all_seed3-7-eps8.csv + budget_attacks: ../../../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_seed3-7-eps8.csv + runtime: ../../../../../results/revision_pipeline/posthoc/runtime_summary_seed3-7-eps8.csv + pseudo: ../../../../../results/revision_pipeline/posthoc/pseudo_label_quality_seed3-7-eps8.csv +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 +output_dir: results/revision_pipeline/final_seed3-7-eps8 diff --git a/configs/revision/extensions/seed3-7-eps8/generated/image_quality.yaml b/configs/revision/extensions/seed3-7-eps8/generated/image_quality.yaml new file mode 100644 index 0000000..32310ff --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/image_quality.yaml @@ -0,0 +1,197 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + images: + raw: ../../../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics_seed3-7-eps8.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated/pseudo.yaml b/configs/revision/extensions/seed3-7-eps8/generated/pseudo.yaml new file mode 100644 index 0000000..73f7ce3 --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/pseudo.yaml @@ -0,0 +1,229 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 3 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=3-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 4 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=4-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 5 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=5-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 6 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=6-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 7 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/clean.npz + authorized_cooked: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=7-eps=8/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples_seed3-7-eps8.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality_seed3-7-eps8.csv diff --git a/configs/revision/extensions/seed3-7-eps8/generated/runtime.yaml b/configs/revision/extensions/seed3-7-eps8/generated/runtime.yaml new file mode 100644 index 0000000..9b1037b --- /dev/null +++ b/configs/revision/extensions/seed3-7-eps8/generated/runtime.yaml @@ -0,0 +1,2 @@ +experiments: [] +output: results/revision_pipeline/posthoc/runtime_summary_seed3-7-eps8.csv diff --git a/configs/revision/final_analysis.yaml b/configs/revision/final_analysis.yaml new file mode 100644 index 0000000..3c02aa7 --- /dev/null +++ b/configs/revision/final_analysis.yaml @@ -0,0 +1,24 @@ +inputs: + cp_pp: ../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../results/revision_pipeline/posthoc/quality_metrics.csv + attacks: ../../results/revision_pipeline/posthoc/adaptive_attack_results_all.csv + # Optional on the first analysis pass; created by the budget-attack job. + budget_attacks: ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv + runtime: ../../results/revision_pipeline/posthoc/runtime_summary.csv + pseudo: ../../results/revision_pipeline/posthoc/pseudo_label_quality.csv + +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 + +significance: + metrics: [acc, macro_f1, auc, macro_auc] + epsilons_pixel_by_dataset: + BloodMNIST: [8] + BreastMNIST: [8] + RetinaMNIST: [8] + attack_types: [finetune, robust_finetune] + epochs: [10] + +output_dir: results/revision_pipeline/final diff --git a/configs/revision/final_analysis_2d_8seeds.yaml b/configs/revision/final_analysis_2d_8seeds.yaml new file mode 100644 index 0000000..60b0827 --- /dev/null +++ b/configs/revision/final_analysis_2d_8seeds.yaml @@ -0,0 +1,25 @@ +inputs: + cp_pp: ../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../results/revision_pipeline/posthoc/quality_metrics_extended_2d_8seeds.csv + attacks: ../../results/revision_pipeline/posthoc/adaptive_attack_results_extended_2d_8seeds.csv + budget_attacks: ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extended_2d_8seeds.csv + runtime: ../../results/revision_pipeline/posthoc/runtime_summary.csv + pseudo: ../../results/revision_pipeline/posthoc/pseudo_label_quality_extended_2d_8seeds.csv + +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 + +# Pre-registered primary tests. Other metrics remain available in the aggregate +# and paper tables but are not added to the multiple-testing family. +significance: + metrics: [acc, macro_f1, auc, macro_auc] + epsilons_pixel_by_dataset: + BloodMNIST: [8] + BreastMNIST: [8] + RetinaMNIST: [8, 12] + attack_types: [finetune, robust_finetune] + epochs: [10] + +output_dir: results/revision_pipeline/final_2d_8seeds diff --git a/configs/revision/final_analysis_2d_six_datasets.yaml b/configs/revision/final_analysis_2d_six_datasets.yaml new file mode 100644 index 0000000..7687472 --- /dev/null +++ b/configs/revision/final_analysis_2d_six_datasets.yaml @@ -0,0 +1,29 @@ +inputs: + cp_pp: ../../results/revision_pipeline_6datasets/summary/cp_pp_summary.csv + quality: ../../results/revision_pipeline_6datasets/posthoc/quality_metrics.csv + attacks: ../../results/revision_pipeline_6datasets/posthoc/adaptive_attack_results.csv + budget_attacks: ../../results/revision_pipeline_6datasets/posthoc/adaptive_finetune_budget_results.csv + runtime: ../../results/revision_pipeline_6datasets/posthoc/runtime_summary.csv + pseudo: ../../results/revision_pipeline_6datasets/posthoc/pseudo_label_quality.csv + +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 + PneumoniaMNIST: 524 + DermaMNIST: 1003 + OrganCMNIST: 2392 + +significance: + metrics: [acc, macro_f1, auc, macro_auc] + epsilons_pixel_by_dataset: + BloodMNIST: [8] + BreastMNIST: [8] + RetinaMNIST: [8, 12] + PneumoniaMNIST: [8] + DermaMNIST: [8] + OrganCMNIST: [8] + attack_types: [finetune, robust_finetune] + epochs: [10] + +output_dir: results/revision_pipeline_6datasets/final diff --git a/configs/revision/final_analysis_extra2d_8seeds.yaml b/configs/revision/final_analysis_extra2d_8seeds.yaml new file mode 100644 index 0000000..27e3095 --- /dev/null +++ b/configs/revision/final_analysis_extra2d_8seeds.yaml @@ -0,0 +1,23 @@ +inputs: + cp_pp: ../../results/revision_pipeline_extra2d/summary/cp_pp_summary.csv + quality: ../../results/revision_pipeline/posthoc/quality_metrics_extra2d.csv + attacks: ../../results/revision_pipeline/posthoc/adaptive_attack_results_all_extra2d.csv + budget_attacks: ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extra2d.csv + runtime: ../../results/revision_pipeline/posthoc/runtime_summary_extra2d.csv + pseudo: ../../results/revision_pipeline/posthoc/pseudo_label_quality_extra2d.csv + +full_subset_sizes: + PneumoniaMNIST: 524 + DermaMNIST: 1003 + OrganCMNIST: 2392 + +significance: + metrics: [acc, macro_f1, auc, macro_auc] + epsilons_pixel_by_dataset: + PneumoniaMNIST: [8] + DermaMNIST: [8] + OrganCMNIST: [8] + attack_types: [finetune, robust_finetune] + epochs: [10] + +output_dir: results/revision_pipeline_extra2d/final_8seeds diff --git a/configs/revision/generated.yaml b/configs/revision/generated.yaml new file mode 100644 index 0000000..eb0f4c3 --- /dev/null +++ b/configs/revision/generated.yaml @@ -0,0 +1,31 @@ +tasks: + metrics: + enabled: false + config: metrics.yaml + attacks: + enabled: true + config: generated/attacks.yaml + finetune_attacks: + enabled: false + config: generated/attacks_finetune.yaml + finetune_budget_attacks: + enabled: false + config: generated/attacks_finetune_budget.yaml + epsilon: + enabled: false + config: epsilon.yaml + pseudo: + enabled: true + config: generated/pseudo.yaml + statistics: + enabled: false + config: statistics.yaml + runtime: + enabled: true + config: generated/runtime.yaml + image_quality: + enabled: true + config: generated/image_quality.yaml + final_analysis: + enabled: false + config: generated/final_analysis.yaml diff --git a/configs/revision/generated/attacks.yaml b/configs/revision/generated/attacks.yaml new file mode 100644 index 0000000..4f344d4 --- /dev/null +++ b/configs/revision/generated/attacks.yaml @@ -0,0 +1,443 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: jpeg + strength: 95 + - type: jpeg + strength: 75 + - type: jpeg + strength: 50 + - type: gaussian_smoothing + strength: + sigma: 0.5 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 1.0 + kernel_size: 5 + - type: gaussian_smoothing + strength: + sigma: 2.0 + kernel_size: 5 + - type: strong_augmentation + strength: 1 + - type: strong_augmentation + strength: 2 + - type: strong_augmentation + strength: 3 + - type: median_filter + strength: 3 + - type: median_filter + strength: 5 +output: results/revision_pipeline/posthoc/adaptive_attack_results.csv diff --git a/configs/revision/generated/attacks_finetune.yaml b/configs/revision/generated/attacks_finetune.yaml new file mode 100644 index 0000000..14b38f1 --- /dev/null +++ b/configs/revision/generated/attacks_finetune.yaml @@ -0,0 +1,445 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id002 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id002 + eval_data: *id001 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id004 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id004 + eval_data: *id003 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id006 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id006 + eval_data: *id005 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id008 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id007 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id008 + eval_data: *id007 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id010 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id009 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id010 + eval_data: *id009 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id012 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id012 + eval_data: *id011 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id014 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id014 + eval_data: *id013 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id016 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id016 + eval_data: *id015 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: &id018 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 5 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + augmentation_strength: 2 + raw_subset: *id018 + eval_data: *id017 +output: results/revision_pipeline/posthoc/adaptive_finetune_results.csv +merge_with: results/revision_pipeline/posthoc/adaptive_attack_results.csv +combined_output: results/revision_pipeline/posthoc/adaptive_attack_results_all.csv diff --git a/configs/revision/generated/attacks_finetune_budget.yaml b/configs/revision/generated/attacks_finetune_budget.yaml new file mode 100644 index 0000000..492d040 --- /dev/null +++ b/configs/revision/generated/attacks_finetune_budget.yaml @@ -0,0 +1,479 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id001 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id002 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id002 + eval_data: *id001 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id003 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id004 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id004 + eval_data: *id003 + augmentation_strength: 2 +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id005 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id006 + inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BloodMNIST/raw_val.npz::labels + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id006 + eval_data: *id005 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id007 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id008 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id008 + eval_data: *id007 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id009 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id010 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id010 + eval_data: *id009 + augmentation_strength: 2 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id011 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id012 + inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/BreastMNIST/raw_val.npz::labels + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id012 + eval_data: *id011 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id013 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id014 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id014 + eval_data: *id013 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id015 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id016 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id016 + eval_data: *id015 + augmentation_strength: 2 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/protected/resnet18/checkpoints/resnet18.pth + eval_data: &id017 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::labels + attacks: + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: &id018 + inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::images + labels: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_val.npz::labels + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 25 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 + - type: finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + - type: robust_finetune + epochs: 10 + learning_rate: 0.0001 + batch_size: 64 + subset_size: 50 + raw_subset: *id018 + eval_data: *id017 + augmentation_strength: 2 +output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv diff --git a/configs/revision/generated/final_analysis.yaml b/configs/revision/generated/final_analysis.yaml new file mode 100644 index 0000000..766dcbb --- /dev/null +++ b/configs/revision/generated/final_analysis.yaml @@ -0,0 +1,12 @@ +inputs: + cp_pp: ../../../results/revision_pipeline/summary/cp_pp_summary.csv + quality: ../../../results/revision_pipeline/posthoc/quality_metrics.csv + attacks: ../../../results/revision_pipeline/posthoc/adaptive_attack_results_all.csv + budget_attacks: ../../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv + runtime: ../../../results/revision_pipeline/posthoc/runtime_summary.csv + pseudo: ../../../results/revision_pipeline/posthoc/pseudo_label_quality.csv +full_subset_sizes: + BloodMNIST: 1712 + BreastMNIST: 78 + RetinaMNIST: 120 +output_dir: results/revision_pipeline/final diff --git a/configs/revision/generated/image_quality.yaml b/configs/revision/generated/image_quality.yaml new file mode 100644 index 0000000..b72b325 --- /dev/null +++ b/configs/revision/generated/image_quality.yaml @@ -0,0 +1,587 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/cooked_test.npz::images + compute_lpips: true +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + images: + raw: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/cooked_test.npz::images + compute_lpips: true +output: results/revision_pipeline/posthoc/quality_metrics.csv diff --git a/configs/revision/generated/pseudo.yaml b/configs/revision/generated/pseudo.yaml new file mode 100644 index 0000000..a110e34 --- /dev/null +++ b/configs/revision/generated/pseudo.yaml @@ -0,0 +1,679 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 8 + predictions: + clean: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 2 + predictions: + clean: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 1 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=1-eps=16/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.00784313725490196 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=2/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.01568627450980392 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=4/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.03137254901960784 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=8/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.047058823529411764 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=12/resnet18/pseudo_labels_train_dataset.npz +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 2 + epsilon: 0.06274509803921569 + num_classes: 5 + predictions: + clean: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/clean.npz + authorized_cooked: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/authorized_cooked.npz + unauthorized_raw: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/predictions/unauthorized_raw.npz + pseudo_labels: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=2-eps=16/resnet18/pseudo_labels_train_dataset.npz +outputs: + samples: results/revision_pipeline/posthoc/pseudo_label_samples.csv + quality: results/revision_pipeline/posthoc/pseudo_label_quality.csv diff --git a/configs/revision/generated/runtime.yaml b/configs/revision/generated/runtime.yaml new file mode 100644 index 0000000..83b2f8e --- /dev/null +++ b/configs/revision/generated/runtime.yaml @@ -0,0 +1,71 @@ +experiments: +- metadata: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../MedMNIST/BloodMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../artifacts/revision_pipeline/BloodMNIST/raw_test.npz::images + cooked_inputs: ../../../artifacts/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../experiments/revision_pipeline/BloodMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +- metadata: + dataset: BreastMNIST + modality: 2d_image + task_type: binary + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../MedMNIST/BreastMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../artifacts/revision_pipeline/BreastMNIST/raw_test.npz::images + cooked_inputs: ../../../artifacts/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../experiments/revision_pipeline/BreastMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +- metadata: + dataset: RetinaMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + seed: 0 + epsilon: 0.03137254901960784 + model: + config_path: ../../MedMNIST/RetinaMNIST/resnet18.yaml + checkpoint: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/protected/resnet18/checkpoints/resnet18.pth + data: + raw_inputs: ../../../artifacts/revision_pipeline/RetinaMNIST/raw_test.npz::images + cooked_inputs: ../../../artifacts/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/cooked_test.npz::images + cooker: + perturbation: ../../../experiments/revision_pipeline/RetinaMNIST-datacook-sur=resnet18-vic=resnet18-seed=0-eps=8/test_perturbation.pt + clamp: + - 0 + - 1 + batch_size: 128 + repeats: 5 + warmup: 1 +output: results/revision_pipeline/posthoc/runtime_summary.csv diff --git a/configs/revision/image_quality.yaml b/configs/revision/image_quality.yaml new file mode 100644 index 0000000..681d39a --- /dev/null +++ b/configs/revision/image_quality.yaml @@ -0,0 +1,7 @@ +experiments: [] +# - metadata: {dataset: BloodMNIST, modality: 2d_image, task_type: multiclass, +# method: datacook, surrogate_model: resnet18, victim_model: resnet18, +# seed: 0, epsilon: 0.031372549, setting: image_quality} +# images: {raw: ../../artifacts/images.npz::raw, cooked: ../../artifacts/images.npz::cooked} +# compute_lpips: true +output: results/image_quality/quality_metrics.csv diff --git a/configs/revision/merge_2d_8seeds.yaml b/configs/revision/merge_2d_8seeds.yaml new file mode 100644 index 0000000..915ea3b --- /dev/null +++ b/configs/revision/merge_2d_8seeds.yaml @@ -0,0 +1,23 @@ +groups: + adaptive_attacks: + inputs: + - ../../results/revision_pipeline/posthoc/adaptive_attack_results_all.csv + - ../../results/revision_pipeline/posthoc/adaptive_attack_results_all_seed3-7-eps8.csv + - ../../results/revision_pipeline/posthoc/adaptive_attack_results_all_retina-eps12.csv + output: results/revision_pipeline/posthoc/adaptive_attack_results_extended_2d_8seeds.csv + finetune_budgets: + inputs: + - ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv + - ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_seed3-7-eps8.csv + - ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_retina-eps12.csv + output: results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extended_2d_8seeds.csv + image_quality: + inputs: + - ../../results/revision_pipeline/posthoc/quality_metrics.csv + - ../../results/revision_pipeline/posthoc/quality_metrics_seed3-7-all.csv + output: results/revision_pipeline/posthoc/quality_metrics_extended_2d_8seeds.csv + pseudo_quality: + inputs: + - ../../results/revision_pipeline/posthoc/pseudo_label_quality.csv + - ../../results/revision_pipeline/posthoc/pseudo_label_quality_seed3-7-all.csv + output: results/revision_pipeline/posthoc/pseudo_label_quality_extended_2d_8seeds.csv diff --git a/configs/revision/merge_2d_six_datasets.yaml b/configs/revision/merge_2d_six_datasets.yaml new file mode 100644 index 0000000..4500553 --- /dev/null +++ b/configs/revision/merge_2d_six_datasets.yaml @@ -0,0 +1,36 @@ +groups: + main_metrics: + inputs: + - ../../results/revision_pipeline/summary/main_metrics.csv + - ../../results/revision_pipeline_extra2d/summary/main_metrics.csv + output: results/revision_pipeline_6datasets/summary/main_metrics.csv + cp_pp: + inputs: + - ../../results/revision_pipeline/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_extra2d/summary/cp_pp_summary.csv + output: results/revision_pipeline_6datasets/summary/cp_pp_summary.csv + adaptive_attacks: + inputs: + - ../../results/revision_pipeline/posthoc/adaptive_attack_results_extended_2d_8seeds.csv + - ../../results/revision_pipeline/posthoc/adaptive_attack_results_all_extra2d.csv + output: results/revision_pipeline_6datasets/posthoc/adaptive_attack_results.csv + finetune_budgets: + inputs: + - ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extended_2d_8seeds.csv + - ../../results/revision_pipeline/posthoc/adaptive_finetune_budget_results_extra2d.csv + output: results/revision_pipeline_6datasets/posthoc/adaptive_finetune_budget_results.csv + image_quality: + inputs: + - ../../results/revision_pipeline/posthoc/quality_metrics_extended_2d_8seeds.csv + - ../../results/revision_pipeline/posthoc/quality_metrics_extra2d.csv + output: results/revision_pipeline_6datasets/posthoc/quality_metrics.csv + pseudo_quality: + inputs: + - ../../results/revision_pipeline/posthoc/pseudo_label_quality_extended_2d_8seeds.csv + - ../../results/revision_pipeline/posthoc/pseudo_label_quality_extra2d.csv + output: results/revision_pipeline_6datasets/posthoc/pseudo_label_quality.csv + runtime: + inputs: + - ../../results/revision_pipeline/posthoc/runtime_summary.csv + - ../../results/revision_pipeline/posthoc/runtime_summary_extra2d.csv + output: results/revision_pipeline_6datasets/posthoc/runtime_summary.csv diff --git a/configs/revision/merge_tabular_epsilon.yaml b/configs/revision/merge_tabular_epsilon.yaml new file mode 100644 index 0000000..e2d72c0 --- /dev/null +++ b/configs/revision/merge_tabular_epsilon.yaml @@ -0,0 +1,25 @@ +groups: + main_metrics: + inputs: + - ../../results/revision_pipeline_tabular/datasets/diabetes/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular/datasets/hepatitis_c/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular/datasets/systemic_diseases/summary/main_metrics.csv + output: results/revision_pipeline_tabular/summary/main_metrics.csv + cp_pp: + inputs: + - ../../results/revision_pipeline_tabular/datasets/diabetes/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular/datasets/hepatitis_c/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular/datasets/systemic_diseases/summary/cp_pp_summary.csv + output: results/revision_pipeline_tabular/summary/cp_pp_summary.csv + epsilon: + inputs: + - ../../results/revision_pipeline_tabular/datasets/diabetes/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular/datasets/hepatitis_c/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular/datasets/systemic_diseases/epsilon_ablation.csv + output: results/revision_pipeline_tabular/epsilon_ablation.csv + runtime: + inputs: + - ../../results/revision_pipeline_tabular/datasets/diabetes/runtime_summary.csv + - ../../results/revision_pipeline_tabular/datasets/hepatitis_c/runtime_summary.csv + - ../../results/revision_pipeline_tabular/datasets/systemic_diseases/runtime_summary.csv + output: results/revision_pipeline_tabular/runtime_summary.csv diff --git a/configs/revision/merge_tabular_legacy_extended.yaml b/configs/revision/merge_tabular_legacy_extended.yaml new file mode 100644 index 0000000..5123a08 --- /dev/null +++ b/configs/revision/merge_tabular_legacy_extended.yaml @@ -0,0 +1,25 @@ +groups: + main_metrics: + inputs: + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/diabetes/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/hepatitis_c/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/systemic_diseases/summary/main_metrics.csv + output: results/revision_pipeline_tabular_legacy_extended/summary/main_metrics.csv + cp_pp: + inputs: + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/diabetes/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/hepatitis_c/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/systemic_diseases/summary/cp_pp_summary.csv + output: results/revision_pipeline_tabular_legacy_extended/summary/cp_pp_summary.csv + epsilon: + inputs: + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/diabetes/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/hepatitis_c/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/systemic_diseases/epsilon_ablation.csv + output: results/revision_pipeline_tabular_legacy_extended/epsilon_ablation.csv + runtime: + inputs: + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/diabetes/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/hepatitis_c/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_extended/datasets/systemic_diseases/runtime_summary.csv + output: results/revision_pipeline_tabular_legacy_extended/runtime_summary.csv diff --git a/configs/revision/merge_tabular_legacy_params.yaml b/configs/revision/merge_tabular_legacy_params.yaml new file mode 100644 index 0000000..2e570df --- /dev/null +++ b/configs/revision/merge_tabular_legacy_params.yaml @@ -0,0 +1,25 @@ +groups: + main_metrics: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/datasets/diabetes/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/hepatitis_c/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/systemic_diseases/summary/main_metrics.csv + output: results/revision_pipeline_tabular_legacy_params/summary/main_metrics.csv + cp_pp: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/datasets/diabetes/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/hepatitis_c/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/systemic_diseases/summary/cp_pp_summary.csv + output: results/revision_pipeline_tabular_legacy_params/summary/cp_pp_summary.csv + epsilon: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/datasets/diabetes/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/hepatitis_c/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/systemic_diseases/epsilon_ablation.csv + output: results/revision_pipeline_tabular_legacy_params/epsilon_ablation.csv + runtime: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/datasets/diabetes/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/hepatitis_c/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/datasets/systemic_diseases/runtime_summary.csv + output: results/revision_pipeline_tabular_legacy_params/runtime_summary.csv diff --git a/configs/revision/merge_tabular_legacy_params_7gpu.yaml b/configs/revision/merge_tabular_legacy_params_7gpu.yaml new file mode 100644 index 0000000..73c38da --- /dev/null +++ b/configs/revision/merge_tabular_legacy_params_7gpu.yaml @@ -0,0 +1,45 @@ +groups: + main_metrics: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/shards/diabetes/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/hepatitis_c/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-low/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-high/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-low/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-high/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-low/summary/main_metrics.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-high/summary/main_metrics.csv + output: results/revision_pipeline_tabular_legacy_params/summary/main_metrics.csv + cp_pp: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/shards/diabetes/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/hepatitis_c/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-low/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-high/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-low/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-high/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-low/summary/cp_pp_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-high/summary/cp_pp_summary.csv + output: results/revision_pipeline_tabular_legacy_params/summary/cp_pp_summary.csv + epsilon: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/shards/diabetes/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/hepatitis_c/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-low/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-high/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-low/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-high/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-low/epsilon_ablation.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-high/epsilon_ablation.csv + output: results/revision_pipeline_tabular_legacy_params/epsilon_ablation.csv + runtime: + inputs: + - ../../results/revision_pipeline_tabular_legacy_params/shards/diabetes/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/hepatitis_c/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-low/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s0-high/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-low/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s1-high/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-low/runtime_summary.csv + - ../../results/revision_pipeline_tabular_legacy_params/shards/systemic-s2-high/runtime_summary.csv + output: results/revision_pipeline_tabular_legacy_params/runtime_summary.csv diff --git a/configs/revision/metrics.yaml b/configs/revision/metrics.yaml new file mode 100644 index 0000000..c589863 --- /dev/null +++ b/configs/revision/metrics.yaml @@ -0,0 +1,21 @@ +seeds: [0, 1, 2] +metadata_defaults: + dataset: BloodMNIST + modality: 2d_image + task_type: multiclass + method: datacook + surrogate_model: resnet18 + victim_model: resnet18 + epsilon: 0.03137254901960784 +# Paths can use {seed}, {epsilon}, and {epsilon_pixel} templates. +experiments: [] +# Example record: +# - num_classes: 8 +# metric_names: [acc, macro_auc, macro_f1, macro_sensitivity, macro_specificity] +# predictions: +# clean: ../../artifacts/seed_{seed}/clean.npz +# authorized_cooked: ../../artifacts/seed_{seed}/authorized_cooked.npz +# unauthorized_raw: ../../artifacts/seed_{seed}/unauthorized_raw.npz +outputs: + main_metrics: results/metrics/main_metrics.csv + cp_pp: results/metrics/cp_pp_summary.csv diff --git a/configs/revision/pseudo.yaml b/configs/revision/pseudo.yaml new file mode 100644 index 0000000..8556125 --- /dev/null +++ b/configs/revision/pseudo.yaml @@ -0,0 +1,6 @@ +experiments: [] +# pseudo_labels: ../../experiment/resnet18/pseudo_labels_train_dataset.npz +# predictions: {clean: ..., authorized_cooked: ..., unauthorized_raw: ...} +outputs: + samples: results/analysis/pseudo_label_samples.csv + quality: results/analysis/pseudo_label_quality.csv diff --git a/configs/revision/runtime.yaml b/configs/revision/runtime.yaml new file mode 100644 index 0000000..1ea55d5 --- /dev/null +++ b/configs/revision/runtime.yaml @@ -0,0 +1,4 @@ +experiments: [] +# Required per record: metadata, model, data.raw_inputs, and cooker. +# cooker: {perturbation: ../../artifacts/test_perturbation.pt, clamp: [0, 1]} +output: results/runtime/runtime_summary.csv diff --git a/configs/revision/statistics.yaml b/configs/revision/statistics.yaml new file mode 100644 index 0000000..2799477 --- /dev/null +++ b/configs/revision/statistics.yaml @@ -0,0 +1,7 @@ +inputs: [] +# - {path: ../../results/metrics/cp_pp_summary.csv, +# experiment_type: main, value_columns: [cp, pp]} +comparisons: [] +# - {method_a: datacook, method_b: EM-Pseudo, metrics: [cp:acc, pp:acc], test: auto} +aggregated_output: results/statistics/aggregated_results.csv +significance_output: results/statistics/significance_tests.csv diff --git a/configs/revision/statistics_tabular_epsilon.yaml b/configs/revision/statistics_tabular_epsilon.yaml new file mode 100644 index 0000000..af9d6b3 --- /dev/null +++ b/configs/revision/statistics_tabular_epsilon.yaml @@ -0,0 +1,13 @@ +inputs: + - path: ../../results/revision_pipeline_tabular/epsilon_ablation.csv + experiment_type: epsilon_ablation + value_columns: + - clean + - authorized_cooked + - unauthorized_raw + - cp + - pp + - residual_utility + +aggregated_output: results/revision_pipeline_tabular/aggregated_results.csv +significance_output: results/revision_pipeline_tabular/significance_tests.csv diff --git a/configs/revision/statistics_tabular_legacy_extended.yaml b/configs/revision/statistics_tabular_legacy_extended.yaml new file mode 100644 index 0000000..a6d213b --- /dev/null +++ b/configs/revision/statistics_tabular_legacy_extended.yaml @@ -0,0 +1,20 @@ +inputs: + - path: ../../results/revision_pipeline_tabular_legacy_extended/epsilon_ablation.csv + experiment_type: historical_protocol_budget_ablation + value_columns: + - clean + - authorized_cooked + - unauthorized_raw + - cp + - pp + - residual_utility + +aggregated_output: results/revision_pipeline_tabular_legacy_extended/aggregated_results.csv +significance_input: ../../results/revision_pipeline_tabular_legacy_extended/epsilon_ablation.csv +significance_output: results/revision_pipeline_tabular_legacy_extended/significance_tests.csv +primary_metrics_by_task: + binary: acc + multiclass: acc + multilabel: subset_acc +# Correct the six budget-wise tests separately for each dataset/comparison. +holm_family: [dataset, comparison] diff --git a/configs/revision/statistics_tabular_legacy_params.yaml b/configs/revision/statistics_tabular_legacy_params.yaml new file mode 100644 index 0000000..326479b --- /dev/null +++ b/configs/revision/statistics_tabular_legacy_params.yaml @@ -0,0 +1,13 @@ +inputs: + - path: ../../results/revision_pipeline_tabular_legacy_params/epsilon_ablation.csv + experiment_type: legacy_parameter_budget_ablation + value_columns: + - clean + - authorized_cooked + - unauthorized_raw + - cp + - pp + - residual_utility + +aggregated_output: results/revision_pipeline_tabular_legacy_params/aggregated_results.csv +significance_output: results/revision_pipeline_tabular_legacy_params/significance_tests.csv diff --git a/configs/revision/tabular_epsilon_diabetes.yaml b/configs/revision/tabular_epsilon_diabetes.yaml new file mode 100644 index 0000000..2b74217 --- /dev/null +++ b/configs/revision/tabular_epsilon_diabetes.yaml @@ -0,0 +1,38 @@ +dataset: + name: Diabetes + task_type: binary + num_classes: 2 + # Relative to this YAML. Override with --data-path or TABULAR_DATA in the job. + data_path: ../../TableData/dataset/diabetes.csv + features: + - Pregnancies + - Glucose + - BloodPressure + - SkinThickness + - Insulin + - BMI + - DiabetesPedigreeFunction + - Age + target: Outcome + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + # Epsilon uses standardized-feature L-inf units, not /255 pixel units. + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0] + attackable_features: [BMI, DiabetesPedigreeFunction] + num_steps: 20 + step_size_ratio: 0.25 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: true + +outputs: + artifact_root: artifacts/revision_pipeline_tabular + result_root: results/revision_pipeline_tabular/datasets/diabetes diff --git a/configs/revision/tabular_epsilon_hepatitis_c.yaml b/configs/revision/tabular_epsilon_hepatitis_c.yaml new file mode 100644 index 0000000..7846d31 --- /dev/null +++ b/configs/revision/tabular_epsilon_hepatitis_c.yaml @@ -0,0 +1,50 @@ +dataset: + name: HepatitisC + task_type: multiclass + num_classes: 5 + data_path: ../../TableData/dataset/HepatitisCdata.csv + features: + - Age + - Sex + - ALB + - ALP + - ALT + - AST + - BIL + - CHE + - CHOL + - CREA + - GGT + - PROT + target: Category + categorical_mappings: + Sex: + "m": 0 + "f": 1 + target_mapping: + "0=Blood Donor": 0 + "0s=suspect Blood Donor": 1 + "1=Hepatitis": 2 + "2=Fibrosis": 3 + "3=Cirrhosis": 4 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0] + attackable_features: [ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + num_steps: 20 + step_size_ratio: 0.25 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: true + +outputs: + artifact_root: artifacts/revision_pipeline_tabular + result_root: results/revision_pipeline_tabular/datasets/hepatitis_c diff --git a/configs/revision/tabular_epsilon_systemic_diseases.yaml b/configs/revision/tabular_epsilon_systemic_diseases.yaml new file mode 100644 index 0000000..8757a69 --- /dev/null +++ b/configs/revision/tabular_epsilon_systemic_diseases.yaml @@ -0,0 +1,64 @@ +dataset: + name: SystemicDiseases + task_type: multilabel + data_path: "../../TableData/dataset/systemic diseases.csv" + features: + - Age + - Gender + - Ethnic + - Cholesterol + - HDL + - Hyperlipidemia + - BMI + - DBP + - SBP + - Smoking + - Hypertension + - Diabetes + - ASCVD + - MI + - Stroke + - dem + - PD + targets: [inciASCVD, inciMI, inciStroke, incidem, inciPD] + categorical_mappings: + Gender: + "Male": 0 + "Female": 1 + Ethnic: + "Non-Whites": 0 + "Whites": 1 + Hyperlipidemia: + "yes": 1 + "no": 0 + Hypertension: + "yes": 1 + "no": 0 + Diabetes: + "yes": 1 + "no": 0 + Smoking: + "yes": 1 + "no": 0 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0] + attackable_features: [Cholesterol, HDL, BMI, DBP, SBP] + num_steps: 20 + step_size_ratio: 0.25 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 256 + learning_rate: 0.001 + class_balance: true + max_positive_weight: 100.0 + +outputs: + artifact_root: artifacts/revision_pipeline_tabular + result_root: results/revision_pipeline_tabular/datasets/systemic_diseases diff --git a/configs/revision/tabular_legacy_exact_diabetes.yaml b/configs/revision/tabular_legacy_exact_diabetes.yaml new file mode 100644 index 0000000..5d3f9e8 --- /dev/null +++ b/configs/revision/tabular_legacy_exact_diabetes.yaml @@ -0,0 +1,44 @@ +dataset: + name: Diabetes + task_type: binary + num_classes: 2 + data_path: ../../TableData/dataset/diabetes.csv + features: [Pregnancies, Glucose, BloodPressure, SkinThickness, Insulin, BMI, DiabetesPedigreeFunction, Age] + target: Outcome + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model_diabete.pt + require_baseline_checkpoint: true + +experiment: + # Historical single-run protocol used by the preliminary table. + seeds: [42] + split_seed: 42 + epsilons: [5.0] + attackable_features: [BMI, DiabetesPedigreeFunction] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + baseline_select_best_on_test: true + baseline_shuffle: true + baseline_global_data_order: true + protected_shuffle: false + protected_global_data_order: true + victim_seed: 42 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + protected_seed: 42 + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_exact + result_root: results/revision_pipeline_tabular_legacy_exact/datasets/diabetes diff --git a/configs/revision/tabular_legacy_exact_hepatitis_c.yaml b/configs/revision/tabular_legacy_exact_hepatitis_c.yaml new file mode 100644 index 0000000..f0f0b58 --- /dev/null +++ b/configs/revision/tabular_legacy_exact_hepatitis_c.yaml @@ -0,0 +1,51 @@ +dataset: + name: HepatitisC + task_type: multiclass + num_classes: 5 + data_path: ../../TableData/dataset/HepatitisCdata.csv + features: [Age, Sex, ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + target: Category + categorical_mappings: + Sex: {m: 1, f: 2} + target_mapping: + "0=Blood Donor": 0 + "0s=suspect Blood Donor": 1 + "1=Hepatitis": 2 + "2=Fibrosis": 3 + "3=Cirrhosis": 4 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model_HepatitisCdata.pt + require_baseline_checkpoint: true + +experiment: + seeds: [42] + split_seed: 42 + epsilons: [5.0] + attackable_features: [ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + baseline_select_best_on_test: true + baseline_shuffle: true + baseline_global_data_order: true + protected_shuffle: false + protected_global_data_order: true + victim_seed: 42 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + protected_seed: 42 + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_exact + result_root: results/revision_pipeline_tabular_legacy_exact/datasets/hepatitis_c diff --git a/configs/revision/tabular_legacy_exact_systemic_diseases.yaml b/configs/revision/tabular_legacy_exact_systemic_diseases.yaml new file mode 100644 index 0000000..b806a07 --- /dev/null +++ b/configs/revision/tabular_legacy_exact_systemic_diseases.yaml @@ -0,0 +1,52 @@ +dataset: + name: SystemicDiseases + task_type: multilabel + data_path: "../../TableData/TableData_oldversion/systemic diseases.csv" + # Keep the duplicated Smoking column because it exists in the original code. + features: [Age, Gender, Ethnic, Cholesterol, HDL, Hyperlipidemia, BMI, DBP, SBP, Smoking, Hypertension, Diabetes, Smoking, ASCVD, MI, Stroke, dem, PD] + targets: [inciASCVD, inciMI, inciStroke, incidem, inciPD] + categorical_mappings: + Gender: {Male: 0, Female: 1} + Ethnic: {Non-Whites: 0, Whites: 1} + Hyperlipidemia: {"yes": 0, "no": 1} + Hypertension: {"yes": 0, "no": 1} + Diabetes: {"yes": 0, "no": 1} + Smoking: {"yes": 0, "no": 1} + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model.pt + require_baseline_checkpoint: true + +experiment: + seeds: [42] + split_seed: 42 + epsilons: [5.0] + # Exact feature mask from TableData/PrivacyAttacker.py. + attackable_features: [Age, Cholesterol, HDL, Hyperlipidemia, BMI, DBP, SBP, ASCVD, MI, Stroke, dem, PD] + # The old source ignores alpha=0.05 and executes alpha=epsilon/4. + num_steps: 80 + step_size_ratio: 0.25 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + baseline_select_best_on_test: true + baseline_shuffle: true + baseline_global_data_order: true + protected_shuffle: true + protected_global_data_order: true + victim_seed: 42 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + protected_seed: 42 + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_exact + result_root: results/revision_pipeline_tabular_legacy_exact/datasets/systemic_diseases diff --git a/configs/revision/tabular_legacy_extended_diabetes.yaml b/configs/revision/tabular_legacy_extended_diabetes.yaml new file mode 100644 index 0000000..3376bd4 --- /dev/null +++ b/configs/revision/tabular_legacy_extended_diabetes.yaml @@ -0,0 +1,42 @@ +dataset: + name: Diabetes + task_type: binary + num_classes: 2 + data_path: ../../TableData/dataset/diabetes.csv + features: [Pregnancies, Glucose, BloodPressure, SkinThickness, Insulin, BMI, DiabetesPedigreeFunction, Age] + target: Outcome + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model_diabete.pt + require_baseline_checkpoint: true + +experiment: + # Fixed historical split/checkpoint; seeds vary victim initialization only. + seeds: [0, 1, 2, 3, 4, 5, 6, 7] + split_seed: 42 + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [BMI, DiabetesPedigreeFunction] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + protected_shuffle: false + protected_global_data_order: true + victim_seed_offset: 0 + protected_seed_offset: 0 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + reset_rng_each_epsilon: true + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_extended + result_root: results/revision_pipeline_tabular_legacy_extended/datasets/diabetes diff --git a/configs/revision/tabular_legacy_extended_hepatitis_c.yaml b/configs/revision/tabular_legacy_extended_hepatitis_c.yaml new file mode 100644 index 0000000..88329eb --- /dev/null +++ b/configs/revision/tabular_legacy_extended_hepatitis_c.yaml @@ -0,0 +1,49 @@ +dataset: + name: HepatitisC + task_type: multiclass + num_classes: 5 + data_path: ../../TableData/dataset/HepatitisCdata.csv + features: [Age, Sex, ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + target: Category + categorical_mappings: + Sex: {m: 1, f: 2} + target_mapping: + "0=Blood Donor": 0 + "0s=suspect Blood Donor": 1 + "1=Hepatitis": 2 + "2=Fibrosis": 3 + "3=Cirrhosis": 4 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model_HepatitisCdata.pt + require_baseline_checkpoint: true + +experiment: + seeds: [0, 1, 2, 3, 4, 5, 6, 7] + split_seed: 42 + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + protected_shuffle: false + protected_global_data_order: true + victim_seed_offset: 0 + protected_seed_offset: 0 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + reset_rng_each_epsilon: true + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_extended + result_root: results/revision_pipeline_tabular_legacy_extended/datasets/hepatitis_c diff --git a/configs/revision/tabular_legacy_extended_systemic_diseases.yaml b/configs/revision/tabular_legacy_extended_systemic_diseases.yaml new file mode 100644 index 0000000..7112166 --- /dev/null +++ b/configs/revision/tabular_legacy_extended_systemic_diseases.yaml @@ -0,0 +1,49 @@ +dataset: + name: SystemicDiseases + task_type: multilabel + data_path: "../../TableData/TableData_oldversion/systemic diseases.csv" + # The duplicate Smoking column is part of the historical 18-D input. + features: [Age, Gender, Ethnic, Cholesterol, HDL, Hyperlipidemia, BMI, DBP, SBP, Smoking, Hypertension, Diabetes, Smoking, ASCVD, MI, Stroke, dem, PD] + targets: [inciASCVD, inciMI, inciStroke, incidem, inciPD] + categorical_mappings: + Gender: {Male: 0, Female: 1} + Ethnic: {Non-Whites: 0, Whites: 1} + Hyperlipidemia: {"yes": 0, "no": 1} + Hypertension: {"yes": 0, "no": 1} + Diabetes: {"yes": 0, "no": 1} + Smoking: {"yes": 0, "no": 1} + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + baseline_checkpoint: ../../TableData/TableData_oldversion/best_model.pt + require_baseline_checkpoint: true + +experiment: + seeds: [0, 1, 2, 3, 4, 5, 6, 7] + split_seed: 42 + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [Age, Cholesterol, HDL, Hyperlipidemia, BMI, DBP, SBP, ASCVD, MI, Stroke, dem, PD] + # Historical multi-label implementation: alpha=epsilon/4 and T=80. + num_steps: 80 + step_size_ratio: 0.25 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + stratify_split: false + preprocess_on_full_data: true + protected_shuffle: true + protected_global_data_order: true + victim_seed_offset: 0 + protected_seed_offset: 0 + victim_initialization_count: 2 + victim_deepcopy_template: true + historical_loader_rng_draws_before_training: 2 + reset_rng_each_epsilon: true + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_extended + result_root: results/revision_pipeline_tabular_legacy_extended/datasets/systemic_diseases diff --git a/configs/revision/tabular_legacy_params_diabetes.yaml b/configs/revision/tabular_legacy_params_diabetes.yaml new file mode 100644 index 0000000..9af3df9 --- /dev/null +++ b/configs/revision/tabular_legacy_params_diabetes.yaml @@ -0,0 +1,37 @@ +dataset: + name: Diabetes + task_type: binary + num_classes: 2 + data_path: ../../TableData/dataset/diabetes.csv + features: + - Pregnancies + - Glucose + - BloodPressure + - SkinThickness + - Insulin + - BMI + - DiabetesPedigreeFunction + - Age + target: Outcome + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + # Budget ablation using the legacy optimizer settings. + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [BMI, DiabetesPedigreeFunction] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_params + result_root: results/revision_pipeline_tabular_legacy_params/datasets/diabetes diff --git a/configs/revision/tabular_legacy_params_hepatitis_c.yaml b/configs/revision/tabular_legacy_params_hepatitis_c.yaml new file mode 100644 index 0000000..6e69feb --- /dev/null +++ b/configs/revision/tabular_legacy_params_hepatitis_c.yaml @@ -0,0 +1,50 @@ +dataset: + name: HepatitisC + task_type: multiclass + num_classes: 5 + data_path: ../../TableData/dataset/HepatitisCdata.csv + features: + - Age + - Sex + - ALB + - ALP + - ALT + - AST + - BIL + - CHE + - CHOL + - CREA + - GGT + - PROT + target: Category + categorical_mappings: + Sex: + "m": 0 + "f": 1 + target_mapping: + "0=Blood Donor": 0 + "0s=suspect Blood Donor": 1 + "1=Hepatitis": 2 + "2=Fibrosis": 3 + "3=Cirrhosis": 4 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [ALB, ALP, ALT, AST, BIL, CHE, CHOL, CREA, GGT, PROT] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_params + result_root: results/revision_pipeline_tabular_legacy_params/datasets/hepatitis_c diff --git a/configs/revision/tabular_legacy_params_systemic_diseases.yaml b/configs/revision/tabular_legacy_params_systemic_diseases.yaml new file mode 100644 index 0000000..15d29a9 --- /dev/null +++ b/configs/revision/tabular_legacy_params_systemic_diseases.yaml @@ -0,0 +1,63 @@ +dataset: + name: SystemicDiseases + task_type: multilabel + data_path: "../../TableData/dataset/systemic diseases.csv" + features: + - Age + - Gender + - Ethnic + - Cholesterol + - HDL + - Hyperlipidemia + - BMI + - DBP + - SBP + - Smoking + - Hypertension + - Diabetes + - ASCVD + - MI + - Stroke + - dem + - PD + targets: [inciASCVD, inciMI, inciStroke, incidem, inciPD] + categorical_mappings: + Gender: + "Male": 0 + "Female": 1 + Ethnic: + "Non-Whites": 0 + "Whites": 1 + Hyperlipidemia: + "yes": 1 + "no": 0 + Hypertension: + "yes": 1 + "no": 0 + Diabetes: + "yes": 1 + "no": 0 + Smoking: + "yes": 1 + "no": 0 + test_size: 0.2 + +model: + hidden_dims: [64, 32] + dropout: 0.3 + +experiment: + seeds: [0, 1, 2] + epsilons: [0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + attackable_features: [Cholesterol, HDL, BMI, DBP, SBP] + num_steps: 300 + step_size: 0.05 + baseline_epochs: 100 + protected_epochs: 100 + batch_size: 32 + learning_rate: 0.001 + class_balance: false + +outputs: + artifact_root: artifacts/revision_pipeline_tabular_legacy_params + result_root: results/revision_pipeline_tabular_legacy_params/datasets/systemic_diseases diff --git a/dataset.py b/dataset.py index 6cea687..505fe2e 100644 --- a/dataset.py +++ b/dataset.py @@ -32,7 +32,7 @@ for name in [ 'PoisonPathMNIST','PoisonDermaMNIST','PoisonOCTMNIST','PoisonPneumoniaMNIST', 'PoisonRetinaMNIST','PoisonBreastMNIST','PoisonBloodMNIST','PoisonTissueMNIST', - 'PoisonOrganAMNIST','PoisonOrganCMNIST','PoisonOrganSMNIST' + 'PoisonOrganAMNIST','PoisonOrganCMNIST','PoisonOrganSMNIST','PoisonChestMNIST' ]: transform_options[name] = transform_options['MedMNIST'] @@ -43,7 +43,10 @@ transform_options[name] = transform_options['MedMNIST3D'] MED2D_RGB = {'PathMNIST','DermaMNIST','RetinaMNIST','BloodMNIST'} -MED2D_GRAY = {'OrganAMNIST','OrganCMNIST','OrganSMNIST','OCTMNIST','PneumoniaMNIST','BreastMNIST','TissueMNIST'} +MED2D_GRAY = { + 'OrganAMNIST','OrganCMNIST','OrganSMNIST','OCTMNIST','PneumoniaMNIST', + 'BreastMNIST','TissueMNIST','ChestMNIST' +} MED3D = {'OrganMNIST3D','NoduleMNIST3D','FractureMNIST3D','AdrenalMNIST3D','VesselMNIST3D','SynapseMNIST3D'} IMG_EXTS = {'.jpg','.jpeg','.png','.ppm','.bmp','.tiff','.npz'} @@ -91,7 +94,7 @@ def _compose(p): # ========================= # Dataset builders # ========================= -def _build_official(dtype: str, split: str, tfm, high_resolution: bool): +def _build_official(dtype: str, split: str, path: str, tfm, high_resolution: bool): """ Build an official MedMNIST dataset. Handles RGB/Grayscale and optional high-resolution resizing. @@ -108,10 +111,17 @@ def _build_official(dtype: str, split: str, tfm, high_resolution: bool): else: as_rgb = True # fallback + root = os.path.abspath(os.path.expanduser(path)) if high_resolution and (_is_gray2d(dtype) or _is_rgb2d(dtype)): - return DataClass(split=split, transform=tfm, download=True, as_rgb=as_rgb, size=224) + return DataClass( + split=split, transform=tfm, download=True, + as_rgb=as_rgb, size=224, root=root, + ) else: - return DataClass(split=split, transform=tfm, download=True, as_rgb=as_rgb) + return DataClass( + split=split, transform=tfm, download=True, + as_rgb=as_rgb, root=root, + ) def _build_poison(base_dtype: str, path: str, split: str, tfm, high_resolution: bool): """ @@ -119,7 +129,10 @@ def _build_poison(base_dtype: str, path: str, split: str, tfm, high_resolution: Prefers a custom class named Poison{BaseType}, falls back to folder/npz loaders if not found. """ - if _is_3d(base_dtype) or high_resolution: + # Multi-label samples cannot be represented by one integer-named class + # folder. ChestMNIST therefore shares the lossless channel-first NPZ + # contract used by volumetric datasets. + if _is_3d(base_dtype) or base_dtype == 'ChestMNIST': return Data3DFolderWithLabel(path, None, transform=tfm) else: return DataFolderWithLabel(path, None, transform=tfm) @@ -130,7 +143,7 @@ def _make_dataset(dtype: str, split: str, path: str, tfm, high_resolution: bool) return _build_poison(_base(dtype), path, split, tfm, high_resolution) base = _base(dtype) if _is_gray2d(base) or _is_rgb2d(base) or _is_3d(base): - return _build_official(base, split, tfm, high_resolution) + return _build_official(base, split, path, tfm, high_resolution) raise ValueError(f"Dataset type {dtype} not implemented.") # ========================= @@ -238,7 +251,7 @@ class Data3DFolderWithLabel(Dataset): 3D or high-resolution dataset loader (current version: loads from a single .npz file). The .npz file must contain: - perturb_images: shape (N, D, H, W) or (N, C, D, H, W) - - perturb_labels: shape (N,) + - perturb_labels: shape (N,) or (N,L) for multi-label data """ def __init__(self, npz_file, pred_idx=None, transform=None): data = np.load(npz_file) @@ -252,7 +265,10 @@ def __len__(self): def __getitem__(self, idx): image, label = self.images[idx], self.labels[idx] - if self.transform: - image = self.transform(image) - return image, label, self.pred_idx[idx] - + # Cooked NPZ arrays are already normalized and channel-first. Applying + # torchvision.ToTensor here would interpret CHW as HWC; tensorize them + # directly instead. This is also the desired representation for 3D. + image = torch.as_tensor(np.asarray(image), dtype=torch.float32) + label = torch.as_tensor(np.asarray(label)) + pred_idx = torch.as_tensor(np.asarray(self.pred_idx[idx])) + return image, label, pred_idx diff --git a/evaluator.py b/evaluator.py index 600c553..682d1fa 100644 --- a/evaluator.py +++ b/evaluator.py @@ -1,10 +1,10 @@ import time +import os +import numpy as np -import models import torch -import torch.optim as optim import util -from torch.autograd import Variable +from metrics.classification_metrics import compute_classification_metrics if torch.cuda.is_available(): device = torch.device('cuda') @@ -17,7 +17,13 @@ def __init__(self, data_loader, logger, config, data_type): self.loss_meters = util.AverageMeter() self.acc_meters = util.AverageMeter() self.acc5_meters = util.AverageMeter() - self.criterion = torch.nn.CrossEntropyLoss() + self.task_type = getattr(config, 'task_type', None) or 'multiclass' + normalized_task = str(self.task_type).replace('-', '').replace('_', '').lower() + self.is_multilabel = normalized_task == 'multilabel' + self.criterion = ( + torch.nn.BCEWithLogitsLoss() if self.is_multilabel + else torch.nn.CrossEntropyLoss() + ) self.data_loader = data_loader self.logger = logger self.log_frequency = config.log_frequency if config.log_frequency is not None else 100 @@ -26,6 +32,10 @@ def __init__(self, data_loader, logger, config, data_type): self.current_acc_top5 = 0 self.confusion_matrix = torch.zeros(config.num_classes, config.num_classes) self.data_type = data_type + self.y_true = [] + self.y_prob = [] + self.recover_y_true = [] + self.recover_y_prob = [] self.recover_loss_meters = util.AverageMeter() self.recover_acc_meters = util.AverageMeter() @@ -33,6 +43,34 @@ def __init__(self, data_loader, logger, config, data_type): self.recover_confusion_matrix = torch.zeros(self.config.num_classes, self.config.num_classes) return + def classification_metrics(self, recover=False): + """Return unified metrics while preserving legacy ACC meters.""" + labels = self.recover_y_true if recover else self.y_true + probabilities = self.recover_y_prob if recover else self.y_prob + if not labels: + raise RuntimeError('No predictions collected; call eval/eval_model first.') + return compute_classification_metrics( + torch.cat(labels).numpy(), y_prob=torch.cat(probabilities).numpy(), + task_type=self.task_type) + + def save_predictions(self, output_path, recover=False): + """Save labels/probabilities in the revision experiment NPZ contract.""" + labels = self.recover_y_true if recover else self.y_true + probabilities = self.recover_y_prob if recover else self.y_prob + if not labels: + raise RuntimeError('No predictions collected; call eval/eval_model first.') + parent = os.path.dirname(output_path) + if parent: + os.makedirs(parent, exist_ok=True) + np.savez_compressed( + output_path, + y_true=torch.cat(labels).numpy(), + y_prob=torch.cat(probabilities).numpy(), + ) + if self.logger is not None: + self.logger.info('Predictions saved at %s', output_path) + return output_path + def _reset_stats(self): self.loss_meters = util.AverageMeter() self.acc_meters = util.AverageMeter() @@ -43,19 +81,18 @@ def _reset_stats(self): self.recover_acc_meters = util.AverageMeter() self.recover_acc5_meters = util.AverageMeter() self.recover_confusion_matrix = torch.zeros(self.config.num_classes, self.config.num_classes) + self.y_true = [] + self.y_prob = [] + self.recover_y_true = [] + self.recover_y_prob = [] return def eval(self, epoch, model): model.eval() - for i, (images, labels) in enumerate(self.data_loader["test_dataset"]): - if self.data_type == 'PathMNIST' or self.data_type == 'DermaMNIST' or self.data_type == 'OCTMNIST' or\ - self.data_type == 'PneumoniaMNIST' or self.data_type == 'RetinaMNIST' or self.data_type == 'BreastMNIST' or \ - self.data_type == 'BloodMNIST' or self.data_type == 'TissueMNIST' or self.data_type == 'OrganAMNIST' or \ - self.data_type == 'OrganCMNIST' or self.data_type == 'OrganSMNIST' or self.data_type == 'OrganMNIST3D'or \ - self.data_type == 'NoduleMNIST3D' or self.data_type =='FractureMNIST3D' or self.data_type == 'AdrenalMNIST3D'or \ - self.data_type =='VesselMNIST3D' or self.data_type =='SynapseMNIST3D': - labels = torch.squeeze(labels, 1).long() - + for i, batch in enumerate(self.data_loader["test_dataset"]): + images, labels = batch[0], batch[1] + if labels.ndim > 1 and labels.shape[1] == 1: + labels = torch.squeeze(labels, 1).long() start = time.time() log_payload = self.eval_batch(images=images, labels=labels, model=model) end = time.time() @@ -67,7 +104,10 @@ def eval(self, epoch, model): if self.logger is not None: self.logger.info(display) model.eval() - for i, (images, labels, _) in enumerate(self.data_loader["recover_test_dataset"]): + for i, batch in enumerate(self.data_loader["recover_test_dataset"]): + images, labels = batch[0], batch[1] + if labels.ndim > 1 and labels.shape[1] == 1: + labels = torch.squeeze(labels, 1).long() start = time.time() recover_log_payload = self.eval_recover_batch(images=images, labels=labels, model=model) @@ -83,8 +123,10 @@ def eval(self, epoch, model): def eval_model(self, epoch, model): model.eval() - for i, (images, labels) in enumerate(self.data_loader["test_dataset"]): - labels = torch.squeeze(labels, 1).long() + for i, batch in enumerate(self.data_loader["test_dataset"]): + images, labels = batch[0], batch[1] + if labels.ndim > 1 and labels.shape[1] == 1: + labels = torch.squeeze(labels, 1).long() start = time.time() log_payload = self.eval_batch(images=images, labels=labels, model=model) @@ -104,16 +146,36 @@ def eval_batch(self, images, labels, model): with torch.no_grad(): pred = model(images) - loss = self.criterion(pred, labels) - if pred.shape[1] >= 5: + if self.is_multilabel: + targets = labels.float() + if pred.shape != targets.shape: + raise ValueError( + f'Multi-label target/logit shapes must match; got {targets.shape} and {pred.shape}.' + ) + loss = self.criterion(pred, targets) + probabilities = torch.sigmoid(pred) + predictions = probabilities >= 0.5 + acc = predictions.eq(targets.bool()).all(dim=1).float().mean() + acc5 = 1 + self.y_true.append(targets.detach().cpu()) + self.y_prob.append(probabilities.detach().cpu()) + elif pred.shape[1] >= 5: + loss = self.criterion(pred, labels) acc, acc5 = util.accuracy(pred, labels, topk=(1, 5)) + _, preds = torch.max(pred, 1) + self.y_true.append(labels.detach().cpu()) + self.y_prob.append(torch.softmax(pred, dim=1).detach().cpu()) + for t, p in zip(labels.view(-1), preds.view(-1)): + self.confusion_matrix[t.long(), p.long()] += 1 else: + loss = self.criterion(pred, labels) acc, = util.accuracy(pred, labels, topk=(1,)) acc5 = 1 - # acc, acc5 = util.accuracy(pred, labels, topk=(1, 5)) - _, preds = torch.max(pred, 1) - for t, p in zip(labels.view(-1), preds.view(-1)): - self.confusion_matrix[t.long(), p.long()] += 1 + _, preds = torch.max(pred, 1) + self.y_true.append(labels.detach().cpu()) + self.y_prob.append(torch.softmax(pred, dim=1).detach().cpu()) + for t, p in zip(labels.view(-1), preds.view(-1)): + self.confusion_matrix[t.long(), p.long()] += 1 self.loss_meters.update(loss.item(), n=images.size(0)) self.acc_meters.update(acc.item(), n=images.size(0)) @@ -131,21 +193,41 @@ def eval_recover_batch(self, images, labels, model): images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True) with torch.no_grad(): pred = model(images) - loss = self.criterion(pred, labels) - if pred.shape[1] >= 5: + if self.is_multilabel: + targets = labels.float() + if pred.shape != targets.shape: + raise ValueError( + f'Multi-label target/logit shapes must match; got {targets.shape} and {pred.shape}.' + ) + loss = self.criterion(pred, targets) + probabilities = torch.sigmoid(pred) + predictions = probabilities >= 0.5 + acc = predictions.eq(targets.bool()).all(dim=1).float().mean() + acc5 = 1 + self.recover_y_true.append(targets.detach().cpu()) + self.recover_y_prob.append(probabilities.detach().cpu()) + elif pred.shape[1] >= 5: + loss = self.criterion(pred, labels) acc, acc5 = util.accuracy(pred, labels, topk=(1, 5)) + _, preds = torch.max(pred, 1) + self.recover_y_true.append(labels.detach().cpu()) + self.recover_y_prob.append(torch.softmax(pred, dim=1).detach().cpu()) + for t, p in zip(labels.view(-1), preds.view(-1)): + self.recover_confusion_matrix[t.long(), p.long()] += 1 else: + loss = self.criterion(pred, labels) acc, = util.accuracy(pred, labels, topk=(1,)) acc5 = 1 - # acc, acc5 = util.accuracy(pred, labels, topk=(1, 5)) - _, preds = torch.max(pred, 1) - for t, p in zip(labels.view(-1), preds.view(-1)): - self.recover_confusion_matrix[t.long(), p.long()] += 1 + _, preds = torch.max(pred, 1) + self.recover_y_true.append(labels.detach().cpu()) + self.recover_y_prob.append(torch.softmax(pred, dim=1).detach().cpu()) + for t, p in zip(labels.view(-1), preds.view(-1)): + self.recover_confusion_matrix[t.long(), p.long()] += 1 self.recover_loss_meters.update(loss.item(), n=images.size(0)) self.recover_acc_meters.update(acc.item(), n=images.size(0)) # self.recover_acc5_meters.update(acc5.item(), n=images.size(0)) - self.acc5_meters.update(acc5, n=images.size(0)) + self.recover_acc5_meters.update(acc5, n=images.size(0)) recover_payload = {"acc": acc.item(), "acc_avg": self.recover_acc_meters.avg, "acc5": acc5, diff --git a/export_revision_predictions.py b/export_revision_predictions.py new file mode 100644 index 0000000..da97527 --- /dev/null +++ b/export_revision_predictions.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Export standardized prediction NPZ files from existing PyTorch checkpoints.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import torch + +from revision_experiments.common import load_config, require_records, resolve_path, setup_logging +from revision_experiments.modeling import import_object, load_model, load_xy_artifact, predict_probabilities, probabilities_from_logits + + +def _from_loader(model, data_spec, task_type, device): + factory = import_object(data_spec["loader_factory"]) + loader = factory(**dict(data_spec.get("kwargs", {}))) + labels, probabilities = [], [] + model.eval() + with torch.no_grad(): + for batch in loader: + if len(batch) < 2: + raise ValueError("Loader batches must contain at least inputs and labels.") + logits = model(batch[0].to(device)) + probabilities.append(probabilities_from_logits(logits, task_type).cpu()) + labels.append(batch[1].detach().cpu()) + return torch.cat(labels).numpy(), torch.cat(probabilities).numpy() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("export-predictions", args.verbose) + config, base_dir = load_config(args.config) + for record in require_records(config, key="exports"): + task_type = record["metadata"]["task_type"] + model = load_model(record["model"], base_dir, args.device) + data_spec = record["data"] + if "loader_factory" in data_spec: + labels, probability = _from_loader(model, data_spec, task_type, args.device) + else: + inputs, labels = load_xy_artifact(data_spec, base_dir) + probability = predict_probabilities( + model, inputs, task_type=task_type, device=args.device, + batch_size=int(record.get("batch_size", 128)), + ) + output = resolve_path(record["output"], base_dir) + output.parent.mkdir(parents=True, exist_ok=True) + np.savez_compressed(output, y_true=labels, y_prob=probability) + logger.info("Wrote %s (%d samples)", output, len(labels)) + + +if __name__ == "__main__": + main() diff --git a/jobs/datacook-224-three-local.sh b/jobs/datacook-224-three-local.sh new file mode 100755 index 0000000..48fbab3 --- /dev/null +++ b/jobs/datacook-224-three-local.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Representative 224x224 MedMNIST+ experiment runner. +# Project: DataCook +# Created: 2026-07-22 +# Commit: 1566c034 +# +# Smoke test (default): +# CUDA_VISIBLE_DEVICES=0 RUN_MODE=smoke bash jobs/datacook-224-three-local.sh +# Full 3-dataset, 3-seed run: +# CUDA_VISIBLE_DEVICES=0 RUN_MODE=full bash jobs/datacook-224-three-local.sh + +set -euo pipefail + +JOB_NAME="datacook-224-three" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +RUN_MODE="${RUN_MODE:-smoke}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="1566c034" + +if [[ "$RUN_MODE" != "smoke" && "$RUN_MODE" != "full" ]]; then + echo "RUN_MODE must be smoke or full, got: $RUN_MODE" >&2 + exit 2 +fi + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-${RUN_MODE}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Mode: $RUN_MODE" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +if [[ "$RUN_MODE" == "smoke" ]]; then + # Validate downloading, 224 loading, cooking, protected training and metrics + # on the smallest dataset before committing the GPU to all nine runs. + run_logged python run_datacook_pipeline.py \ + --config configs/pipelines/breastmnist224.yaml \ + --seeds 0 \ + --epsilons 8 +else + # The smoke result is automatically detected and reused. + run_logged python run_datacook_suite.py \ + --config configs/pipelines/medmnist_224_three.yaml + + # Export paired 224 artifacts and generate post-hoc configs without touching + # the existing 28x28 artifacts/results. + run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_224_three.yaml \ + --artifact-root artifacts/revision_pipeline_224 \ + --config-dir configs/revision/224/generated \ + --output-suffix 224 \ + --attacks-epsilon 8 \ + --runtime-seed 0 \ + --runtime-epsilon 8 \ + --strict + + run_logged python run_revision_experiments.py \ + --config configs/revision/224/generated.yaml \ + --tasks pseudo image_quality runtime attacks \ + --device cuda +fi + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Main: results/revision_pipeline_224/" | tee -a "$LOG_FILE" +echo "Post-hoc: results/revision_pipeline/posthoc/*_224.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-2d-eight-seeds-local.sh b/jobs/datacook-2d-eight-seeds-local.sh new file mode 100755 index 0000000..fe162ea --- /dev/null +++ b/jobs/datacook-2d-eight-seeds-local.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Complete the current three 2D datasets with eight seeds and targeted attacks. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c034 +# +# Example: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-2d-eight-seeds-local.sh + +set -euo pipefail + +JOB_NAME="datacook-2d-eight-seeds" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +for required in \ + results/revision_pipeline/posthoc/adaptive_attack_results_all.csv \ + results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv \ + results/revision_pipeline/posthoc/quality_metrics.csv \ + results/revision_pipeline/posthoc/pseudo_label_quality.csv; do + if [[ ! -f "$required" ]]; then + echo "Missing prerequisite: $required" | tee -a "$LOG_FILE" + exit 2 + fi +done + +# Phase 1: 75 new main runs. Existing seed 0-2 runs are untouched and reused. +run_logged python run_datacook_suite.py \ + --config configs/pipelines/medmnist_three.yaml \ + --seeds 3 4 5 6 7 \ + --epsilons 2 4 8 12 16 + +# Phase 2: quality and pseudo-label outputs for every new seed/epsilon run. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --seeds 3 4 5 6 7 \ + --config-dir configs/revision/extensions/seed3-7-all/generated \ + --output-suffix seed3-7-all \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/extensions/seed3-7-all/generated.yaml \ + --tasks pseudo image_quality \ + --device cuda + +# Phase 3: epsilon=8 attacks for the five newly added seeds. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --seeds 3 4 5 6 7 \ + --epsilons 8 \ + --attacks-epsilon 8 \ + --finetune-epochs 10 \ + --budget-sizes 25 50 \ + --config-dir configs/revision/extensions/seed3-7-eps8/generated \ + --output-suffix seed3-7-eps8 \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/extensions/seed3-7-eps8/generated.yaml \ + --tasks attacks finetune_attacks finetune_budget_attacks \ + --device cuda + +# Phase 4: RetinaMNIST epsilon=12 attacks for all eight seeds. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --datasets RetinaMNIST \ + --seeds 0 1 2 3 4 5 6 7 \ + --epsilons 12 \ + --attacks-epsilon 12 \ + --finetune-epochs 10 \ + --budget-sizes 25 50 \ + --config-dir configs/revision/extensions/retina-eps12/generated \ + --output-suffix retina-eps12 \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/extensions/retina-eps12/generated.yaml \ + --tasks attacks finetune_attacks finetune_budget_attacks \ + --device cuda + +# Phase 5: conflict-checked merge and final primary statistical analysis. +run_logged python -m analysis.merge_result_shards \ + --config configs/revision/merge_2d_8seeds.yaml + +run_logged python -m analysis.finalize_revision_results \ + --config configs/revision/final_analysis_2d_8seeds.yaml + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Main summary: results/revision_pipeline/summary/" | tee -a "$LOG_FILE" +echo "Extended post-hoc: results/revision_pipeline/posthoc/*extended_2d_8seeds.csv" | tee -a "$LOG_FILE" +echo "Final analysis: results/revision_pipeline/final_2d_8seeds/" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-3d-four-local.sh b/jobs/datacook-3d-four-local.sh new file mode 100644 index 0000000..2c2f00f --- /dev/null +++ b/jobs/datacook-3d-four-local.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Local single-GPU runner generated by the run-experiment skill. +# Project: DataCook +# Created: 2026-07-22 +# Commit: 1566c03404 +# +# Run inside screen: bash jobs/datacook-3d-four-local.sh [GPU_ID] + +set -euo pipefail + +JOB_NAME="datacook-3d-four" +LOG_DIR="outputs/logs/${JOB_NAME}" +OUTPUT_DIR="results/revision_pipeline_3d" +GPU_ID="${1:-0}" +GIT_COMMIT="1566c03404" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +if command -v conda >/dev/null 2>&1; then + source "$(conda info --base)/etc/profile.d/conda.sh" + conda activate datacook +fi + +cd "$(dirname "$0")/.." +export CUDA_VISIBLE_DEVICES="$GPU_ID" +export CUBLAS_WORKSPACE_CONFIG=:4096:8 + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $GPU_ID" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +set +e +python -u run_datacook_suite.py \ + --config configs/pipelines/medmnist3d_four.yaml 2>&1 | tee -a "$LOG_FILE" +EXIT_CODE=${PIPESTATUS[0]} +set -e + +echo "=========================================" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Exit: $EXIT_CODE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" +exit "$EXIT_CODE" diff --git a/jobs/datacook-3d-pseudo-runtime-local.sh b/jobs/datacook-3d-pseudo-runtime-local.sh new file mode 100644 index 0000000..28246da --- /dev/null +++ b/jobs/datacook-3d-pseudo-runtime-local.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# 3D pseudo-label and runtime post-hoc analysis for completed DataCook runs. +# Project: DataCook +# Created: 2026-07-23 +# Commit: 1566c034 +# +# Foreground run inside an existing screen session: +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-3d-pseudo-runtime-local.sh + +set -euo pipefail + +JOB_NAME="datacook-3d-pseudo-runtime" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="1566c034" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +# Reuse all 36 completed 3D runs. This only exports test NPZ artifacts and +# generates pseudo/runtime configs; it does not retrain any model. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist3d_four.yaml \ + --artifact-root artifacts/revision_pipeline_3d \ + --result-root results/revision_pipeline_3d \ + --config-dir configs/revision/3d/generated \ + --tasks pseudo runtime \ + --runtime-seed 0 \ + --runtime-epsilon 8 \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/3d/generated.yaml \ + --tasks pseudo runtime \ + --device cuda + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Pseudo: results/revision_pipeline_3d/posthoc/pseudo_label_quality.csv" | tee -a "$LOG_FILE" +echo "Runtime: results/revision_pipeline_3d/posthoc/runtime_summary.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-3d-quality-local.sh b/jobs/datacook-3d-quality-local.sh new file mode 100755 index 0000000..1903601 --- /dev/null +++ b/jobs/datacook-3d-quality-local.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# 3D image-quality analysis for all completed MedMNIST3D DataCook runs. +# Project: DataCook +# Created: 2026-07-24 +# Commit: 1566c03404 +# +# This is CPU post-processing and does not retrain any model. +# Foreground run inside an existing screen session: +# CONDA_ENV=datacook bash jobs/datacook-3d-quality-local.sh + +set -euo pipefail + +JOB_NAME="datacook-3d-quality" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/results/revision_pipeline_3d/posthoc" +CONDA_ENV="${CONDA_ENV:-datacook}" +GIT_COMMIT="1566c03404" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." +fi + +cd "$PROJECT_ROOT" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "Device: CPU" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +# Re-export cooked volumes from the current perturbations. --force is required +# so a later Synapse rerun cannot leave a stale cooked_test.npz in the analysis. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist3d_four.yaml \ + --artifact-root artifacts/revision_pipeline_3d \ + --result-root results/revision_pipeline_3d \ + --config-dir configs/revision/3d/generated \ + --tasks image_quality \ + --force \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/3d/generated.yaml \ + --tasks image_quality + +QUALITY_FILE="$OUTPUT_DIR/quality_metrics.csv" +EXPECTED_LINES=37 +ACTUAL_LINES="$(wc -l < "$QUALITY_FILE" | tr -d ' ')" +if [[ "$ACTUAL_LINES" -ne "$EXPECTED_LINES" ]]; then + echo "Expected 36 quality rows plus header, found $ACTUAL_LINES lines." | tee -a "$LOG_FILE" >&2 + exit 3 +fi + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Rows: 36 (4 datasets x 3 seeds x 3 epsilon values)" | tee -a "$LOG_FILE" +echo "Metrics: 3D SSIM, PSNR, L2 norm, Linf norm" | tee -a "$LOG_FILE" +echo "LPIPS/FID: not computed for 3D volumes" | tee -a "$LOG_FILE" +echo "Output: results/revision_pipeline_3d/posthoc/quality_metrics.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-3d-smoke-local.sh b/jobs/datacook-3d-smoke-local.sh new file mode 100644 index 0000000..991f457 --- /dev/null +++ b/jobs/datacook-3d-smoke-local.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Local single-GPU experiment runner generated for the 3D DataCook smoke test. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c03404 +# +# Run inside screen: bash jobs/datacook-3d-smoke-local.sh [GPU_ID] + +set -euo pipefail + +JOB_NAME="datacook-3d-smoke" +LOG_DIR="outputs/logs/${JOB_NAME}" +OUTPUT_DIR="results/revision_pipeline_3d" +GPU_ID="${1:-0}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +if command -v conda >/dev/null 2>&1; then + source "$(conda info --base)/etc/profile.d/conda.sh" + conda activate datacook +fi + +cd "$(dirname "$0")/.." +export CUDA_VISIBLE_DEVICES="$GPU_ID" + +echo "Job: $JOB_NAME" | tee "$LOG_FILE" +echo "GPU: $GPU_ID" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Command: python -u run_datacook_suite.py --config configs/pipelines/medmnist3d_smoke.yaml" | tee -a "$LOG_FILE" + +set +e +python -u run_datacook_suite.py \ + --config configs/pipelines/medmnist3d_smoke.yaml 2>&1 | tee -a "$LOG_FILE" +EXIT_CODE=${PIPESTATUS[0]} +set -e + +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Exit: $EXIT_CODE" | tee -a "$LOG_FILE" +exit "$EXIT_CODE" diff --git a/jobs/datacook-bloodmnist-revision-local.sh b/jobs/datacook-bloodmnist-revision-local.sh new file mode 100755 index 0000000..475cc76 --- /dev/null +++ b/jobs/datacook-bloodmnist-revision-local.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Local experiment runner — generated from the run-experiment workflow +# Project: DataCook +# Created: 2026-07-20 +# +# Attached: bash jobs/datacook-bloodmnist-revision-local.sh +# tmux: tmux new-session -d -s datacook-bloodmnist-revision 'bash jobs/datacook-bloodmnist-revision-local.sh' + +set -euo pipefail + +JOB_NAME="datacook-bloodmnist-revision" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +GIT_COMMIT="$(git -C "$PROJECT_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" +export GIT_COMMIT + +RUN_COMMAND=(python run_datacook_pipeline.py --config configs/pipelines/bloodmnist.yaml) +echo "Running: ${RUN_COMMAND[*]}" | tee -a "$LOG_FILE" +echo "-----------------------------------------" | tee -a "$LOG_FILE" + +set +e +"${RUN_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE" +EXIT_CODE=${PIPESTATUS[0]} +set -e + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Exit: $EXIT_CODE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +exit "$EXIT_CODE" diff --git a/jobs/datacook-chestmnist-local.sh b/jobs/datacook-chestmnist-local.sh new file mode 100755 index 0000000..51795e9 --- /dev/null +++ b/jobs/datacook-chestmnist-local.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# ChestMNIST multi-label DataCook runner. +# Project: DataCook +# Created: 2026-07-22 +# Commit: 1566c034 +# +# Validation run (default, seed 0 / epsilon 8): +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook RUN_MODE=smoke bash jobs/datacook-chestmnist-local.sh +# Three-seed main experiment plus post-hoc analysis: +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook RUN_MODE=full bash jobs/datacook-chestmnist-local.sh + +set -euo pipefail + +JOB_NAME="datacook-chestmnist" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +RUN_MODE="${RUN_MODE:-smoke}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="1566c034" + +if [[ "$RUN_MODE" != "smoke" && "$RUN_MODE" != "full" ]]; then + echo "RUN_MODE must be smoke or full, got: $RUN_MODE" >&2 + exit 2 +fi + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-${RUN_MODE}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Mode: $RUN_MODE" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +if [[ "$RUN_MODE" == "smoke" ]]; then + run_logged python run_datacook_pipeline.py \ + --config configs/pipelines/chestmnist.yaml +else + # The completed smoke run is automatically reused. + run_logged python run_datacook_pipeline.py \ + --config configs/pipelines/chestmnist.yaml \ + --seeds 0 1 2 \ + --epsilons 8 + + run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_multilabel.yaml \ + --seeds 0 1 2 \ + --epsilons 8 \ + --artifact-root artifacts/revision_pipeline_multilabel \ + --config-dir configs/revision/chestmnist/generated \ + --output-suffix chestmnist \ + --attacks-epsilon 8 \ + --runtime-seed 0 \ + --runtime-epsilon 8 \ + --strict + + run_logged python run_revision_experiments.py \ + --config configs/revision/chestmnist/generated.yaml \ + --tasks pseudo image_quality runtime attacks finetune_attacks finetune_budget_attacks \ + --device cuda +fi + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Main: results/revision_pipeline_multilabel/" | tee -a "$LOG_FILE" +echo "Post-hoc suffix: *_chestmnist.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-extra-three-2d-eight-seeds-local.sh b/jobs/datacook-extra-three-2d-eight-seeds-local.sh new file mode 100755 index 0000000..6bb7b2d --- /dev/null +++ b/jobs/datacook-extra-three-2d-eight-seeds-local.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Second-server job: three additional 2D datasets, eight seeds, five epsilons. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c034 +# +# Example: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-extra-three-2d-eight-seeds-local.sh + +set -euo pipefail + +JOB_NAME="datacook-extra-three-2d-eight-seeds" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +SMOKE_ONLY="${SMOKE_ONLY:-0}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +# Smoke test. These three runs are reused by the full grid below. +run_logged python run_datacook_suite.py \ + --config configs/pipelines/medmnist_extra_three.yaml \ + --seeds 0 \ + --epsilons 8 + +if [[ "$SMOKE_ONLY" == "1" ]]; then + echo "Smoke test completed; SMOKE_ONLY=1, stopping before the full grid." | tee -a "$LOG_FILE" + exit 0 +fi + +# Full grid: 3 datasets x 8 seeds x 5 epsilon values = 120 runs. +run_logged python run_datacook_suite.py \ + --config configs/pipelines/medmnist_extra_three.yaml \ + --seeds 0 1 2 3 4 5 6 7 \ + --epsilons 2 4 8 12 16 + +# Export all post-hoc artifacts. Adaptive attacks are generated only for epsilon=8. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_extra_three.yaml \ + --artifact-root artifacts/revision_pipeline_extra2d \ + --config-dir configs/revision/extensions/extra2d/generated \ + --attacks-epsilon 8 \ + --runtime-seed 0 \ + --runtime-epsilon 8 \ + --finetune-epochs 10 \ + --budget-sizes 25 50 \ + --output-suffix extra2d \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/extensions/extra2d/generated.yaml \ + --tasks pseudo image_quality runtime attacks finetune_attacks finetune_budget_attacks \ + --device cuda + +run_logged python -m analysis.finalize_revision_results \ + --config configs/revision/final_analysis_extra2d_8seeds.yaml + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Main results: results/revision_pipeline_extra2d/" | tee -a "$LOG_FILE" +echo "Post-hoc shards: results/revision_pipeline/posthoc/*_extra2d.csv" | tee -a "$LOG_FILE" +echo "Final analysis: results/revision_pipeline_extra2d/final_8seeds/" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-finetune-attacks-local.sh b/jobs/datacook-finetune-attacks-local.sh new file mode 100755 index 0000000..68225fa --- /dev/null +++ b/jobs/datacook-finetune-attacks-local.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Adaptive fine-tuning attacks for completed three-dataset DataCook runs. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c034 +# +# Example: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-finetune-attacks-local.sh + +set -euo pipefail + +JOB_NAME="datacook-finetune-attacks" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +OLD_RESULTS="results/revision_pipeline/posthoc/adaptive_attack_results.csv" +if [[ ! -f "$OLD_RESULTS" ]]; then + echo "Missing $OLD_RESULTS; run the inference attacks before fine-tuning attacks." | tee -a "$LOG_FILE" + exit 2 +fi + +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --attacks-epsilon 8 \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks finetune_attacks \ + --device cuda + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Outputs:" | tee -a "$LOG_FILE" +echo " results/revision_pipeline/posthoc/adaptive_finetune_results.csv" | tee -a "$LOG_FILE" +echo " results/revision_pipeline/posthoc/adaptive_attack_results_all.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-finetune-budget-and-analysis-local.sh b/jobs/datacook-finetune-budget-and-analysis-local.sh new file mode 100755 index 0000000..e0445c5 --- /dev/null +++ b/jobs/datacook-finetune-budget-and-analysis-local.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Fixed-budget adaptive fine-tuning attacks followed by final statistical analysis. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c034 +# +# Example: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-finetune-budget-and-analysis-local.sh + +set -euo pipefail + +JOB_NAME="datacook-finetune-budget-and-analysis" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +for required in \ + results/revision_pipeline/posthoc/adaptive_attack_results_all.csv \ + results/revision_pipeline/posthoc/adaptive_finetune_results.csv \ + results/revision_pipeline/summary/cp_pp_summary.csv; do + if [[ ! -f "$required" ]]; then + echo "Missing prerequisite: $required" | tee -a "$LOG_FILE" + exit 2 + fi +done + +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml \ + --attacks-epsilon 8 \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks finetune_budget_attacks \ + --device cuda + +run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks final_analysis + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Budget result: results/revision_pipeline/posthoc/adaptive_finetune_budget_results.csv" | tee -a "$LOG_FILE" +echo "Final tables: results/revision_pipeline/final/" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-medmnist-additional-two-local.sh b/jobs/datacook-medmnist-additional-two-local.sh new file mode 100755 index 0000000..c879b22 --- /dev/null +++ b/jobs/datacook-medmnist-additional-two-local.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Local experiment runner generated from the run-experiment workflow. +# Project: DataCook +# Created: 2026-07-20 +# Commit: 1566c034 +# +# Run: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-medmnist-additional-two-local.sh + +set -euo pipefail + +JOB_NAME="datacook-medmnist-additional-two" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: ${CUDA_VISIBLE_DEVICES:-all visible GPUs}" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" +export GIT_COMMIT="1566c034" + +RUN_COMMAND=(python run_datacook_suite.py --config configs/pipelines/medmnist_additional_two.yaml) +echo "Running: ${RUN_COMMAND[*]}" | tee -a "$LOG_FILE" +echo "-----------------------------------------" | tee -a "$LOG_FILE" + +set +e +"${RUN_COMMAND[@]}" 2>&1 | tee -a "$LOG_FILE" +EXIT_CODE=${PIPESTATUS[0]} +set -e + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Exit: $EXIT_CODE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +exit "$EXIT_CODE" diff --git a/jobs/datacook-merge-six-2d-results-local.sh b/jobs/datacook-merge-six-2d-results-local.sh new file mode 100755 index 0000000..47f35a6 --- /dev/null +++ b/jobs/datacook-merge-six-2d-results-local.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# CPU-only merge after both 2D experiment servers have been synchronized. + +set -euo pipefail + +JOB_NAME="datacook-merge-six-2d-results" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +run_logged python -m analysis.merge_result_shards \ + --config configs/revision/merge_2d_six_datasets.yaml + +run_logged python -m analysis.finalize_revision_results \ + --config configs/revision/final_analysis_2d_six_datasets.yaml + +echo "Final six-dataset results: results/revision_pipeline_6datasets/" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-recompute-psnr-2d-local.sh b/jobs/datacook-recompute-psnr-2d-local.sh new file mode 100755 index 0000000..7e78456 --- /dev/null +++ b/jobs/datacook-recompute-psnr-2d-local.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Recompute 2D image quality after the aggregate-MSE PSNR fix, then rebuild tables. +# Project: DataCook +# Created: 2026-07-22 +# Commit: 1566c034 +# +# Run in the current screen session: +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-recompute-psnr-2d-local.sh + +set -euo pipefail + +JOB_NAME="datacook-recompute-psnr-2d" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="1566c034" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +# Recompute all 120 quality rows. No training/checkpoint is touched. +run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks image_quality \ + --device cuda + +run_logged python run_revision_experiments.py \ + --config configs/revision/extensions/seed3-7-all/generated.yaml \ + --tasks image_quality \ + --device cuda + +# Rebuild the extended quality shard and all final paper-ready tables. +run_logged python -m analysis.merge_result_shards \ + --config configs/revision/merge_2d_8seeds.yaml + +run_logged python -m analysis.finalize_revision_results \ + --config configs/revision/final_analysis_2d_8seeds.yaml + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Quality: results/revision_pipeline/posthoc/quality_metrics_extended_2d_8seeds.csv" | tee -a "$LOG_FILE" +echo "Final: results/revision_pipeline/final_2d_8seeds/" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-revision-posthoc-local.sh b/jobs/datacook-revision-posthoc-local.sh new file mode 100755 index 0000000..e00b5ad --- /dev/null +++ b/jobs/datacook-revision-posthoc-local.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Incremental post-hoc runner for completed DataCook main experiments. +# Project: DataCook +# Created: 2026-07-21 +# Commit: 1566c034 +# +# Run: CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook bash jobs/datacook-revision-posthoc-local.sh + +set -euo pipefail + +JOB_NAME="datacook-revision-posthoc" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: 1566c034" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "GPU: ${CUDA_VISIBLE_DEVICES:-all visible GPUs}" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" +export GIT_COMMIT="1566c034" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +set +e +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist_three.yaml +EXIT_CODE=${PIPESTATUS[0]} + +if [[ "$EXIT_CODE" -eq 0 ]]; then + run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks pseudo image_quality runtime \ + --device cuda + EXIT_CODE=${PIPESTATUS[0]} +fi + +if [[ "$EXIT_CODE" -eq 0 ]]; then + run_logged python run_revision_experiments.py \ + --config configs/revision/generated.yaml \ + --tasks attacks \ + --device cuda + EXIT_CODE=${PIPESTATUS[0]} +fi +set -e + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Exit: $EXIT_CODE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +exit "$EXIT_CODE" diff --git a/jobs/datacook-synapse3d-rerun-local.sh b/jobs/datacook-synapse3d-rerun-local.sh new file mode 100755 index 0000000..d5e054f --- /dev/null +++ b/jobs/datacook-synapse3d-rerun-local.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Foreground single-GPU rerun for all SynapseMNIST3D seeds and epsilon values. +# Project: DataCook +# Created: 2026-07-23 +# Commit: 1566c03404 +# +# This intentionally overwrites the existing SynapseMNIST3D primary outputs. +# Run inside an existing screen session: +# CONDA_ENV=datacook bash jobs/datacook-synapse3d-rerun-local.sh 0 + +set -euo pipefail + +JOB_NAME="datacook-synapse3d-rerun" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/results/revision_pipeline_3d" +CONDA_ENV="${CONDA_ENV:-datacook}" +GPU_ID="${1:-0}" +GIT_COMMIT="1566c03404" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." +fi + +cd "$PROJECT_ROOT" +export CUDA_VISIBLE_DEVICES="$GPU_ID" +export CUBLAS_WORKSPACE_CONFIG=:4096:8 + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $GPU_ID" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +# --force reruns perturbation, protected-model training, and metrics for: +# seeds 0/1/2 x epsilon 8/16/32. Other 3D datasets are not retrained. +run_logged python -u run_datacook_pipeline.py \ + --config configs/pipelines/synapsemnist3d.yaml \ + --force + +# Re-export derived artifacts and rebuild the complete four-dataset 3D +# pseudo-label/runtime tables so that no row still references old Synapse runs. +run_logged python prepare_revision_posthoc.py \ + --suite-config configs/pipelines/medmnist3d_four.yaml \ + --artifact-root artifacts/revision_pipeline_3d \ + --result-root results/revision_pipeline_3d \ + --config-dir configs/revision/3d/generated \ + --tasks pseudo runtime \ + --runtime-seed 0 \ + --runtime-epsilon 8 \ + --force \ + --strict + +run_logged python run_revision_experiments.py \ + --config configs/revision/3d/generated.yaml \ + --tasks pseudo runtime \ + --device cuda + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Primary: results/revision_pipeline_3d/summary" | tee -a "$LOG_FILE" +echo "Pseudo: results/revision_pipeline_3d/posthoc/pseudo_label_quality.csv" | tee -a "$LOG_FILE" +echo "Runtime: results/revision_pipeline_3d/posthoc/runtime_summary.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-tabular-epsilon-local.sh b/jobs/datacook-tabular-epsilon-local.sh new file mode 100755 index 0000000..1e9a9d6 --- /dev/null +++ b/jobs/datacook-tabular-epsilon-local.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Three-dataset tabular epsilon ablation. +# Project: DataCook +# Created: 2026-07-23 +# Commit: 1566c034 +# +# Foreground run inside an existing screen session: +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook \ +# bash jobs/datacook-tabular-epsilon-local.sh +# +# Optional custom dataset locations: +# DIABETES_DATA=/path/to/diabetes.csv \ +# HEPATITIS_DATA=/path/to/HepatitisCdata.csv \ +# SYSTEMIC_DATA="/path/to/systemic diseases.csv" \ +# CUDA_VISIBLE_DEVICES=0 bash jobs/datacook-tabular-epsilon-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-epsilon" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +DIABETES_DATA="${DIABETES_DATA:-${TABULAR_DATA:-$PROJECT_ROOT/TableData/dataset/diabetes.csv}}" +HEPATITIS_DATA="${HEPATITIS_DATA:-$PROJECT_ROOT/TableData/dataset/HepatitisCdata.csv}" +SYSTEMIC_DATA="${SYSTEMIC_DATA:-$PROJECT_ROOT/TableData/dataset/systemic diseases.csv}" +TABULAR_DEVICE="${TABULAR_DEVICE:-cuda}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="1566c034" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Device: $TABULAR_DEVICE" | tee -a "$LOG_FILE" +echo "Data 1: $DIABETES_DATA" | tee -a "$LOG_FILE" +echo "Data 2: $HEPATITIS_DATA" | tee -a "$LOG_FILE" +echo "Data 3: $SYSTEMIC_DATA" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +for data_path in "$DIABETES_DATA" "$HEPATITIS_DATA" "$SYSTEMIC_DATA"; do + if [[ ! -f "$data_path" ]]; then + echo "Missing tabular CSV: $data_path" | tee -a "$LOG_FILE" >&2 + exit 2 + fi +done + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the currently active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_epsilon_diabetes.yaml \ + --data-path "$DIABETES_DATA" \ + --device "$TABULAR_DEVICE" + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_epsilon_hepatitis_c.yaml \ + --data-path "$HEPATITIS_DATA" \ + --device "$TABULAR_DEVICE" + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_epsilon_systemic_diseases.yaml \ + --data-path "$SYSTEMIC_DATA" \ + --device "$TABULAR_DEVICE" + +run_logged python -m analysis.merge_result_shards \ + --config configs/revision/merge_tabular_epsilon.yaml + +run_logged python -m analysis.aggregate_results \ + --config configs/revision/statistics_tabular_epsilon.yaml + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Runs: results/revision_pipeline_tabular/epsilon_ablation.csv (45 runs)" | tee -a "$LOG_FILE" +echo "Aggregated: results/revision_pipeline_tabular/aggregated_results.csv" | tee -a "$LOG_FILE" +echo "Runtime: results/revision_pipeline_tabular/runtime_summary.csv" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/jobs/datacook-tabular-legacy-exact-local.sh b/jobs/datacook-tabular-legacy-exact-local.sh new file mode 100755 index 0000000..f717b9b --- /dev/null +++ b/jobs/datacook-tabular-legacy-exact-local.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Strict reconstruction of the three preliminary tabular DataCook results. +# The historical scripts declared a CUDA device but never moved their models or +# tensors to it, so exact reconstruction must execute on CPU. +# Run in the foreground inside screen: +# CONDA_ENV=datacook bash jobs/datacook-tabular-legacy-exact-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-legacy-exact" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" +MASTER_LOG="$LOG_DIR/$JOB_NAME-$(date +%Y%m%d-%H%M%S).log" + +if [[ -n "${CONDA_PREFIX:-}" && -x "$CONDA_PREFIX/bin/python" ]]; then + echo "Using active environment: $CONDA_PREFIX" | tee "$MASTER_LOG" +elif command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" + echo "Activated conda environment: $CONDA_ENV" | tee "$MASTER_LOG" +else + echo "No conda command found; using active Python: $(command -v python)" | tee "$MASTER_LOG" +fi + +cd "$PROJECT_ROOT" +echo "Protocol: strict historical reconstruction; CPU, split seed=42, epsilon=5" | tee -a "$MASTER_LOG" + +CHECKPOINT_DIR="$PROJECT_ROOT/TableData/TableData_oldversion" +verify_checkpoint() { + local filename="$1" + local expected_hash="$2" + checkpoint="$CHECKPOINT_DIR/$filename" + if [[ ! -f "$checkpoint" ]]; then + echo "Missing historical checkpoint: $checkpoint" | tee -a "$MASTER_LOG" >&2 + exit 2 + fi + actual_hash="$(sha256sum "$checkpoint" | awk '{print $1}')" + if [[ "$actual_hash" != "$expected_hash" ]]; then + echo "Checkpoint hash mismatch: $checkpoint" | tee -a "$MASTER_LOG" >&2 + echo "Expected $expected_hash, found $actual_hash" | tee -a "$MASTER_LOG" >&2 + exit 2 + fi + echo "$actual_hash $checkpoint" | tee -a "$MASTER_LOG" +} +verify_checkpoint best_model_diabete.pt f731a062af185a5c74836088467100ffa1191998d11a5c76f687b30615956a1c +verify_checkpoint best_model_HepatitisCdata.pt 09f0ae5b47ad285c87e8a798f0c5ebb568b2463593736f16ae1eb72a4c365e7e +verify_checkpoint best_model.pt 04dd310159eaf16815e8e277fb2c75eff347b8f62b783d37e43e5aaa9e133d66 + +run_one() { + local name="$1" + local config="$2" + local log="$LOG_DIR/$name.log" + echo "Starting $name on CPU" | tee -a "$MASTER_LOG" + python -u run_tabular_epsilon_ablation.py \ + --config "$config" --device cpu --force 2>&1 | tee "$log" + echo "Completed $name" | tee -a "$MASTER_LOG" +} + +# Sequential execution avoids CPU scheduling differences and mirrors the old runs. +run_one diabetes configs/revision/tabular_legacy_exact_diabetes.yaml +run_one hepatitis_c configs/revision/tabular_legacy_exact_hepatitis_c.yaml +run_one systemic_diseases configs/revision/tabular_legacy_exact_systemic_diseases.yaml + +python summarize_tabular_legacy_reproduction.py --strict 2>&1 | tee -a "$MASTER_LOG" +echo "Strict legacy reconstruction completed." | tee -a "$MASTER_LOG" diff --git a/jobs/datacook-tabular-legacy-extended-local.sh b/jobs/datacook-tabular-legacy-extended-local.sh new file mode 100755 index 0000000..9c96a7d --- /dev/null +++ b/jobs/datacook-tabular-legacy-extended-local.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Eight-seed perturbation-budget study under the faithfully reconstructed +# historical tabular protocols. This is CPU-only because the old code executed +# its models and tensors on CPU despite declaring a CUDA device. +# +# Foreground run inside screen: +# CONDA_ENV=datacook bash jobs/datacook-tabular-legacy-extended-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-legacy-extended" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +MASTER_LOG="$LOG_DIR/$JOB_NAME-$(date +%Y%m%d-%H%M%S).log" + +if [[ -n "${CONDA_PREFIX:-}" && -x "$CONDA_PREFIX/bin/python" ]]; then + echo "Using active environment: $CONDA_PREFIX" | tee "$MASTER_LOG" +elif command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" + echo "Activated conda environment: $CONDA_ENV" | tee "$MASTER_LOG" +else + echo "No conda command found; using active Python: $(command -v python)" | tee "$MASTER_LOG" +fi + +cd "$PROJECT_ROOT" +echo "Protocol: fixed historical split/checkpoints; 8 victim seeds; 6 budgets" | tee -a "$MASTER_LOG" +echo "Grid: 3 datasets x 8 seeds x 6 budgets = 144 runs" | tee -a "$MASTER_LOG" +echo "Diabetes/HepatitisC: T=300, alpha=0.05" | tee -a "$MASTER_LOG" +echo "SystemicDiseases: T=80, alpha=epsilon/4" | tee -a "$MASTER_LOG" +python -c 'import platform, sklearn, torch; print(f"Python={platform.python_version()} PyTorch={torch.__version__} sklearn={sklearn.__version__}")' | tee -a "$MASTER_LOG" + +CHECKPOINT_DIR="$PROJECT_ROOT/TableData/TableData_oldversion" +verify_checkpoint() { + local filename="$1" + local expected_hash="$2" + checkpoint="$CHECKPOINT_DIR/$filename" + if [[ ! -f "$checkpoint" ]]; then + echo "Missing historical checkpoint: $checkpoint" | tee -a "$MASTER_LOG" >&2 + exit 2 + fi + actual_hash="$(sha256sum "$checkpoint" | awk '{print $1}')" + if [[ "$actual_hash" != "$expected_hash" ]]; then + echo "Checkpoint hash mismatch: $checkpoint" | tee -a "$MASTER_LOG" >&2 + exit 2 + fi + echo "$actual_hash $checkpoint" | tee -a "$MASTER_LOG" +} +verify_checkpoint best_model_diabete.pt f731a062af185a5c74836088467100ffa1191998d11a5c76f687b30615956a1c +verify_checkpoint best_model_HepatitisCdata.pt 09f0ae5b47ad285c87e8a798f0c5ebb568b2463593736f16ae1eb72a4c365e7e +verify_checkpoint best_model.pt 04dd310159eaf16815e8e277fb2c75eff347b8f62b783d37e43e5aaa9e133d66 + +run_dataset() { + local name="$1" + local config="$2" + local log="$LOG_DIR/$name.log" + echo "Starting $name on CPU; log=$log" | tee -a "$MASTER_LOG" + python -u run_tabular_epsilon_ablation.py \ + --config "$config" --device cpu >"$log" 2>&1 +} + +run_dataset diabetes configs/revision/tabular_legacy_extended_diabetes.yaml & +pid_diabetes=$! +run_dataset hepatitis_c configs/revision/tabular_legacy_extended_hepatitis_c.yaml & +pid_hepatitis=$! +run_dataset systemic_diseases configs/revision/tabular_legacy_extended_systemic_diseases.yaml & +pid_systemic=$! + +failed=0 +for spec in "diabetes:$pid_diabetes" "hepatitis_c:$pid_hepatitis" "systemic_diseases:$pid_systemic"; do + name="${spec%%:*}" + pid="${spec##*:}" + if wait "$pid"; then + echo "Completed $name" | tee -a "$MASTER_LOG" + else + echo "FAILED $name; inspect $LOG_DIR/$name.log" | tee -a "$MASTER_LOG" >&2 + failed=1 + fi +done +if (( failed != 0 )); then + exit 1 +fi + +python -m analysis.merge_result_shards \ + --config configs/revision/merge_tabular_legacy_extended.yaml \ + 2>&1 | tee -a "$MASTER_LOG" +python -m analysis.aggregate_results \ + --config configs/revision/statistics_tabular_legacy_extended.yaml \ + 2>&1 | tee -a "$MASTER_LOG" +python -m analysis.tabular_legacy_statistics \ + --config configs/revision/statistics_tabular_legacy_extended.yaml \ + 2>&1 | tee -a "$MASTER_LOG" + +echo "All 144 runs and final aggregation completed." | tee -a "$MASTER_LOG" diff --git a/jobs/datacook-tabular-legacy-params-5gpu-local.sh b/jobs/datacook-tabular-legacy-params-5gpu-local.sh new file mode 100644 index 0000000..72d77ab --- /dev/null +++ b/jobs/datacook-tabular-legacy-params-5gpu-local.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Five-GPU sharded tabular DataCook budget ablation. +# Default physical GPU IDs: 0,1,3,4,5. +# +# Run in the foreground inside screen: +# GPU_LIST=0,1,3,4,5 CONDA_ENV=datacook \ +# bash jobs/datacook-tabular-legacy-params-5gpu-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-legacy-params-5gpu" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +GPU_LIST="${GPU_LIST:-0,1,3,4,5}" +DIABETES_DATA="${DIABETES_DATA:-$PROJECT_ROOT/TableData/dataset/diabetes.csv}" +HEPATITIS_DATA="${HEPATITIS_DATA:-$PROJECT_ROOT/TableData/dataset/HepatitisCdata.csv}" +SYSTEMIC_DATA="${SYSTEMIC_DATA:-$PROJECT_ROOT/TableData/dataset/systemic diseases.csv}" +ARTIFACT_BASE="$PROJECT_ROOT/artifacts/revision_pipeline_tabular_legacy_params/shards" +RESULT_BASE="$PROJECT_ROOT/results/revision_pipeline_tabular_legacy_params/shards" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" "$ARTIFACT_BASE" "$RESULT_BASE" +MASTER_LOG="$LOG_DIR/$JOB_NAME-$(date +%Y%m%d-%H%M%S).log" + +IFS=',' read -r -a GPUS <<< "$GPU_LIST" +if (( ${#GPUS[@]} < 5 )); then + echo "GPU_LIST must provide at least five comma-separated GPU IDs." >&2 + exit 2 +fi + +for data_path in "$DIABETES_DATA" "$HEPATITIS_DATA" "$SYSTEMIC_DATA"; do + if [[ ! -f "$data_path" ]]; then + echo "Missing tabular CSV: $data_path" >&2 + exit 2 + fi +done + +if [[ -n "${CONDA_PREFIX:-}" && -x "$CONDA_PREFIX/bin/python" ]]; then + echo "Using active environment: $CONDA_PREFIX" | tee "$MASTER_LOG" +elif command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" + echo "Activated conda environment: $CONDA_ENV" | tee "$MASTER_LOG" +else + echo "No conda command found; using active Python: $(command -v python)" | tee "$MASTER_LOG" +fi + +cd "$PROJECT_ROOT" +echo "GPU list: $GPU_LIST" | tee -a "$MASTER_LOG" +echo "Grid: 3 datasets x 3 seeds x 6 budgets = 54 runs" | tee -a "$MASTER_LOG" +echo "Parameters: step_size=0.05, num_steps=300, batch_size=32" | tee -a "$MASTER_LOG" + +run_shard() { + local gpu="$1" + local shard="$2" + local config="$3" + local data_path="$4" + shift 4 + local shard_log="$LOG_DIR/$shard.log" + echo "Starting $shard on physical GPU $gpu; log=$shard_log" | tee -a "$MASTER_LOG" + CUDA_VISIBLE_DEVICES="$gpu" python -u run_tabular_epsilon_ablation.py \ + --config "$config" \ + --data-path "$data_path" \ + --device cuda \ + --artifact-root "$ARTIFACT_BASE/$shard" \ + --result-root "$RESULT_BASE/$shard" \ + "$@" >"$shard_log" 2>&1 +} + +pids=() +names=() + +( + run_shard "${GPUS[0]}" systemic-s0-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 0 --epsilons 0.1 0.25 0.5 1 +) & +pids+=("$!"); names+=("systemic-s0-low") + +( + run_shard "${GPUS[1]}" systemic-s0-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 0 --epsilons 2 5 + run_shard "${GPUS[1]}" systemic-s1-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 1 --epsilons 0.1 0.25 +) & +pids+=("$!"); names+=("systemic-s0-high+systemic-s1-low") + +( + run_shard "${GPUS[2]}" systemic-s1-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 1 --epsilons 0.5 1 2 5 +) & +pids+=("$!"); names+=("systemic-s1-high") + +( + run_shard "${GPUS[3]}" systemic-s2-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 2 --epsilons 0.1 0.25 0.5 + run_shard "${GPUS[3]}" diabetes \ + configs/revision/tabular_legacy_params_diabetes.yaml "$DIABETES_DATA" +) & +pids+=("$!"); names+=("systemic-s2-low+diabetes") + +( + run_shard "${GPUS[4]}" systemic-s2-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 2 --epsilons 1 2 5 + run_shard "${GPUS[4]}" hepatitis_c \ + configs/revision/tabular_legacy_params_hepatitis_c.yaml "$HEPATITIS_DATA" +) & +pids+=("$!"); names+=("systemic-s2-high+hepatitis_c") + +failed=0 +for index in "${!pids[@]}"; do + if wait "${pids[$index]}"; then + echo "Completed ${names[$index]}" | tee -a "$MASTER_LOG" + else + echo "FAILED ${names[$index]}" | tee -a "$MASTER_LOG" >&2 + failed=1 + fi +done +if (( failed != 0 )); then + echo "At least one shard failed; merge was not run." | tee -a "$MASTER_LOG" >&2 + exit 1 +fi + +python -m analysis.merge_result_shards \ + --config configs/revision/merge_tabular_legacy_params_7gpu.yaml \ + 2>&1 | tee -a "$MASTER_LOG" +python -m analysis.aggregate_results \ + --config configs/revision/statistics_tabular_legacy_params.yaml \ + 2>&1 | tee -a "$MASTER_LOG" + +echo "All 54 runs and final aggregation completed." | tee -a "$MASTER_LOG" diff --git a/jobs/datacook-tabular-legacy-params-7gpu-local.sh b/jobs/datacook-tabular-legacy-params-7gpu-local.sh new file mode 100644 index 0000000..20efe57 --- /dev/null +++ b/jobs/datacook-tabular-legacy-params-7gpu-local.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Seven-GPU sharded tabular DataCook budget ablation. +# Run in the foreground inside screen: +# GPU_LIST=0,1,2,3,4,5,6 CONDA_ENV=datacook \ +# bash jobs/datacook-tabular-legacy-params-7gpu-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-legacy-params-7gpu" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +GPU_LIST="${GPU_LIST:-0,1,2,3,4,5,6}" +DIABETES_DATA="${DIABETES_DATA:-$PROJECT_ROOT/TableData/dataset/diabetes.csv}" +HEPATITIS_DATA="${HEPATITIS_DATA:-$PROJECT_ROOT/TableData/dataset/HepatitisCdata.csv}" +SYSTEMIC_DATA="${SYSTEMIC_DATA:-$PROJECT_ROOT/TableData/dataset/systemic diseases.csv}" +ARTIFACT_BASE="$PROJECT_ROOT/artifacts/revision_pipeline_tabular_legacy_params/shards" +RESULT_BASE="$PROJECT_ROOT/results/revision_pipeline_tabular_legacy_params/shards" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" "$ARTIFACT_BASE" "$RESULT_BASE" +MASTER_LOG="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +IFS=',' read -r -a GPUS <<< "$GPU_LIST" +if (( ${#GPUS[@]} < 7 )); then + echo "GPU_LIST must provide at least seven comma-separated GPU IDs." >&2 + exit 2 +fi + +for data_path in "$DIABETES_DATA" "$HEPATITIS_DATA" "$SYSTEMIC_DATA"; do + if [[ ! -f "$data_path" ]]; then + echo "Missing tabular CSV: $data_path" >&2 + exit 2 + fi +done + +if [[ -n "${CONDA_PREFIX:-}" && -x "$CONDA_PREFIX/bin/python" ]]; then + echo "Using active environment: $CONDA_PREFIX" | tee "$MASTER_LOG" +elif command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" + echo "Activated conda environment: $CONDA_ENV" | tee "$MASTER_LOG" +else + echo "No conda command found; using active Python: $(command -v python)" | tee "$MASTER_LOG" +fi + +cd "$PROJECT_ROOT" +echo "GPU list: $GPU_LIST" | tee -a "$MASTER_LOG" +echo "Grid: 3 datasets x 3 seeds x 6 budgets = 54 runs" | tee -a "$MASTER_LOG" +echo "Parameters: step_size=0.05, num_steps=300, batch_size=32" | tee -a "$MASTER_LOG" + +run_shard() { + local gpu="$1" + local shard="$2" + local config="$3" + local data_path="$4" + shift 4 + local shard_log="$LOG_DIR/${shard}.log" + echo "Starting $shard on physical GPU $gpu; log=$shard_log" | tee -a "$MASTER_LOG" + CUDA_VISIBLE_DEVICES="$gpu" python -u run_tabular_epsilon_ablation.py \ + --config "$config" \ + --data-path "$data_path" \ + --device cuda \ + --artifact-root "$ARTIFACT_BASE/$shard" \ + --result-root "$RESULT_BASE/$shard" \ + "$@" >"$shard_log" 2>&1 +} + +pids=() +names=() + +run_shard "${GPUS[0]}" systemic-s0-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 0 --epsilons 0.1 0.25 0.5 & +pids+=("$!"); names+=("systemic-s0-low") + +run_shard "${GPUS[1]}" systemic-s0-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 0 --epsilons 1 2 5 & +pids+=("$!"); names+=("systemic-s0-high") + +run_shard "${GPUS[2]}" systemic-s1-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 1 --epsilons 0.1 0.25 0.5 & +pids+=("$!"); names+=("systemic-s1-low") + +run_shard "${GPUS[3]}" systemic-s1-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 1 --epsilons 1 2 5 & +pids+=("$!"); names+=("systemic-s1-high") + +run_shard "${GPUS[4]}" systemic-s2-low \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 2 --epsilons 0.1 0.25 0.5 & +pids+=("$!"); names+=("systemic-s2-low") + +run_shard "${GPUS[5]}" systemic-s2-high \ + configs/revision/tabular_legacy_params_systemic_diseases.yaml "$SYSTEMIC_DATA" \ + --seeds 2 --epsilons 1 2 5 & +pids+=("$!"); names+=("systemic-s2-high") + +( + run_shard "${GPUS[6]}" diabetes \ + configs/revision/tabular_legacy_params_diabetes.yaml "$DIABETES_DATA" + run_shard "${GPUS[6]}" hepatitis_c \ + configs/revision/tabular_legacy_params_hepatitis_c.yaml "$HEPATITIS_DATA" +) & +pids+=("$!"); names+=("diabetes+hepatitis_c") + +failed=0 +for index in "${!pids[@]}"; do + if wait "${pids[$index]}"; then + echo "Completed ${names[$index]}" | tee -a "$MASTER_LOG" + else + echo "FAILED ${names[$index]}" | tee -a "$MASTER_LOG" >&2 + failed=1 + fi +done +if (( failed != 0 )); then + echo "At least one shard failed; merge was not run." | tee -a "$MASTER_LOG" >&2 + exit 1 +fi + +python -m analysis.merge_result_shards \ + --config configs/revision/merge_tabular_legacy_params_7gpu.yaml \ + 2>&1 | tee -a "$MASTER_LOG" +python -m analysis.aggregate_results \ + --config configs/revision/statistics_tabular_legacy_params.yaml \ + 2>&1 | tee -a "$MASTER_LOG" + +echo "All 54 runs and final aggregation completed." | tee -a "$MASTER_LOG" diff --git a/jobs/datacook-tabular-legacy-params-local.sh b/jobs/datacook-tabular-legacy-params-local.sh new file mode 100644 index 0000000..c8fe8ac --- /dev/null +++ b/jobs/datacook-tabular-legacy-params-local.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tabular DataCook budget ablation with the legacy optimizer parameters. +# Project: DataCook +# Created: 2026-10-22 +# Commit: no-git +# +# Foreground run inside an existing screen session: +# CUDA_VISIBLE_DEVICES=0 CONDA_ENV=datacook \ +# bash jobs/datacook-tabular-legacy-params-local.sh + +set -euo pipefail + +JOB_NAME="datacook-tabular-legacy-params" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LOG_DIR="$PROJECT_ROOT/outputs/logs/$JOB_NAME" +OUTPUT_DIR="$PROJECT_ROOT/outputs/$JOB_NAME" +CONDA_ENV="${CONDA_ENV:-datacook}" +DIABETES_DATA="${DIABETES_DATA:-$PROJECT_ROOT/TableData/dataset/diabetes.csv}" +HEPATITIS_DATA="${HEPATITIS_DATA:-$PROJECT_ROOT/TableData/dataset/HepatitisCdata.csv}" +SYSTEMIC_DATA="${SYSTEMIC_DATA:-$PROJECT_ROOT/TableData/dataset/systemic diseases.csv}" +TABULAR_DEVICE="${TABULAR_DEVICE:-cuda}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" +export GIT_COMMIT="no-git" + +mkdir -p "$LOG_DIR" "$OUTPUT_DIR" +LOG_FILE="$LOG_DIR/${JOB_NAME}-$(date +%Y%m%d-%H%M%S).log" + +echo "=========================================" | tee "$LOG_FILE" +echo "Job: $JOB_NAME" | tee -a "$LOG_FILE" +echo "Start: $(date)" | tee -a "$LOG_FILE" +echo "Commit: $GIT_COMMIT" | tee -a "$LOG_FILE" +echo "GPU: $CUDA_VISIBLE_DEVICES" | tee -a "$LOG_FILE" +echo "Device: $TABULAR_DEVICE" | tee -a "$LOG_FILE" +echo "Params: epsilon=[0.1,0.25,0.5,1,2,5], step_size=0.05, num_steps=300, batch_size=32" | tee -a "$LOG_FILE" +echo "Training: Adam, lr=0.001, unweighted CE/BCEWithLogits" | tee -a "$LOG_FILE" +echo "Log: $LOG_FILE" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" + +for data_path in "$DIABETES_DATA" "$HEPATITIS_DATA" "$SYSTEMIC_DATA"; do + if [[ ! -f "$data_path" ]]; then + echo "Missing tabular CSV: $data_path" | tee -a "$LOG_FILE" >&2 + exit 2 + fi +done + +if command -v conda >/dev/null 2>&1; then + eval "$(conda shell.bash hook)" + conda activate "$CONDA_ENV" +else + echo "conda command not found; using the active Python environment." | tee -a "$LOG_FILE" +fi + +cd "$PROJECT_ROOT" + +run_logged() { + echo "Running: $*" | tee -a "$LOG_FILE" + "$@" 2>&1 | tee -a "$LOG_FILE" +} + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_diabetes.yaml \ + --data-path "$DIABETES_DATA" --device "$TABULAR_DEVICE" + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_hepatitis_c.yaml \ + --data-path "$HEPATITIS_DATA" --device "$TABULAR_DEVICE" + +run_logged python run_tabular_epsilon_ablation.py \ + --config configs/revision/tabular_legacy_params_systemic_diseases.yaml \ + --data-path "$SYSTEMIC_DATA" --device "$TABULAR_DEVICE" + +run_logged python -m analysis.merge_result_shards \ + --config configs/revision/merge_tabular_legacy_params.yaml + +run_logged python -m analysis.aggregate_results \ + --config configs/revision/statistics_tabular_legacy_params.yaml + +echo "-----------------------------------------" | tee -a "$LOG_FILE" +echo "End: $(date)" | tee -a "$LOG_FILE" +echo "Runs: results/revision_pipeline_tabular_legacy_params (54 runs)" | tee -a "$LOG_FILE" +echo "=========================================" | tee -a "$LOG_FILE" diff --git a/main.py b/main.py index caf8178..baf8c65 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,7 @@ import os import shutil import time +import random import numpy as np import dataset import mlconfig @@ -23,6 +24,12 @@ parser.add_argument('--data_parallel', action='store_true', default=False) parser.add_argument('--train', action='store_true', default=False) parser.add_argument('--save_frequency', default=-1, type=int) +parser.add_argument('--deterministic', action='store_true', default=False, + help='Use deterministic PyTorch algorithms for reproducible multi-seed runs') +parser.add_argument('--export_predictions_dir', type=str, default='', + help='Directory for standardized revision prediction NPZ files') +parser.add_argument('--evaluation_role', choices=['clean', 'protected'], default='protected', + help='Export clean or authorized/unauthorized prediction names') # Datasets Options parser.add_argument('--train_face', action='store_true', default=False) parser.add_argument('--train_portion', default=1.0, type=float) @@ -35,6 +42,8 @@ parser.add_argument('--train_data_path', type=str, default='../datasets') parser.add_argument('--test_data_path', type=str, default='../datasets') parser.add_argument('--recover_test_data_path', type=str, default='../datasets') +parser.add_argument('--high_resolution', action='store_true', default=False, + help='Load the official 224x224 MedMNIST+ split') args = parser.parse_args() # Set up Experiments @@ -51,11 +60,19 @@ # CUDA Options logger.info("PyTorch Version: %s" % (torch.__version__)) +random.seed(args.seed) +np.random.seed(args.seed) +torch.manual_seed(args.seed) +if args.deterministic and hasattr(torch, 'use_deterministic_algorithms'): + try: + torch.use_deterministic_algorithms(True, warn_only=True) + except TypeError: + torch.use_deterministic_algorithms(True) if torch.cuda.is_available(): - torch.manual_seed(args.seed) - torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.enabled = True - torch.backends.cudnn.benchmark = True + torch.backends.cudnn.benchmark = not args.deterministic + torch.backends.cudnn.deterministic = args.deterministic device = torch.device('cuda') device_list = [torch.cuda.get_device_name(i) for i in range(0, torch.cuda.device_count())] logger.info("GPU List: %s" % (device_list)) @@ -138,7 +155,8 @@ def main(): train_batch_size=args.train_batch_size, eval_batch_size=args.eval_batch_size, num_of_workers=args.num_of_workers, - seed=args.seed,high_resolution=False) + seed=args.seed, + high_resolution=args.high_resolution) logger.info('Training Dataset: %s' % str(datasets_generator.datasets['train_dataset'])) logger.info('Test Dataset: %s' % str(datasets_generator.datasets['test_dataset'])) @@ -185,6 +203,18 @@ def main(): if args.train: train(starting_epoch, model, optimizer, scheduler, criterion, trainer, evaluator, ENV, data_loader) + if args.export_predictions_dir: + logger.info('Running final prediction export for role=%s', args.evaluation_role) + evaluator._reset_stats() + evaluator.eval(0, model) + if args.evaluation_role == 'clean': + evaluator.save_predictions(os.path.join(args.export_predictions_dir, 'clean.npz')) + else: + evaluator.save_predictions(os.path.join(args.export_predictions_dir, 'unauthorized_raw.npz')) + evaluator.save_predictions( + os.path.join(args.export_predictions_dir, 'authorized_cooked.npz'), + recover=True) + if __name__ == '__main__': for arg in vars(args): diff --git a/metrics/__init__.py b/metrics/__init__.py new file mode 100644 index 0000000..9e7b9c5 --- /dev/null +++ b/metrics/__init__.py @@ -0,0 +1,12 @@ +"""Reusable metrics for the DataCook revision experiments.""" + +from .classification_metrics import compute_classification_metrics, compute_ece +from .cp_pp_metrics import compute_cp_pp +from .image_quality import compute_image_quality_metrics + +__all__ = [ + "compute_classification_metrics", + "compute_ece", + "compute_cp_pp", + "compute_image_quality_metrics", +] diff --git a/metrics/classification_metrics.py b/metrics/classification_metrics.py new file mode 100644 index 0000000..97b2f8c --- /dev/null +++ b/metrics/classification_metrics.py @@ -0,0 +1,282 @@ +"""Unified binary, multiclass and multilabel classification metrics. + +The public functions accept NumPy arrays, Python sequences, or CPU/GPU tensors. +Undefined AUC values (for example, a test split containing only one class) are +reported as ``nan`` instead of aborting a long experiment. +""" + +from __future__ import annotations + +import warnings +from typing import Any + +import numpy as np +from sklearn.metrics import ( + accuracy_score, + balanced_accuracy_score, + f1_score, + hamming_loss, + precision_score, + recall_score, + roc_auc_score, +) + + +_TASK_ALIASES = { + "binary-class": "binary", + "binary_class": "binary", + "multi-class": "multiclass", + "multi_class": "multiclass", + "multilabel": "multilabel", + "multi-label": "multilabel", + "multi_label": "multilabel", +} + + +def _numpy(value: Any) -> np.ndarray | None: + if value is None: + return None + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + return np.asarray(value) + + +def _safe_divide(numerator: float, denominator: float) -> float: + return float(numerator / denominator) if denominator else 0.0 + + +def _safe_auc(y_true: np.ndarray, y_score: np.ndarray, **kwargs: Any) -> float: + try: + return float(roc_auc_score(y_true, y_score, **kwargs)) + except ValueError as exc: + warnings.warn(f"AUC is undefined for this split: {exc}", RuntimeWarning) + return float("nan") + + +def _as_binary_probability(y_prob: np.ndarray) -> np.ndarray: + if y_prob.ndim == 1: + return y_prob.astype(float) + if y_prob.ndim == 2 and y_prob.shape[1] == 1: + return y_prob[:, 0].astype(float) + if y_prob.ndim == 2 and y_prob.shape[1] == 2: + if not np.allclose(y_prob.sum(axis=1), 1.0, atol=1e-4): + raise ValueError("Two-column binary y_prob rows must sum to 1; pass softmax probabilities, not logits.") + return y_prob[:, 1].astype(float) + raise ValueError("Binary y_prob must have shape (N,), (N, 1), or (N, 2).") + + +def _validate_probabilities(y_prob: np.ndarray, name: str = "y_prob") -> None: + if not np.all(np.isfinite(y_prob)): + raise ValueError(f"{name} contains NaN or infinite values.") + if np.any(y_prob < 0) or np.any(y_prob > 1): + raise ValueError(f"{name} must contain probabilities in [0, 1], not logits.") + + +def _multiclass_auc( + y_true: np.ndarray, y_prob: np.ndarray, average: str +) -> float: + """OvR AUC that tolerates classes missing from a test split.""" + aucs, weights = [], [] + for class_index in range(y_prob.shape[1]): + binary_true = (y_true == class_index).astype(int) + if np.unique(binary_true).size < 2: + continue + aucs.append(roc_auc_score(binary_true, y_prob[:, class_index])) + weights.append(float(binary_true.sum())) + if not aucs: + warnings.warn("AUC is undefined because no class has both outcomes.", RuntimeWarning) + return float("nan") + if average == "macro": + return float(np.mean(aucs)) + return float(np.average(aucs, weights=weights)) + + +def _multiclass_specificity( + y_true: np.ndarray, y_pred: np.ndarray, num_classes: int +) -> tuple[float, float]: + values, weights = [], [] + for class_index in range(num_classes): + positive = y_true == class_index + negative = ~positive + tn = np.logical_and(negative, y_pred != class_index).sum() + fp = np.logical_and(negative, y_pred == class_index).sum() + values.append(_safe_divide(float(tn), float(tn + fp))) + weights.append(float(positive.sum())) + macro = float(np.mean(values)) + weighted = float(np.average(values, weights=weights)) if sum(weights) else macro + return macro, weighted + + +def compute_ece( + y_true: Any, + y_prob: Any, + n_bins: int = 15, + task_type: str | None = None, +) -> float: + """Compute equal-width expected calibration error. + + Binary and multiclass use top-label confidence. Multilabel ECE treats every + label decision as a binary prediction, which makes the returned value easy + to compare across datasets with different label counts. + """ + if n_bins <= 0: + raise ValueError("n_bins must be positive.") + truth = _numpy(y_true) + prob = _numpy(y_prob) + if truth is None or prob is None: + raise ValueError("y_true and y_prob are required for ECE.") + task = _TASK_ALIASES.get(str(task_type).lower(), str(task_type).lower()) if task_type else None + _validate_probabilities(prob) + + if task == "multilabel" or (task is None and truth.ndim == 2 and prob.ndim == 2 and prob.shape[1] != 2): + if truth.shape != prob.shape: + raise ValueError("Multilabel y_true and y_prob must have the same shape.") + predicted = prob >= 0.5 + confidence = np.where(predicted, prob, 1.0 - prob).reshape(-1) + correct = (predicted == truth.astype(bool)).astype(float).reshape(-1) + elif prob.ndim == 2 and prob.shape[1] > 1 and task != "binary": + if prob.shape[0] != truth.reshape(-1).shape[0]: + raise ValueError("y_true and y_prob contain different sample counts.") + predicted = np.argmax(prob, axis=1) + confidence = np.max(prob, axis=1) + correct = (predicted == truth.reshape(-1).astype(int)).astype(float) + else: + binary_prob = _as_binary_probability(prob) + binary_truth = truth.reshape(-1).astype(int) + if binary_prob.shape[0] != binary_truth.shape[0]: + raise ValueError("y_true and y_prob contain different sample counts.") + predicted = binary_prob >= 0.5 + confidence = np.where(predicted, binary_prob, 1.0 - binary_prob) + correct = (predicted == binary_truth.astype(bool)).astype(float) + + edges = np.linspace(0.0, 1.0, n_bins + 1) + # Right-inclusive final bin prevents confidence == 1 from being discarded. + bin_ids = np.minimum(np.digitize(confidence, edges[1:-1], right=False), n_bins - 1) + ece = 0.0 + for bin_index in range(n_bins): + mask = bin_ids == bin_index + if np.any(mask): + ece += float(mask.mean()) * abs(float(correct[mask].mean()) - float(confidence[mask].mean())) + return float(ece) + + +def compute_classification_metrics( + y_true: Any, + y_pred: Any = None, + y_prob: Any = None, + task_type: str = "multiclass", + threshold: float = 0.5, +) -> dict[str, float]: + """Compute task-appropriate classification metrics. + + ``y_prob`` must contain probabilities. Passing logits is deliberately an + error so that softmax/sigmoid mistakes cannot silently corrupt AUC or ECE. + Predictions are inferred from probabilities when ``y_pred`` is omitted. + """ + task = _TASK_ALIASES.get(task_type.lower(), task_type.lower()) + if task not in {"binary", "multiclass", "multilabel"}: + raise ValueError(f"Unsupported task_type {task_type!r}.") + if not 0 <= threshold <= 1: + raise ValueError("threshold must be in [0, 1].") + + truth = _numpy(y_true) + pred = _numpy(y_pred) + prob = _numpy(y_prob) + if truth is None or truth.size == 0: + raise ValueError("y_true must contain at least one sample.") + if prob is not None: + _validate_probabilities(prob) + + if task == "binary": + truth = truth.reshape(-1).astype(int) + if not set(np.unique(truth)).issubset({0, 1}): + raise ValueError("Binary y_true must contain only 0 and 1.") + binary_prob = _as_binary_probability(prob) if prob is not None else None + if pred is None: + if binary_prob is None: + raise ValueError("Either y_pred or y_prob is required.") + pred = (binary_prob >= threshold).astype(int) + pred = pred.reshape(-1).astype(int) + if pred.shape != truth.shape: + raise ValueError("y_true and y_pred contain different sample counts.") + tn = np.logical_and(truth == 0, pred == 0).sum() + fp = np.logical_and(truth == 0, pred == 1).sum() + fn = np.logical_and(truth == 1, pred == 0).sum() + tp = np.logical_and(truth == 1, pred == 1).sum() + return { + "acc": float(accuracy_score(truth, pred)), + "balanced_acc": float(balanced_accuracy_score(truth, pred)), + "auc": _safe_auc(truth, binary_prob) if binary_prob is not None else float("nan"), + "macro_f1": float(f1_score(truth, pred, average="macro", zero_division=0)), + "weighted_f1": float(f1_score(truth, pred, average="weighted", zero_division=0)), + "precision": float(precision_score(truth, pred, zero_division=0)), + "sensitivity": _safe_divide(float(tp), float(tp + fn)), + "specificity": _safe_divide(float(tn), float(tn + fp)), + "ece": compute_ece(truth, binary_prob, task_type=task) if binary_prob is not None else float("nan"), + } + + if task == "multiclass": + truth = truth.reshape(-1).astype(int) + if pred is None: + if prob is None: + raise ValueError("Either y_pred or y_prob is required.") + if prob.ndim != 2: + raise ValueError("Multiclass y_prob must have shape (N, C).") + pred = np.argmax(prob, axis=1) + pred = pred.reshape(-1).astype(int) + if pred.shape != truth.shape: + raise ValueError("y_true and y_pred contain different sample counts.") + if prob is not None and (prob.ndim != 2 or prob.shape[0] != truth.shape[0]): + raise ValueError("Multiclass y_prob must have shape (N, C).") + if prob is not None and not np.allclose(prob.sum(axis=1), 1.0, atol=1e-4): + raise ValueError("Multiclass y_prob rows must sum to 1; pass softmax probabilities, not logits.") + num_classes = prob.shape[1] if prob is not None else int(max(truth.max(), pred.max()) + 1) + macro_specificity, _ = _multiclass_specificity(truth, pred, num_classes) + return { + "acc": float(accuracy_score(truth, pred)), + "balanced_acc": float(balanced_accuracy_score(truth, pred)), + "macro_auc": _multiclass_auc(truth, prob, "macro") if prob is not None else float("nan"), + "weighted_auc": _multiclass_auc(truth, prob, "weighted") if prob is not None else float("nan"), + "macro_f1": float(f1_score(truth, pred, average="macro", zero_division=0)), + "weighted_f1": float(f1_score(truth, pred, average="weighted", zero_division=0)), + "macro_precision": float(precision_score(truth, pred, average="macro", zero_division=0)), + "macro_sensitivity": float(recall_score(truth, pred, average="macro", zero_division=0)), + "macro_specificity": macro_specificity, + "ece": compute_ece(truth, prob, task_type=task) if prob is not None else float("nan"), + } + + truth = np.asarray(truth).astype(int) + if truth.ndim != 2: + raise ValueError("Multilabel y_true must have shape (N, L).") + if pred is None: + if prob is None: + raise ValueError("Either y_pred or y_prob is required.") + pred = (prob >= threshold).astype(int) + pred = np.asarray(pred).astype(int) + if pred.shape != truth.shape: + raise ValueError("Multilabel y_true and y_pred must have the same shape.") + if prob is not None and prob.shape != truth.shape: + raise ValueError("Multilabel y_true and y_prob must have the same shape.") + + tn = np.logical_and(truth == 0, pred == 0).sum(axis=0) + fp = np.logical_and(truth == 0, pred == 1).sum(axis=0) + macro_specificity = float(np.mean([_safe_divide(float(a), float(a + b)) for a, b in zip(tn, fp)])) + micro_specificity = _safe_divide(float(tn.sum()), float((tn + fp).sum())) + macro_auc = float(np.nanmean([ + _safe_auc(truth[:, i], prob[:, i]) for i in range(truth.shape[1]) + ])) if prob is not None else float("nan") + return { + "subset_acc": float(accuracy_score(truth, pred)), + "hamming_loss": float(hamming_loss(truth, pred)), + "micro_auc": _safe_auc(truth.reshape(-1), prob.reshape(-1)) if prob is not None else float("nan"), + "macro_auc": macro_auc, + "micro_f1": float(f1_score(truth, pred, average="micro", zero_division=0)), + "macro_f1": float(f1_score(truth, pred, average="macro", zero_division=0)), + "micro_precision": float(precision_score(truth, pred, average="micro", zero_division=0)), + "macro_precision": float(precision_score(truth, pred, average="macro", zero_division=0)), + "micro_sensitivity": float(recall_score(truth, pred, average="micro", zero_division=0)), + "macro_sensitivity": float(recall_score(truth, pred, average="macro", zero_division=0)), + "micro_specificity": micro_specificity, + "macro_specificity": macro_specificity, + "ece": compute_ece(truth, prob, task_type=task) if prob is not None else float("nan"), + } diff --git a/metrics/cp_pp_metrics.py b/metrics/cp_pp_metrics.py new file mode 100644 index 0000000..0d79acf --- /dev/null +++ b/metrics/cp_pp_metrics.py @@ -0,0 +1,84 @@ +"""Generalized copyright/privacy (CP/PP) calculations.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +import numpy as np + + +def _random_baseline( + metric: str, + task_type: str | None, + num_classes: int | None, + label_prevalence: Sequence[float] | None, +) -> float: + name = metric.lower() + if "auc" in name: + return 0.5 + task = (task_type or "").replace("-", "").replace("_", "").lower() + if task == "binary": + if name in {"acc", "balanced_acc", "sensitivity", "specificity"}: + return 0.5 + elif task == "multiclass" and num_classes and num_classes > 0: + if name in {"acc", "balanced_acc", "macro_f1", "macro_precision", "macro_sensitivity"}: + return 1.0 / num_classes + if name == "macro_specificity": + return 1.0 - 1.0 / num_classes + elif task == "multilabel" and label_prevalence is not None: + prevalence = np.asarray(label_prevalence, dtype=float) + if name == "subset_acc": + return float(np.prod(prevalence**2 + (1.0 - prevalence) ** 2)) + if name in {"micro_auc", "macro_auc", "micro_sensitivity", "macro_sensitivity", "micro_specificity", "macro_specificity"}: + return 0.5 + return float("nan") + + +def compute_cp_pp( + clean_metrics: Mapping[str, float], + authorized_metrics: Mapping[str, float], + unauthorized_metrics: Mapping[str, float], + metric_names: Iterable[str], + random_guess_metrics: Mapping[str, float] | None = None, + *, + task_type: str | None = None, + num_classes: int | None = None, + label_prevalence: Sequence[float] | None = None, +) -> dict[str, dict[str, float]]: + """Calculate PP, CP and residual utility for any common metric set. + + PP is ``authorized_cooked - clean`` and CP is ``unauthorized_raw - clean``. + Negative CP therefore represents utility degradation, preserving the sign + convention used by the original DataCook experiments. + """ + output: dict[str, dict[str, float]] = {} + for metric in metric_names: + missing = [ + name + for name, values in ( + ("clean_metrics", clean_metrics), + ("authorized_metrics", authorized_metrics), + ("unauthorized_metrics", unauthorized_metrics), + ) + if metric not in values + ] + if missing: + raise KeyError(f"Metric {metric!r} is missing from {', '.join(missing)}.") + clean = float(clean_metrics[metric]) + authorized = float(authorized_metrics[metric]) + unauthorized = float(unauthorized_metrics[metric]) + if random_guess_metrics is not None and metric in random_guess_metrics: + random_guess = float(random_guess_metrics[metric]) + else: + random_guess = _random_baseline(metric, task_type, num_classes, label_prevalence) + output[metric] = { + "clean": clean, + "authorized_cooked": authorized, + "unauthorized_raw": unauthorized, + "cp": unauthorized - clean, + "pp": authorized - clean, + "random_guess": random_guess, + "residual_utility": unauthorized - random_guess if np.isfinite(random_guess) else float("nan"), + } + return output diff --git a/metrics/image_quality.py b/metrics/image_quality.py new file mode 100644 index 0000000..cd241a3 --- /dev/null +++ b/metrics/image_quality.py @@ -0,0 +1,223 @@ +"""Image quality and perturbation-norm metrics for DataCook.""" + +from __future__ import annotations + +import warnings +from typing import Any + +import numpy as np +from skimage.metrics import structural_similarity + + +def _numpy(value: Any) -> np.ndarray: + if hasattr(value, "detach"): + value = value.detach().cpu().numpy() + return np.asarray(value, dtype=np.float32) + + +def _nchw(images: np.ndarray) -> np.ndarray: + if images.ndim == 3: # N,H,W grayscale + return images[:, None, :, :] + if images.ndim != 4: + raise ValueError("Images must have shape (N,H,W), (N,C,H,W), or (N,H,W,C).") + if images.shape[1] in (1, 3): + return images + if images.shape[-1] in (1, 3): + return np.transpose(images, (0, 3, 1, 2)) + raise ValueError("Could not infer image channel dimension; only grayscale and RGB are supported.") + + +def _ncdhw(volumes: np.ndarray) -> np.ndarray: + if volumes.ndim != 5: + raise ValueError("Volumes must have shape (N,C,D,H,W) or (N,D,H,W,C).") + if volumes.shape[1] in (1, 3): + return volumes + if volumes.shape[-1] in (1, 3): + return np.transpose(volumes, (0, 4, 1, 2, 3)) + raise ValueError( + "Could not infer volume channel dimension; only one- and three-channel " + "volumes are supported." + ) + + +def _canonical_samples(values: np.ndarray) -> tuple[np.ndarray, bool]: + if values.ndim == 5: + return _ncdhw(values), True + return _nchw(values), False + + +def _summary(values: list[float]) -> tuple[float, float]: + array = np.asarray(values, dtype=float) + if np.all(np.isposinf(array)): + return float("inf"), 0.0 + if np.all(np.isneginf(array)): + return float("-inf"), 0.0 + return float(np.mean(array)), float(np.std(array, ddof=1)) if len(array) > 1 else 0.0 + + +def _lpips_values( + raw: np.ndarray, + cooked: np.ndarray, + device: str | None, + batch_size: int = 64, +) -> list[float]: + """Compute LPIPS in bounded batches, resizing tiny MedMNIST images. + + AlexNet's pooling stack cannot process the native 28x28 MedMNIST + resolution. Resizing is applied only for LPIPS; SSIM, PSNR and norm + metrics continue to use the original pixels. + """ + if batch_size <= 0: + raise ValueError("LPIPS batch_size must be positive.") + try: + import lpips # type: ignore + import torch + import torch.nn.functional as F + except ImportError: + warnings.warn( + "LPIPS is not installed; lpips_mean/lpips_std will be NaN. Install it with `pip install lpips`.", + RuntimeWarning, + ) + return [float("nan")] * raw.shape[0] + + selected_device = device or ("cuda" if torch.cuda.is_available() else "cpu") + try: + model = lpips.LPIPS(net="alex").to(selected_device).eval() + except OSError as exc: + raise RuntimeError(f"Failed to load LPIPS model: {exc}") from exc + + raw_tensor = torch.from_numpy(raw) + cooked_tensor = torch.from_numpy(cooked) + distances = [] + with torch.no_grad(): + for start in range(0, len(raw_tensor), batch_size): + raw_batch = raw_tensor[start:start + batch_size].to(selected_device) + cooked_batch = cooked_tensor[start:start + batch_size].to(selected_device) + if raw_batch.shape[1] == 1: + raw_batch = raw_batch.repeat(1, 3, 1, 1) + cooked_batch = cooked_batch.repeat(1, 3, 1, 1) + if min(raw_batch.shape[-2:]) < 64: + raw_batch = F.interpolate( + raw_batch, size=(64, 64), mode="bilinear", align_corners=False + ) + cooked_batch = F.interpolate( + cooked_batch, size=(64, 64), mode="bilinear", align_corners=False + ) + # Input arrays use [0,1]; LPIPS expects [-1,1]. + batch_distances = model(raw_batch * 2 - 1, cooked_batch * 2 - 1) + distances.append(batch_distances.reshape(-1).detach().cpu()) + return torch.cat(distances).numpy().astype(float).tolist() + + +def compute_image_quality_metrics( + raw_images: Any, + cooked_images: Any, + *, + data_range: float | None = None, + compute_lpips: bool = True, + device: str | None = None, +) -> dict[str, float]: + """Return SSIM/PSNR/LPIPS and perturbation-norm summaries. + + Both inputs must already use the same scale. If ``data_range`` is omitted, + [0,1] and [0,255] inputs are recognized; other ranges use their joint span. + Images may be NCHW/NHWC and volumes may be NCDHW/NDHWC. For volumes, SSIM + is computed over all three spatial dimensions and LPIPS must be disabled. + + ``psnr_mean`` is the dataset-level PSNR computed from the aggregate MSE. + This avoids an undefined infinite arithmetic mean when only a subset of + quantized images is pixel-identical. ``psnr_std`` is the sample standard + deviation over finite per-image PSNR values. If every image is identical, + PSNR remains correctly reported as infinity with zero standard deviation. + """ + raw, is_volume = _canonical_samples(_numpy(raw_images)) + cooked, cooked_is_volume = _canonical_samples(_numpy(cooked_images)) + if is_volume != cooked_is_volume: + raise ValueError("raw_images and cooked_images must both be images or both be volumes.") + if raw.shape != cooked.shape: + raise ValueError(f"raw_images and cooked_images have different shapes: {raw.shape} vs {cooked.shape}.") + if raw.shape[0] == 0: + raise ValueError("At least one image is required.") + if not np.all(np.isfinite(raw)) or not np.all(np.isfinite(cooked)): + raise ValueError("Images contain NaN or infinite values.") + joint_min = float(min(raw.min(), cooked.min())) + joint_max = float(max(raw.max(), cooked.max())) + if data_range is None: + if joint_min >= 0 and joint_max <= 1.0 + 1e-6: + data_range = 1.0 + elif joint_min >= 0 and joint_max <= 255.0 + 1e-6: + data_range = 255.0 + else: + data_range = joint_max - joint_min + if data_range <= 0: + data_range = 1.0 + + ssim_values, psnr_values, mse_values, l2_values, linf_values = [], [], [], [], [] + for raw_sample, cooked_sample in zip(raw, cooked): + if is_volume: + raw_spatial = np.transpose(raw_sample, (1, 2, 3, 0)) + cooked_spatial = np.transpose(cooked_sample, (1, 2, 3, 0)) + spatial_shape = raw_spatial.shape[:3] + else: + raw_spatial = np.transpose(raw_sample, (1, 2, 0)) + cooked_spatial = np.transpose(cooked_sample, (1, 2, 0)) + spatial_shape = raw_spatial.shape[:2] + win_size = min(7, *spatial_shape) + if win_size % 2 == 0: + win_size -= 1 + if win_size < 3: + sample_type = "volumes" if is_volume else "images" + raise ValueError(f"SSIM requires {sample_type} at least 3 pixels per spatial dimension.") + channel_axis = -1 if raw_spatial.shape[-1] > 1 else None + if channel_axis is None: + raw_spatial = raw_spatial[..., 0] + cooked_spatial = cooked_spatial[..., 0] + ssim_values.append(float(structural_similarity( + raw_spatial, + cooked_spatial, + data_range=data_range, + channel_axis=channel_axis, + win_size=win_size, + ))) + difference = (cooked_sample - raw_sample).reshape(-1) + mse = float(np.mean(np.square(difference.astype(np.float64)))) + mse_values.append(mse) + if mse > 0: + psnr_values.append(float(10.0 * np.log10((data_range ** 2) / mse))) + l2_values.append(float(np.linalg.norm(difference, ord=2))) + linf_values.append(float(np.max(np.abs(difference)))) + + if compute_lpips and is_volume: + raise ValueError( + "LPIPS is a 2D image metric and is not supported for 3D volumes; " + "set compute_lpips=False." + ) + if compute_lpips: + normalized_raw = np.clip((raw - joint_min) / data_range, 0.0, 1.0) + normalized_cooked = np.clip((cooked - joint_min) / data_range, 0.0, 1.0) + lpips_values = _lpips_values(normalized_raw, normalized_cooked, device) + else: + lpips_values = [float("nan")] * raw.shape[0] + + result: dict[str, float] = {} + for name, values in ( + ("ssim", ssim_values), + ("lpips", lpips_values), + ("l2_norm", l2_values), + ("linf_norm", linf_values), + ): + if np.all(np.isnan(values)): + mean, std = float("nan"), float("nan") + else: + mean, std = _summary(values) + result[f"{name}_mean"] = mean + result[f"{name}_std"] = std + aggregate_mse = float(np.mean(mse_values)) + if aggregate_mse == 0: + result["psnr_mean"] = float("inf") + result["psnr_std"] = 0.0 + else: + result["psnr_mean"] = float(10.0 * np.log10((data_range ** 2) / aggregate_mse)) + result["psnr_std"] = _summary(psnr_values)[1] + result["fid"] = float("nan") + return result diff --git a/models/.DS_Store b/models/.DS_Store deleted file mode 100644 index 38734ca2de71d90578b12a191d5ff30a57f26d5c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJ8Hu~5S@u#2;8`IxmU;y7U7)02atb(6aoc8igc=cE+5TrJ{W}TCXgn)ftj~E znx|!7q0xwlw%_Mhk+q04a6`FRn43K}pV>=h6bQ#VPI7>M$h&m2>c}_IdtuEgkdX>d0V;4;z`hR!Zden?K>u_g_y_=8 zBJGB?&l13531Cee1CfDgP=P_!95FQL$d|0EiDO{UMRWMjJXv!>Q9m8;FJ3NM0~x6R z6__fpi0#_?{~P?t{68geM+KA!1r*gxx&q`b_#;GW1zQV hY^)t`yeR65t?|4jj)6``-swR8445u7D)83|+yF)|6_)@2 diff --git a/models/__init__.py b/models/__init__.py index eda0c69..3d4e2a8 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -12,6 +12,7 @@ mlconfig.register(torch.optim.lr_scheduler.StepLR) mlconfig.register(torch.optim.lr_scheduler.ExponentialLR) mlconfig.register(torch.nn.CrossEntropyLoss) +mlconfig.register(torch.nn.BCEWithLogitsLoss) # Models mlconfig.register(ResNet.ResNet) @@ -50,4 +51,3 @@ def cross_entropy(input, target, size_average=True): return torch.mean(torch.sum(-target * logsoftmax(input), dim=1)) else: return torch.sum(torch.sum(-target * logsoftmax(input), dim=1)) - diff --git a/perturbation.py b/perturbation.py index f7db9db..85353cf 100644 --- a/perturbation.py +++ b/perturbation.py @@ -3,6 +3,7 @@ import os import shutil import time +import random import torch import numpy as np from pathlib import Path @@ -16,6 +17,7 @@ from trainer import Trainer from evaluator import Evaluator +from volume_io import save_cooked_array_npz, save_cooked_volume_npz from sklearn.cluster import KMeans @@ -36,6 +38,10 @@ def parse_args(): parser.add_argument('--config_path', type=str, default='configs/MedMNIST', help='Path to configuration YAML') parser.add_argument('--load_model', action='store_true', help='Whether to load pretrained model') parser.add_argument('--data_parallel', action='store_true', help='Use DataParallel for multi-GPU') + parser.add_argument('--deterministic', action='store_true', + help='Use deterministic PyTorch algorithms for reproducible multi-seed runs') + parser.add_argument('--export_predictions_dir', type=str, default='', + help='Save clean surrogate predictions using the revision NPZ contract') # Dataset options parser.add_argument('--train_batch_size', type=int, default=128) @@ -47,6 +53,8 @@ def parse_args(): parser.add_argument('--test_data_path', type=str, default='../datasets') parser.add_argument('--recover_test_data_type', type=str, default='PathMNIST') parser.add_argument('--recover_test_data_path', type=str, default='../datasets') + parser.add_argument('--high_resolution', action='store_true', + help='Load the official 224x224 MedMNIST+ split') # fingerprint options parser.add_argument('--universal_stop_error', type=float, default=0.5) @@ -57,6 +65,8 @@ def parse_args(): parser.add_argument('--noise_shape', type=int, nargs='+', default=[10, 3, 32, 32]) parser.add_argument('--test_noise_shape', type=int, nargs='+', default=[10, 3, 32, 32]) parser.add_argument('--epsilon', type=float, default=8, help='Max size of fingerprint') + parser.add_argument('--epsilon_scale', choices=['pixel', 'unit'], default='pixel', + help='Interpret epsilon/step_size in 0-255 pixel units (legacy) or directly in [0,1] units') parser.add_argument('--num_steps', type=int, default=1, help='Number of PGD steps') parser.add_argument('--step_size', type=float, default=0.8, help='Step size for PGD') @@ -66,6 +76,11 @@ def parse_args(): parser.add_argument('--g_net_num_epoch', default=50, type=int) return parser.parse_args() + +def _is_multilabel_task(task_type): + normalized = str(task_type or '').replace('-', '').replace('_', '').replace(' ', '').lower() + return normalized == 'multilabel' or normalized.startswith('multilabel,') + def setup_environment(args): """ Setup experiment directories, logger, and device. @@ -76,9 +91,15 @@ def setup_environment(args): Returns: tuple: exp_path, checkpoint_path, fingerprint_path, logger, device, num_classes """ - # Convert epsilon and step size from pixel range to [0,1] - args.epsilon /= 255.0 - args.step_size /= 255.0 + # Legacy scripts pass 8 for 8/255. Revision configs may pass 8/255 + # directly and select --epsilon_scale unit. + if args.epsilon_scale == 'pixel': + args.epsilon /= 255.0 + args.step_size /= 255.0 + if args.epsilon < 0 or args.epsilon > 1: + raise ValueError('Normalized epsilon must be in [0, 1].') + if args.step_size <= 0: + raise ValueError('step_size must be positive.') # Setup experiment directories if args.exp_name == '': @@ -95,10 +116,19 @@ def setup_environment(args): logger = util.setup_logger(name=args.version, log_file=os.path.join(exp_path, args.version + ".log")) # Setup device + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.deterministic and hasattr(torch, 'use_deterministic_algorithms'): + try: + torch.use_deterministic_algorithms(True, warn_only=True) + except TypeError: + torch.use_deterministic_algorithms(True) if torch.cuda.is_available(): - torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) torch.backends.cudnn.enabled = True - torch.backends.cudnn.benchmark = True + torch.backends.cudnn.benchmark = not args.deterministic + torch.backends.cudnn.deterministic = args.deterministic device = torch.device('cuda') logger.info("Using GPU: %s", torch.cuda.get_device_name(0)) else: @@ -112,9 +142,32 @@ def setup_environment(args): # Get number of classes from MedMNIST info info = INFO[args.train_data_type.lower()] num_classes = len(info['label']) + args.task_type = 'multilabel' if _is_multilabel_task(info.get('task')) else None return exp_path, checkpoint_path, fingerprint_path, logger, device, num_classes + +def save_pseudo_label_artifacts(model, data_loader, output_dir, device, logger): + """Persist the surrogate predictions actually available during cooking. + + Files are NPZ rather than CSV so multilabel probabilities/logits retain their + shape. ``run_pseudo_label_analysis.py`` converts them to the required long CSV. + """ + os.makedirs(output_dir, exist_ok=True) + model.eval() + for split in ('train_dataset', 'test_dataset'): + labels_all, logits_all = [], [] + with torch.no_grad(): + for batch in data_loader[split]: + images, labels = batch[0].to(device), batch[1] + logits_all.append(model(images).detach().cpu()) + labels_all.append(labels.detach().cpu()) + labels = torch.cat(labels_all).numpy() + logits = torch.cat(logits_all).numpy() + path = os.path.join(output_dir, f'pseudo_labels_{split}.npz') + np.savez_compressed(path, sample_id=np.arange(len(labels)), y_true=labels, logits=logits) + logger.info('Pseudo-label artifact saved at %s', path) + def train(args, config, logger, model, optimizer, scheduler, criterion, trainer, evaluator, ENV, ckpt_path): """ Pretrain the model to extract robust data features. @@ -124,53 +177,59 @@ def train(args, config, logger, model, optimizer, scheduler, criterion, trainer, model_file = os.path.join(ckpt_path, f"{args.version}_{config.epochs}.pth") logger.info('Model Saved at %s', model_file) + starting_epoch = 0 if os.path.exists(model_file): - latest_model_path = model_file - logger.info("Loading model from %s", latest_model_path) - checkpoint = torch.load(latest_model_path) + logger.info("Loading model from %s", model_file) + model_device = next(model.parameters()).device + checkpoint = torch.load(model_file, map_location=model_device) model.load_state_dict(checkpoint['model_state_dict']) - optimizer.load_state_dict(checkpoint['optimizer_state_dict']) - scheduler.load_state_dict(checkpoint['scheduler_state_dict']) + if checkpoint.get('optimizer_state_dict') is not None: + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) + if checkpoint.get('scheduler_state_dict') is not None: + scheduler.load_state_dict(checkpoint['scheduler_state_dict']) + starting_epoch = int(checkpoint.get('epoch', 0)) + if 'ENV' in checkpoint: + ENV.update(checkpoint['ENV']) + trainer.global_step = ENV.get('global_step', trainer.global_step) + logger.info("Resuming surrogate training from epoch %d/%d", starting_epoch, config.epochs) + + if starting_epoch >= config.epochs: logger.info("Evaluating model without fine-tuning...") + evaluator._reset_stats() evaluator.eval_model(0, model) payload = ('Eval Loss:%.4f\tEval acc: %.2f' % (evaluator.loss_meters.avg, evaluator.acc_meters.avg*100)) logger.info(payload) - return - - else: - logger.info("Starting or continuing training process...") - for epoch in range(config.epochs): - logger.info("") - logger.info("="*20 + "Training Epoch %d" % (epoch) + "="*20) - - # Train - ENV['global_step'] = trainer.train_model(epoch, model, criterion, optimizer) - ENV['train_history'].append(trainer.acc_meters.avg*100) - scheduler.step() - - # Eval - logger.info("="*20 + "Eval Epoch %d" % (epoch) + "="*20) - evaluator.eval_model(epoch, model) - payload = ('Eval Loss:%.4f\tEval acc: %.2f' % (evaluator.loss_meters.avg, evaluator.acc_meters.avg*100)) - logger.info(payload) - ENV['eval_history'].append(evaluator.acc_meters.avg*100) - ENV['curren_acc'] = evaluator.acc_meters.avg*100 - - # Reset Stats - trainer._reset_stats() - evaluator._reset_stats() - - # Save Model - target_model = model.module if args.data_parallel else model - util.save_model(ENV=ENV, - epoch=epoch, - model=target_model, - optimizer=optimizer, - scheduler=scheduler, - filename=model_file) - logger.info('Model Saved at %s', model_file) return + logger.info("Starting or continuing training process...") + for epoch in range(starting_epoch, config.epochs): + logger.info("") + logger.info("="*20 + "Training Epoch %d" % (epoch) + "="*20) + + ENV['global_step'] = trainer.train_model(epoch, model, criterion, optimizer) + ENV['train_history'].append(trainer.acc_meters.avg*100) + scheduler.step() + + logger.info("="*20 + "Eval Epoch %d" % (epoch) + "="*20) + evaluator.eval_model(epoch, model) + payload = ('Eval Loss:%.4f\tEval acc: %.2f' % (evaluator.loss_meters.avg, evaluator.acc_meters.avg*100)) + logger.info(payload) + ENV['eval_history'].append(evaluator.acc_meters.avg*100) + ENV['curren_acc'] = evaluator.acc_meters.avg*100 + + trainer._reset_stats() + evaluator._reset_stats() + + target_model = model.module if args.data_parallel else model + util.save_model(ENV=ENV, + epoch=epoch, + model=target_model, + optimizer=optimizer, + scheduler=scheduler, + filename=model_file) + logger.info('Model Saved at %s', model_file) + return + def check_ssim(lsp_noise, lsp_test_noise, data_loader, num_classes, perturbation_path, device, logger): """ Compute SSIM for LSP-generated perturbations on train and test sets, @@ -368,7 +427,8 @@ def sample_wise_perturbation(args, logger, num_classes, perturbation_path, nois recover_test_data_path=args.recover_test_data_path, recover_test_data_type=args.recover_test_data_type, num_of_workers=args.num_of_workers, - seed=args.seed, no_train_augments=True) + seed=args.seed, no_train_augments=True, + high_resolution=args.high_resolution) data_loader = datasets_generator.getDataLoader(train_shuffle=False, train_drop_last=False) @@ -384,7 +444,12 @@ def sample_wise_perturbation(args, logger, num_classes, perturbation_path, nois idx = 0 for images, _ in ds: for image in images: - patch_cord, _ = noise_generator._patch_noise_extend_to_img(noise[idx], image.shape, args.patch_location) + if image.ndim == 4: + patch_cord, _ = noise_generator._patch_noise_to_3d_img( + noise[idx], image.shape, args.patch_location) + else: + patch_cord, _ = noise_generator._patch_noise_extend_to_img( + noise[idx], image.shape, args.patch_location) cord_list.append(patch_cord) idx += 1 @@ -402,16 +467,21 @@ def sample_wise_perturbation(args, logger, num_classes, perturbation_path, nois idx = 0 for images, labels in loader: images, labels = images.to(device), labels.to(device) + if labels.ndim > 1 and labels.shape[1] == 1: + labels = torch.squeeze(labels, 1).long() batch_fingerprint, start_idx = [], idx for i, image in enumerate(images): - c, h, w = image.shape - mask = torch.zeros((c, h, w), dtype=torch.float32) - x1, x2, y1, y2 = cords[idx] patch = fingerprint[idx] if isinstance(patch, np.ndarray): patch = torch.tensor(patch) - mask[:, x1:x2, y1:y2] = patch + mask = torch.zeros_like(image, device='cpu') + if image.ndim == 4: + x1, x2, y1, y2, z1, z2 = cords[idx] + mask[:, x1:x2, y1:y2, z1:z2] = patch + else: + x1, x2, y1, y2 = cords[idx] + mask[:, x1:x2, y1:y2] = patch batch_fingerprint.append(mask) idx += 1 @@ -430,8 +500,13 @@ def sample_wise_perturbation(args, logger, num_classes, perturbation_path, nois raise ValueError(f"Unknown attack type {args.attack_type}") for i, d in enumerate(delta): - x1, x2, y1, y2 = cords[start_idx + i] - d_patch = d[:, x1:x2, y1:y2] + cord = cords[start_idx + i] + if d.ndim == 4: + x1, x2, y1, y2, z1, z2 = cord + d_patch = d[:, x1:x2, y1:y2, z1:z2] + else: + x1, x2, y1, y2 = cord + d_patch = d[:, x1:x2, y1:y2] fingerprint[start_idx + i] = d_patch.detach().cpu() # --------------------------- @@ -474,7 +549,7 @@ def add_samplewise_perturbation(args, logger, num_classes, perturbation_path, ra } def _ensure_labels_shape(name, labels): - if args.train_data_type in MED2D: + if args.train_data_type in MED2D and labels.ndim > 1 and labels.shape[1] == 1: return torch.squeeze(labels, 1).long() return labels @@ -489,9 +564,14 @@ def _process_split(loader_key, noise_bank, coords, prefix): logger.info(f"Skip {prefix}: noise_bank is None") return 0.0 - _mkdir_split(prefix) + is_3d = len(noise_bank.shape) == 5 + is_multilabel = _is_multilabel_task(getattr(args, 'task_type', None)) + save_as_npz = is_3d or is_multilabel + if not save_as_npz: + _mkdir_split(prefix) idx = 0 ssim_total, num_images = 0.0, 0 + volume_images, volume_labels = [], [] for images, labels in data_loader[loader_key]: images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True) @@ -499,28 +579,54 @@ def _process_split(loader_key, noise_bank, coords, prefix): for j, (image, label) in enumerate(zip(images, labels)): # prepare patch-only noise mask - c, h, w = image.shape sample_noise = torch.tensor(noise_bank[idx]).to(device) if not torch.is_tensor(noise_bank) else noise_bank[idx].to(device) - x1, x2, y1, y2 = coords[idx] - mask = np.zeros((c, h, w), np.float32) - mask[:, x1:x2, y1:y2] = sample_noise.detach().cpu().numpy() - sample_noise = torch.from_numpy(mask).to(device) + mask = torch.zeros_like(image) + if image.ndim == 4: + x1, x2, y1, y2, z1, z2 = coords[idx] + mask[:, x1:x2, y1:y2, z1:z2] = sample_noise + else: + x1, x2, y1, y2 = coords[idx] + mask[:, x1:x2, y1:y2] = sample_noise + sample_noise = mask # apply noise - images[j] = images[j] + sample_noise + original = images[j].clone() + images[j] = torch.clamp(images[j] + sample_noise, 0, 1) # save - gt = label.item() - save_image(images[j], os.path.join(perturbation_path, '..', f'{prefix}', str(gt), f'{j}_{idx}.png')) - save_image(sample_noise, os.path.join(perturbation_path, '..', f'{prefix}_noisy', str(gt), f'{j}_{idx}.png')) - - # compute SSIM (original image ≈ images[j] - sample_noise) - img_np = (images[j] - sample_noise).squeeze(0).permute(1, 2, 0).detach().cpu().numpy() - pert_np = images[j].squeeze(0).permute(1, 2, 0).detach().cpu().numpy() - ssim_total += ssim(img_np, pert_np, channel_axis=2, data_range=1) + if save_as_npz: + volume_images.append(images[j].detach().cpu().numpy().astype(np.float32)) + if is_multilabel: + volume_labels.append(label.detach().cpu().numpy().astype(np.int64)) + else: + volume_labels.append(label.item()) + else: + gt = label.item() + save_image(images[j], os.path.join(perturbation_path, '..', f'{prefix}', str(gt), f'{j}_{idx}.png')) + save_image(sample_noise, os.path.join(perturbation_path, '..', f'{prefix}_noisy', str(gt), f'{j}_{idx}.png')) + + # Compute SSIM on the full volume for 3D and channel-last images for 2D. + if is_3d: + img_np = original.squeeze(0).detach().cpu().numpy() + pert_np = images[j].squeeze(0).detach().cpu().numpy() + ssim_total += ssim(img_np, pert_np, data_range=1) + else: + img_np = original.permute(1, 2, 0).detach().cpu().numpy() + pert_np = images[j].permute(1, 2, 0).detach().cpu().numpy() + ssim_total += ssim(img_np, pert_np, channel_axis=2, data_range=1) num_images += 1 idx += 1 + if save_as_npz: + output_dir = Path(perturbation_path).parent / prefix + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / 'perturb_data.npz' + if is_3d: + save_cooked_volume_npz(output_path, volume_images, volume_labels) + else: + save_cooked_array_npz(output_path, volume_images, volume_labels) + logger.info('Cooked NPZ split saved at %s', output_path) + avg = ssim_total / max(1, num_images) logger.info(f'Average {prefix} SSIM: {avg:.4f}') return avg @@ -550,7 +656,8 @@ def sample_wise_perturbation_min_min(args, logger, num_classes, perturbation_pat recover_test_data_path=args.recover_test_data_path, recover_test_data_type=args.recover_test_data_type, num_of_workers=args.num_of_workers, - seed=args.seed, no_train_augments=True) + seed=args.seed, no_train_augments=True, + high_resolution=args.high_resolution) data_loader = datasets_generator.getDataLoader(train_shuffle=False, train_drop_last=False) @@ -649,7 +756,7 @@ def sample_wise_perturbation_min_min(args, logger, num_classes, perturbation_pat elif args.attack_type == 'EM-Pseudo': perturb_img, eta = noise_generator.datacook_v1(images, labels, model, optimizer, criterion, random_noise=batch_noise) else: - raise('Invalid attack') + raise ValueError('Invalid attack') for i, delta in enumerate(eta): x1, x2, y1, y2 = mask_cord_list[batch_start_idx+i] @@ -692,7 +799,7 @@ def sample_wise_perturbation_min_min(args, logger, num_classes, perturbation_pat elif args.attack_type == 'EM-Pseudo': perturb_img, eta = noise_generator.datacook_v1(images, labels, model, optimizer, criterion, random_noise=batch_test_noise) else: - raise('Invalid attack') + raise ValueError('Invalid attack') for i, delta in enumerate(eta): x1, x2, y1, y2 = test_mask_cord_list[batch_test_start_idx+i] @@ -717,7 +824,7 @@ def sample_wise_perturbation_min_min(args, logger, num_classes, perturbation_pat elif args.attack_type == 'EM-Pseudo': condition = (number_attack <= 5) and (error_rate > args.universal_stop_error) - add_samplewise_perturbation(args, logger, num_classes, perturbation_path, random_noise, random_test_noise, data_loader, device, eval_target='train_dataset', mask_cord_list=mask_cord_list) + add_samplewise_perturbation(args, logger, num_classes, perturbation_path, random_noise, random_test_noise, data_loader, device, mask_cord_list=mask_cord_list, test_mask_cord_list=test_mask_cord_list) return random_noise, random_test_noise @@ -728,20 +835,14 @@ def samplewise_perturbation_eval(args, random_noise, random_test_noise, data_loa model = model.to(device) idx = 0 - # Datasets that need label squeeze from (B,1) -> (B,) - MED2D = { - 'PathMNIST','DermaMNIST','OCTMNIST','PneumoniaMNIST','RetinaMNIST', - 'BreastMNIST','BloodMNIST','TissueMNIST','OrganAMNIST','OrganCMNIST','OrganSMNIST' - } - def _ensure_labels_shape(name, labels): - if args.train_data_type in MED2D: + if labels.ndim > 1 and labels.shape[1] == 1: return torch.squeeze(labels, 1).long() return labels for i, (images, labels) in enumerate(data_loader[eval_target]): images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True) - labels = _ensure_labels_shape(loader_key, labels).to(device) + labels = _ensure_labels_shape(eval_target, labels).to(device) if random_noise is not None: for j, (image, label) in enumerate(zip(images, labels)): @@ -749,17 +850,30 @@ def _ensure_labels_shape(name, labels): sample_noise = torch.tensor(random_noise[idx]).to(device) else: sample_noise = random_noise[idx].to(device) - c, h, w = image.shape[0], image.shape[1], image.shape[2] - mask = np.zeros((c, h, w), np.float32) - x1, x2, y1, y2 = mask_cord_list[idx] - mask[:, x1: x2, y1: y2] = sample_noise.cpu().numpy() - sample_noise = torch.from_numpy(mask).to(device) + mask = torch.zeros_like(image) + if image.ndim == 4: + x1, x2, y1, y2, z1, z2 = mask_cord_list[idx] + mask[:, x1:x2, y1:y2, z1:z2] = sample_noise + else: + x1, x2, y1, y2 = mask_cord_list[idx] + mask[:, x1:x2, y1:y2] = sample_noise + sample_noise = mask images[j] = images[j] + sample_noise idx += 1 pred = model(images) - err = (pred.data.max(1)[1] != labels.data).float().sum() - loss = torch.nn.CrossEntropyLoss()(pred, labels) + if _is_multilabel_task(getattr(args, 'task_type', None)): + targets = labels.float() + if pred.shape != targets.shape: + raise ValueError( + f'Multi-label target/logit shapes must match; got {targets.shape} and {pred.shape}.' + ) + predicted = torch.sigmoid(pred) >= 0.5 + err = (~predicted.eq(targets.bool()).all(dim=1)).float().sum() + loss = torch.nn.BCEWithLogitsLoss()(pred, targets) + else: + err = (pred.data.max(1)[1] != labels.data).float().sum() + loss = torch.nn.CrossEntropyLoss()(pred, labels) loss_meter.update(loss.item(), len(labels)) err_meter.update(err / len(labels)) @@ -776,6 +890,11 @@ def main(): # Load configuration config_file = os.path.join(args.config_path, args.version + '.yaml') config = mlconfig.load(config_file) + config_task = getattr(config, 'task_type', None) + if args.task_type == 'multilabel' and not _is_multilabel_task(config_task): + raise ValueError( + f'{args.train_data_type} is multi-label but {config_file} does not set task_type: multilabel.' + ) config.set_immutable() shutil.copyfile(config_file, os.path.join(exp_path, args.version + '.yaml')) @@ -788,7 +907,8 @@ def main(): recover_test_data_path=args.recover_test_data_path, recover_test_data_type=args.recover_test_data_type, num_of_workers=args.num_of_workers, - seed=args.seed) + seed=args.seed, + high_resolution=args.high_resolution) data_loader = datasets_generator.getDataLoader(train_shuffle=False, train_drop_last=False) @@ -815,6 +935,7 @@ def main(): model = torch.nn.DataParallel(model) if args.load_model: + checkpoint_path_file = os.path.join(ckpt_path, f"{args.version}_{config.epochs}.pth") checkpoint = util.load_model(filename=checkpoint_path_file, model=model, optimizer=optimizer, @@ -851,16 +972,23 @@ def main(): elif args.attack_type in ['datacook', 'datacook_adv', 'Adv']: train(args, config, logger, model, optimizer, scheduler, criterion, trainer, evaluator, ENV, ckpt_path) + evaluator._reset_stats() evaluator.eval_model(0, model) payload = ('Eval Loss:%.4f\tEval acc: %.2f' % (evaluator.loss_meters.avg, evaluator.acc_meters.avg*100)) logger.info(payload) + if args.attack_type in ['datacook', 'datacook_adv']: + save_pseudo_label_artifacts(model, data_loader, exp_path, device, logger) + if args.export_predictions_dir: + evaluator.save_predictions(os.path.join(args.export_predictions_dir, 'clean.npz')) noise,test_noise = sample_wise_perturbation(args, logger, num_classes, fingerprint_path, noise_generator, trainer, evaluator, model, criterion, optimizer, scheduler, random_noise,random_test_noise, device, ENV) elif args.attack_type == 'EM' or args.attack_type == 'EM-Pseudo': + if args.attack_type == 'EM-Pseudo': + save_pseudo_label_artifacts(model, data_loader, exp_path, device, logger) noise,test_noise = sample_wise_perturbation_min_min(args, logger, num_classes, fingerprint_path, noise_generator, trainer, evaluator, model, criterion, optimizer, scheduler, random_noise,random_test_noise, device, ENV) else: - raise('Not implemented yet') + raise NotImplementedError(f'Attack type {args.attack_type!r} is not implemented') torch.save(noise, os.path.join(args.exp_name, 'perturbation.pt')) logger.info(noise) @@ -877,4 +1005,4 @@ def main(): start = time.time() main() end = time.time() - print(f"Execution time: {(end - start) / 60:.2f} minutes") \ No newline at end of file + print(f"Execution time: {(end - start) / 60:.2f} minutes") diff --git a/prepare_revision_posthoc.py b/prepare_revision_posthoc.py new file mode 100755 index 0000000..6921290 --- /dev/null +++ b/prepare_revision_posthoc.py @@ -0,0 +1,910 @@ +#!/usr/bin/env python3 +"""Incrementally prepare post-hoc artifacts/configs from completed pipeline runs. + +The script never trains a model and never modifies files inside an experiment +run. It reads existing prediction, pseudo-label, checkpoint and ``ae_test`` +artifacts, exports paired NumPy arrays under ``artifacts/``, and writes generated +Task 3/5/7/8 configs. Existing valid exports are reused unless ``--force`` is +passed. +""" + +from __future__ import annotations + +import argparse +import csv +import logging +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np +import yaml +from PIL import Image + +from run_datacook_suite import load_pipeline_configs + + +PROJECT_ROOT = Path(__file__).resolve().parent +POSTHOC_TASKS = { + "pseudo", + "runtime", + "image_quality", + "attacks", + "finetune_attacks", + "finetune_budget_attacks", +} +RAW_VALIDATION_TASKS = {"finetune_attacks", "finetune_budget_attacks"} + + +@dataclass(frozen=True) +class RunSpec: + dataset: str + modality: str + task_type: str + num_classes: int + surrogate: str + victim: str + method: str + seed: int + epsilon_pixel: float + epsilon: float + run_name: str + run_dir: Path + model_config: Path + data_path: Path + test_count: int + high_resolution: bool = False + + @property + def predictions(self) -> dict[str, Path]: + root = self.run_dir / "predictions" + return { + "clean": root / "clean.npz", + "authorized_cooked": root / "authorized_cooked.npz", + "unauthorized_raw": root / "unauthorized_raw.npz", + } + + @property + def pseudo_labels(self) -> Path: + return self.run_dir / self.surrogate / "pseudo_labels_train_dataset.npz" + + @property + def cooked_dir(self) -> Path: + return self.run_dir / self.surrogate / "ae_test" + + @property + def test_perturbation(self) -> Path: + return self.run_dir / "test_perturbation.pt" + + @property + def checkpoint(self) -> Path: + return self.run_dir / "protected" / self.victim / "checkpoints" / f"{self.victim}.pth" + + def metadata(self) -> dict[str, Any]: + return { + "dataset": self.dataset, + "modality": self.modality, + "task_type": self.task_type, + "method": self.method, + "surrogate_model": self.surrogate, + "victim_model": self.victim, + "seed": self.seed, + "epsilon": self.epsilon, + } + + +def _resolve(value: str | Path, base: Path = PROJECT_ROOT) -> Path: + path = Path(value).expanduser() + return path.resolve() if path.is_absolute() else (base / path).resolve() + + +def _slug(value: Any) -> str: + return re.sub(r"[^A-Za-z0-9_.=-]+", "-", str(value)).strip("-") + + +def _run_name(dataset: str, method: str, surrogate: str, victim: str, seed: int, epsilon: float) -> str: + return _slug( + f"{dataset}-{method}-sur={surrogate}-vic={victim}-seed={seed}-eps={epsilon:g}" + ) + + +def load_run_specs( + suite_path: Path, + selected_datasets: set[str] | None = None, + selected_seeds: set[int] | None = None, + selected_epsilons: set[float] | None = None, +) -> list[RunSpec]: + """Expand the runs produced by a suite invocation. + + ``run_datacook_suite.py --seeds/--epsilons`` overrides the values stored in + each pipeline YAML. Mirror that behavior here so post-hoc preparation can + discover runs created with command-line overrides (for example seeds 3-7 + when the base YAML contains seeds 0-2). + """ + specs: list[RunSpec] = [] + for pipeline_path in load_pipeline_configs(suite_path): + config = yaml.safe_load(pipeline_path.read_text(encoding="utf-8")) or {} + dataset_cfg = config["dataset"] + dataset_name = str(dataset_cfg["name"]) + if selected_datasets and dataset_name not in selected_datasets: + continue + models = config["models"] + experiment = config.get("experiment", {}) + surrogate = str(models.get("surrogate", "resnet18")) + victim = str(models.get("victim", surrogate)) + method = str(experiment.get("attack_type", "datacook")) + scale = str(experiment.get("epsilon_scale", "pixel")) + experiment_root = _resolve(config.get("experiment_root", "experiments/revision_pipeline")) + config_path = _resolve(dataset_cfg["config_path"]) + data_path = _resolve(dataset_cfg.get("data_path", "../datasets")) + test_shape = dataset_cfg.get("test_noise_shape") or [] + if not test_shape: + raise ValueError(f"{pipeline_path} has no dataset.test_noise_shape.") + seed_values = ( + sorted(selected_seeds) + if selected_seeds is not None + else config.get("seeds", [0]) + ) + epsilon_values = ( + sorted(selected_epsilons) + if selected_epsilons is not None + else config.get("epsilons", [experiment.get("epsilon", 8)]) + ) + for seed in seed_values: + for epsilon_pixel in epsilon_values: + epsilon_pixel = float(epsilon_pixel) + normalized = epsilon_pixel / 255.0 if scale == "pixel" else epsilon_pixel + run_name = _run_name( + dataset_name, method, surrogate, victim, int(seed), epsilon_pixel + ) + specs.append(RunSpec( + dataset=dataset_name, + modality=str(dataset_cfg.get("modality", "2d_image")), + task_type=str(dataset_cfg.get("task_type", "multiclass")), + num_classes=int(dataset_cfg["num_classes"]), + surrogate=surrogate, + victim=victim, + method=method, + seed=int(seed), + epsilon_pixel=epsilon_pixel, + epsilon=float(normalized), + run_name=run_name, + run_dir=experiment_root / run_name, + model_config=config_path / f"{victim}.yaml", + data_path=data_path, + test_count=int(test_shape[0]), + high_resolution=bool(dataset_cfg.get("high_resolution", False)), + )) + if not specs: + raise ValueError("No runs matched the suite configuration and command-line overrides.") + return specs + + +def _load_official_split(spec: RunSpec, split: str) -> tuple[np.ndarray, np.ndarray]: + """Load an official MedMNIST split with the same preprocessing as dataset.py.""" + import medmnist + from medmnist import INFO + + gray_2d = { + "OrganAMNIST", "OrganCMNIST", "OrganSMNIST", "OCTMNIST", + "PneumoniaMNIST", "BreastMNIST", "TissueMNIST", "ChestMNIST", + } + info = INFO[spec.dataset.lower()] + data_class = getattr(medmnist, info["python_class"]) + # dataset.py intentionally uses MedMNIST's default root for official data; + # keeping the same constructor here guarantees identical split ordering. + if spec.modality == "3d_volume": + from util import Transform3D + + transform = Transform3D() + as_rgb = False + else: + from torchvision import transforms + + transform = transforms.Compose([transforms.ToTensor()]) + as_rgb = spec.dataset in gray_2d + dataset_kwargs = { + "split": split, + "transform": transform, + "download": True, + "as_rgb": as_rgb, + } + if spec.high_resolution: + dataset_kwargs["size"] = 224 + dataset = data_class( + **dataset_kwargs, + ) + images, labels = [], [] + for image, label in dataset: + array = image.detach().cpu().numpy() if hasattr(image, "detach") else np.asarray(image) + images.append(np.asarray(array, dtype=np.float32)) + label_array = np.asarray(label) + if spec.task_type.replace("-", "").replace("_", "").lower() == "multilabel": + labels.append(label_array.reshape(-1).astype(np.int64)) + else: + labels.append(int(label_array.reshape(-1)[0])) + stacked = np.stack(images) + targets = np.asarray(labels, dtype=np.int64) + if split == "test" and len(stacked) != spec.test_count: + raise ValueError( + f"{spec.dataset} test count mismatch: config={spec.test_count}, loaded={len(stacked)}." + ) + return stacked, targets + + +def _load_official_test_set(spec: RunSpec) -> tuple[np.ndarray, np.ndarray]: + """Backward-compatible wrapper for the official test split.""" + return _load_official_split(spec, "test") + + +def _atomic_npz(path: Path, **arrays: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp.npz") + np.savez_compressed(temporary, **arrays) + temporary.replace(path) + + +def _validate_npz(path: Path, expected_count: int, required_keys: tuple[str, ...]) -> bool: + if not path.is_file(): + return False + try: + with np.load(path, allow_pickle=False) as archive: + return all(key in archive and len(archive[key]) == expected_count for key in required_keys) + except (OSError, ValueError, KeyError): + return False + + +def _validate_xy_npz(path: Path, expected_count: int | None = None) -> bool: + """Return whether an NPZ contains a non-empty, aligned images/labels pair.""" + if not path.is_file(): + return False + try: + with np.load(path, allow_pickle=False) as archive: + if "images" not in archive or "labels" not in archive: + return False + count = len(archive["images"]) + return count > 0 and len(archive["labels"]) == count and ( + expected_count is None or count == expected_count + ) + except (OSError, ValueError, KeyError): + return False + + +def export_raw_test( + spec: RunSpec, + output: Path, + *, + force: bool, + logger: logging.Logger, +) -> tuple[np.ndarray, np.ndarray]: + if output.is_file() and not force: + if not _validate_npz(output, spec.test_count, ("images", "labels")): + raise ValueError(f"Existing raw export is invalid; rerun with --force: {output}") + with np.load(output, allow_pickle=False) as archive: + return np.asarray(archive["images"]), np.asarray(archive["labels"]) + logger.info("Loading official %s test split.", spec.dataset) + images, labels = _load_official_test_set(spec) + _atomic_npz(output, images=images, labels=labels) + logger.info("Exported raw test artifact: %s", output) + return images, labels + + +def export_raw_validation( + spec: RunSpec, + output: Path, + *, + force: bool, + logger: logging.Logger, +) -> tuple[np.ndarray, np.ndarray]: + """Export the official validation split used by adaptive fine-tuning attacks.""" + if output.is_file() and not force: + if not _validate_xy_npz(output): + raise ValueError(f"Existing validation export is invalid; rerun with --force: {output}") + with np.load(output, allow_pickle=False) as archive: + return np.asarray(archive["images"]), np.asarray(archive["labels"]) + logger.info("Loading official %s validation split.", spec.dataset) + images, labels = _load_official_split(spec, "val") + _atomic_npz(output, images=images, labels=labels) + logger.info("Exported raw validation artifact: %s", output) + return images, labels + + +def _cooked_index(path: Path) -> int: + suffix = path.stem.rsplit("_", 1)[-1] + if not suffix.isdigit(): + raise ValueError(f"Cooked filename does not end in a global sample index: {path}") + return int(suffix) + + +def export_cooked_test( + cooked_dir: Path, + raw_images: np.ndarray, + raw_labels: np.ndarray, + output: Path, + *, + force: bool, + logger: logging.Logger, + perturbation_path: Path | None = None, +) -> None: + count = len(raw_images) + if output.is_file() and not force: + if not _validate_npz(output, count, ("images", "labels")): + raise ValueError(f"Existing cooked export is invalid; rerun with --force: {output}") + logger.info("Reusing cooked test artifact: %s", output) + return + cooked_npz = cooked_dir / "perturb_data.npz" + if cooked_npz.is_file(): + with np.load(cooked_npz, allow_pickle=False) as archive: + if "perturb_images" not in archive or "perturb_labels" not in archive: + raise ValueError( + f"Cooked NPZ must contain perturb_images and perturb_labels: {cooked_npz}" + ) + cooked_images = np.asarray(archive["perturb_images"], dtype=np.float32) + cooked_labels = np.asarray(archive["perturb_labels"]) + if len(cooked_images) != count or cooked_images.shape != raw_images.shape: + raise ValueError( + f"Cooked/raw shape mismatch for {cooked_npz}: " + f"{cooked_images.shape} != {raw_images.shape}." + ) + if cooked_labels.shape != raw_labels.shape or not np.array_equal(cooked_labels, raw_labels): + raise ValueError( + f"Cooked/raw labels differ for {cooked_npz}: " + f"{cooked_labels.shape} != {raw_labels.shape}." + ) + _atomic_npz(output, images=cooked_images, labels=cooked_labels) + logger.info("Exported cooked NPZ artifact: %s", output) + return + paths = [path for path in cooked_dir.glob("*/*") if path.is_file() and path.suffix.lower() == ".png"] + if len(paths) != count: + if perturbation_path is None or not perturbation_path.is_file(): + raise ValueError( + f"Expected {count} cooked PNGs in {cooked_dir}, found {len(paths)}, " + "and no test perturbation is available for recovery." + ) + logger.warning( + "Expected %d cooked PNGs in %s, found %d; reconstructing from %s.", + count, + cooked_dir, + len(paths), + perturbation_path, + ) + import torch + + try: + perturbation = torch.load( + perturbation_path, map_location="cpu", weights_only=False + ) + except TypeError: + perturbation = torch.load(perturbation_path, map_location="cpu") + if torch.is_tensor(perturbation): + perturbation = perturbation.detach().cpu().numpy() + perturbation = np.asarray(perturbation, dtype=np.float32) + if perturbation.shape != raw_images.shape: + raise ValueError( + f"Cannot reconstruct cooked test images: perturbation shape " + f"{perturbation.shape} != raw shape {raw_images.shape}." + ) + cooked_float = np.clip( + np.asarray(raw_images, dtype=np.float32) + perturbation, 0.0, 1.0 + ) + # torchvision.save_image uses x*255 + 0.5 followed by uint8 + # conversion. Reproduce that quantization so recovered arrays match + # the PNG inputs used by the protected-model evaluation. + cooked_images = ( + np.floor(cooked_float * 255.0 + 0.5) + .clip(0, 255) + .astype(np.uint8) + .astype(np.float32) + / 255.0 + ) + _atomic_npz(output, images=cooked_images, labels=raw_labels) + logger.info("Recovered cooked test artifact: %s", output) + return + ordered: list[Path | None] = [None] * count + for path in paths: + index = _cooked_index(path) + if index >= count: + raise ValueError(f"Cooked sample index {index} is outside [0,{count}) in {path}.") + if ordered[index] is not None: + raise ValueError(f"Duplicate cooked sample index {index}: {ordered[index]} and {path}.") + ordered[index] = path + folder_label = int(path.parent.name) + if folder_label != int(raw_labels[index]): + raise ValueError( + f"Cooked/raw label mismatch at sample {index}: {folder_label} != {raw_labels[index]}." + ) + missing = [index for index, path in enumerate(ordered) if path is None] + if missing: + raise ValueError(f"Cooked samples are missing global indices: {missing[:20]}") + cooked = [] + for path in ordered: + assert path is not None + image = np.asarray(Image.open(path).convert("RGB"), dtype=np.float32) / 255.0 + cooked.append(np.transpose(image, (2, 0, 1))) + cooked_images = np.stack(cooked).astype(np.float32, copy=False) + if cooked_images.shape != raw_images.shape: + raise ValueError( + f"Cooked/raw shape mismatch for {cooked_dir}: {cooked_images.shape} != {raw_images.shape}." + ) + _atomic_npz(output, images=cooked_images, labels=raw_labels) + logger.info("Exported cooked test artifact: %s", output) + + +def _relative(path: Path, config_dir: Path) -> str: + return Path(os.path.relpath(path, config_dir)).as_posix() + + +def _prediction_record(spec: RunSpec, config_dir: Path) -> dict[str, Any]: + return { + "metadata": spec.metadata(), + "num_classes": spec.num_classes, + "predictions": { + key: _relative(path, config_dir) for key, path in spec.predictions.items() + }, + } + + +def _write_yaml(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + temporary.replace(path) + + +def _write_manifest(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def generate_configs( + specs: list[RunSpec], + artifact_root: Path, + config_dir: Path, + *, + attacks_epsilon: float, + runtime_seed: int, + runtime_epsilon: float, + output_suffix: str = "", + finetune_epochs: tuple[int, ...] = (5, 10), + budget_sizes: tuple[int, ...] = (25, 50), + tasks: set[str] | None = None, + result_root: str | Path = "results/revision_pipeline", +) -> dict[str, int]: + selected_tasks = set(POSTHOC_TASKS if tasks is None else tasks) + unknown_tasks = selected_tasks - POSTHOC_TASKS + if unknown_tasks: + raise ValueError(f"Unknown post-hoc tasks: {sorted(unknown_tasks)}") + suffix = f"_{_slug(output_suffix)}" if output_suffix else "" + result_root = Path(result_root).as_posix().rstrip("/") + if not finetune_epochs or any(epoch <= 0 for epoch in finetune_epochs): + raise ValueError("finetune_epochs must contain positive integers.") + if not budget_sizes or any(size <= 0 for size in budget_sizes): + raise ValueError("budget_sizes must contain positive integers.") + attack_output = f"{result_root}/posthoc/adaptive_attack_results{suffix}.csv" + finetune_output = f"{result_root}/posthoc/adaptive_finetune_results{suffix}.csv" + combined_attack_output = f"{result_root}/posthoc/adaptive_attack_results_all{suffix}.csv" + budget_output = f"{result_root}/posthoc/adaptive_finetune_budget_results{suffix}.csv" + pseudo_samples_output = f"{result_root}/posthoc/pseudo_label_samples{suffix}.csv" + pseudo_quality_output = f"{result_root}/posthoc/pseudo_label_quality{suffix}.csv" + image_quality_output = f"{result_root}/posthoc/quality_metrics{suffix}.csv" + runtime_output = f"{result_root}/posthoc/runtime_summary{suffix}.csv" + pseudo_records, quality_records, runtime_records = [], [], [] + attack_records, finetune_records, finetune_budget_records = [], [], [] + for spec in specs: + raw_path = artifact_root / spec.dataset / "raw_test.npz" + raw_val_path = artifact_root / spec.dataset / "raw_val.npz" + cooked_path = artifact_root / spec.run_name / "cooked_test.npz" + predictions_ready = all(path.is_file() for path in spec.predictions.values()) + base = _prediction_record(spec, config_dir) + if ( + "pseudo" in selected_tasks + and predictions_ready + and spec.pseudo_labels.is_file() + ): + pseudo_records.append({ + **base, + "pseudo_labels": _relative(spec.pseudo_labels, config_dir), + }) + if _validate_npz(raw_path, spec.test_count, ("images", "labels")) and _validate_npz( + cooked_path, spec.test_count, ("images", "labels") + ): + if "image_quality" in selected_tasks: + quality_records.append({ + "metadata": spec.metadata(), + "images": { + "raw": _relative(raw_path, config_dir) + "::images", + "cooked": _relative(cooked_path, config_dir) + "::images", + }, + "compute_lpips": spec.modality != "3d_volume", + }) + if ( + "runtime" in selected_tasks + and + spec.seed == runtime_seed + and np.isclose(spec.epsilon_pixel, runtime_epsilon) + and spec.checkpoint.is_file() + and spec.model_config.is_file() + and spec.test_perturbation.is_file() + ): + runtime_records.append({ + "metadata": spec.metadata(), + "model": { + "config_path": _relative(spec.model_config, config_dir), + "checkpoint": _relative(spec.checkpoint, config_dir), + }, + "data": { + "raw_inputs": _relative(raw_path, config_dir) + "::images", + "cooked_inputs": _relative(cooked_path, config_dir) + "::images", + }, + "cooker": { + "perturbation": _relative(spec.test_perturbation, config_dir), + "clamp": [0, 1], + }, + "batch_size": 16 if ( + spec.high_resolution or spec.modality == "3d_volume" + ) else 128, + "repeats": 5, + "warmup": 1, + }) + if ( + selected_tasks & { + "attacks", "finetune_attacks", "finetune_budget_attacks" + } + and + predictions_ready + and _validate_npz(raw_path, spec.test_count, ("images", "labels")) + and np.isclose(spec.epsilon_pixel, attacks_epsilon) + and spec.checkpoint.is_file() + and spec.model_config.is_file() + ): + if "attacks" in selected_tasks: + attack_records.append({ + **base, + "model": { + "config_path": _relative(spec.model_config, config_dir), + "checkpoint": _relative(spec.checkpoint, config_dir), + }, + "eval_data": { + "inputs": _relative(raw_path, config_dir) + "::images", + "labels": _relative(raw_path, config_dir) + "::labels", + }, + "attacks": [ + {"type": "jpeg", "strength": quality} for quality in (95, 75, 50) + ] + [ + {"type": "gaussian_smoothing", "strength": {"sigma": sigma, "kernel_size": 5}} + for sigma in (0.5, 1.0, 2.0) + ] + [ + {"type": "strong_augmentation", "strength": strength} + for strength in (1, 2, 3) + ] + [ + {"type": "median_filter", "strength": kernel} for kernel in (3, 5) + ], + }) + if ( + selected_tasks & {"finetune_attacks", "finetune_budget_attacks"} + and _validate_xy_npz(raw_val_path) + ): + with np.load(raw_val_path, allow_pickle=False) as archive: + raw_val_count = len(archive["labels"]) + raw_subset = { + "inputs": _relative(raw_val_path, config_dir) + "::images", + "labels": _relative(raw_val_path, config_dir) + "::labels", + } + eval_data = { + "inputs": _relative(raw_path, config_dir) + "::images", + "labels": _relative(raw_path, config_dir) + "::labels", + } + if "finetune_attacks" in selected_tasks: + finetune_records.append({ + **base, + "model": { + "config_path": _relative(spec.model_config, config_dir), + "checkpoint": _relative(spec.checkpoint, config_dir), + }, + "eval_data": eval_data, + "attacks": [ + { + "type": "finetune", + "epochs": epochs, + "learning_rate": 1.0e-4, + "batch_size": 16 if spec.high_resolution else 64, + "raw_subset": raw_subset, + "eval_data": eval_data, + } + for epochs in finetune_epochs + ] + [ + { + "type": "robust_finetune", + "epochs": epochs, + "learning_rate": 1.0e-4, + "batch_size": 16 if spec.high_resolution else 64, + "augmentation_strength": 2, + "raw_subset": raw_subset, + "eval_data": eval_data, + } + for epochs in finetune_epochs + ], + }) + budget_attacks = [ + { + "type": attack_type, + "epochs": 10, + "learning_rate": 1.0e-4, + "batch_size": 16 if spec.high_resolution else 64, + "subset_size": subset_size, + "raw_subset": raw_subset, + "eval_data": eval_data, + **({"augmentation_strength": 2} if attack_type == "robust_finetune" else {}), + } + for subset_size in budget_sizes + if subset_size <= raw_val_count + for attack_type in ("finetune", "robust_finetune") + ] + if ( + "finetune_budget_attacks" in selected_tasks + and budget_attacks + ): + finetune_budget_records.append({ + **base, + "model": { + "config_path": _relative(spec.model_config, config_dir), + "checkpoint": _relative(spec.checkpoint, config_dir), + }, + "eval_data": eval_data, + "attacks": budget_attacks, + }) + + full_subset_sizes: dict[str, int] = {} + for spec in specs: + raw_val_path = artifact_root / spec.dataset / "raw_val.npz" + if spec.dataset not in full_subset_sizes and _validate_xy_npz(raw_val_path): + with np.load(raw_val_path, allow_pickle=False) as archive: + full_subset_sizes[spec.dataset] = len(archive["labels"]) + + payloads = { + "pseudo.yaml": { + "experiments": pseudo_records, + "outputs": { + "samples": pseudo_samples_output, + "quality": pseudo_quality_output, + }, + }, + "image_quality.yaml": { + "experiments": quality_records, + "output": image_quality_output, + }, + "runtime.yaml": { + "experiments": runtime_records, + "include_average": True, + "output": runtime_output, + }, + "attacks.yaml": { + "experiments": attack_records, + "output": attack_output, + }, + "attacks_finetune.yaml": { + "experiments": finetune_records, + "output": finetune_output, + "merge_with": attack_output, + "combined_output": combined_attack_output, + }, + "attacks_finetune_budget.yaml": { + "experiments": finetune_budget_records, + "output": budget_output, + }, + "final_analysis.yaml": { + "inputs": { + "cp_pp": _relative(PROJECT_ROOT / result_root / "summary/cp_pp_summary.csv", config_dir), + "quality": _relative(PROJECT_ROOT / image_quality_output, config_dir), + "attacks": _relative(PROJECT_ROOT / combined_attack_output, config_dir), + "budget_attacks": _relative(PROJECT_ROOT / budget_output, config_dir), + "runtime": _relative(PROJECT_ROOT / runtime_output, config_dir), + "pseudo": _relative(PROJECT_ROOT / pseudo_quality_output, config_dir), + }, + "full_subset_sizes": full_subset_sizes, + "output_dir": f"{result_root}/final{suffix}", + }, + } + for filename, payload in payloads.items(): + _write_yaml(config_dir / filename, payload) + + master_path = config_dir.parent / "generated.yaml" + relative_config_dir = Path(os.path.relpath(config_dir, master_path.parent)).as_posix() + _write_yaml(master_path, { + "tasks": { + "metrics": {"enabled": False, "config": "metrics.yaml"}, + "attacks": {"enabled": bool(attack_records), "config": f"{relative_config_dir}/attacks.yaml"}, + "finetune_attacks": {"enabled": False, "config": f"{relative_config_dir}/attacks_finetune.yaml"}, + "finetune_budget_attacks": {"enabled": False, "config": f"{relative_config_dir}/attacks_finetune_budget.yaml"}, + "epsilon": {"enabled": False, "config": "epsilon.yaml"}, + "pseudo": {"enabled": bool(pseudo_records), "config": f"{relative_config_dir}/pseudo.yaml"}, + "statistics": {"enabled": False, "config": "statistics.yaml"}, + "runtime": {"enabled": bool(runtime_records), "config": f"{relative_config_dir}/runtime.yaml"}, + "image_quality": {"enabled": bool(quality_records), "config": f"{relative_config_dir}/image_quality.yaml"}, + "final_analysis": {"enabled": False, "config": f"{relative_config_dir}/final_analysis.yaml"}, + } + }) + return { + "pseudo": len(pseudo_records), + "image_quality": len(quality_records), + "runtime": len(runtime_records), + "attacks": len(attack_records), + "finetune_attacks": len(finetune_records), + "finetune_budget_attacks": len(finetune_budget_records), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite-config", default="configs/pipelines/medmnist_three.yaml") + parser.add_argument("--artifact-root", default="artifacts/revision_pipeline") + parser.add_argument("--result-root", default="results/revision_pipeline") + parser.add_argument("--config-dir", default="configs/revision/generated") + parser.add_argument( + "--tasks", + nargs="+", + choices=sorted(POSTHOC_TASKS), + help=( + "Prepare only the selected post-hoc tasks. By default all task " + "configs are prepared for backward compatibility." + ), + ) + parser.add_argument("--datasets", nargs="+", help="Optional dataset-name filter.") + parser.add_argument( + "--seeds", nargs="+", type=int, + help="Override the seed list in every selected pipeline, matching run_datacook_suite.py.", + ) + parser.add_argument( + "--epsilons", nargs="+", type=float, + help="Override the pixel-scale epsilon list in every selected pipeline.", + ) + parser.add_argument("--attacks-epsilon", type=float, default=8.0) + parser.add_argument("--runtime-seed", type=int, default=0) + parser.add_argument("--runtime-epsilon", type=float, default=8.0) + parser.add_argument("--output-suffix", default="", help="Append a safe suffix to generated post-hoc result filenames.") + parser.add_argument("--finetune-epochs", nargs="+", type=int, default=[5, 10]) + parser.add_argument("--budget-sizes", nargs="+", type=int, default=[25, 50]) + parser.add_argument("--force", action="store_true", help="Re-export derived NPZ files only; training artifacts remain read-only.") + parser.add_argument("--dry-run", action="store_true", help="Inventory expected runs without writing or loading datasets.") + parser.add_argument("--strict", action="store_true", help="Fail if any expected primary artifact is missing.") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s | %(levelname)s | %(message)s", + ) + logger = logging.getLogger("revision-posthoc") + suite_path = _resolve(args.suite_config) + artifact_root = _resolve(args.artifact_root) + config_dir = _resolve(args.config_dir) + selected_tasks = set(args.tasks or POSTHOC_TASKS) + specs = load_run_specs( + suite_path, + set(args.datasets) if args.datasets else None, + set(args.seeds) if args.seeds else None, + set(args.epsilons) if args.epsilons else None, + ) + logger.info("Discovered %d expected runs across %d datasets.", len(specs), len({item.dataset for item in specs})) + + manifest_rows: list[dict[str, Any]] = [] + raw_cache: dict[str, tuple[np.ndarray, np.ndarray]] = {} + raw_validation_exported: set[str] = set() + missing_primary_count = 0 + for spec in specs: + runtime_candidate = ( + "runtime" in selected_tasks + and spec.seed == args.runtime_seed + and np.isclose(spec.epsilon_pixel, args.runtime_epsilon) + ) + quality_candidate = "image_quality" in selected_tasks + attack_candidate = ( + bool(selected_tasks & { + "attacks", "finetune_attacks", "finetune_budget_attacks", + }) + and np.isclose(spec.epsilon_pixel, args.attacks_epsilon) + ) + needs_raw_test = runtime_candidate or quality_candidate or attack_candidate + needs_cooked_test = runtime_candidate or quality_candidate + required: dict[str, Path] = {} + if "pseudo" in selected_tasks or attack_candidate: + required.update({ + f"prediction_{key}": path + for key, path in spec.predictions.items() + }) + if "pseudo" in selected_tasks: + required["pseudo_labels"] = spec.pseudo_labels + if needs_cooked_test: + required["test_perturbation"] = spec.test_perturbation + if runtime_candidate or attack_candidate: + required["checkpoint"] = spec.checkpoint + required["model_config"] = spec.model_config + missing = [name for name, path in required.items() if not path.is_file()] + if missing: + missing_primary_count += 1 + logger.warning("%s missing: %s", spec.run_name, ", ".join(missing)) + raw_path = artifact_root / spec.dataset / "raw_test.npz" + raw_val_path = artifact_root / spec.dataset / "raw_val.npz" + cooked_path = artifact_root / spec.run_name / "cooked_test.npz" + if ( + not args.dry_run + and attack_candidate + and selected_tasks & RAW_VALIDATION_TASKS + and spec.dataset not in raw_validation_exported + ): + export_raw_validation(spec, raw_val_path, force=args.force, logger=logger) + raw_validation_exported.add(spec.dataset) + if ( + not args.dry_run + and needs_raw_test + and spec.dataset not in raw_cache + ): + raw_cache[spec.dataset] = export_raw_test( + spec, raw_path, force=args.force, logger=logger + ) + if ( + not args.dry_run + and needs_cooked_test + and (spec.cooked_dir.is_dir() or spec.test_perturbation.is_file()) + ): + if spec.dataset not in raw_cache: + raw_cache[spec.dataset] = export_raw_test( + spec, raw_path, force=args.force, logger=logger + ) + raw_images, raw_labels = raw_cache[spec.dataset] + export_cooked_test( + spec.cooked_dir, + raw_images, + raw_labels, + cooked_path, + force=args.force, + logger=logger, + perturbation_path=spec.test_perturbation, + ) + manifest_rows.append({ + "dataset": spec.dataset, + "seed": spec.seed, + "epsilon_pixel": spec.epsilon_pixel, + "epsilon": spec.epsilon, + "run_name": spec.run_name, + "primary_complete": not missing, + "missing_primary": ";".join(missing), + "raw_export": str(raw_path), + "raw_validation_export": str(raw_val_path), + "cooked_export": str(cooked_path), + }) + + if args.dry_run: + logger.info("Dry run complete: %d/%d expected runs have missing primary artifacts.", missing_primary_count, len(specs)) + if args.strict and missing_primary_count: + raise SystemExit(2) + return + + _write_manifest(artifact_root / "manifest.csv", manifest_rows) + counts = generate_configs( + specs, + artifact_root, + config_dir, + attacks_epsilon=args.attacks_epsilon, + runtime_seed=args.runtime_seed, + runtime_epsilon=args.runtime_epsilon, + output_suffix=args.output_suffix, + finetune_epochs=tuple(args.finetune_epochs), + budget_sizes=tuple(args.budget_sizes), + tasks=selected_tasks, + result_root=args.result_root, + ) + logger.info("Generated configs: %s", ", ".join(f"{key}={value}" for key, value in counts.items())) + logger.info("Manifest: %s", artifact_root / "manifest.csv") + logger.info("Master config: %s", config_dir.parent / "generated.yaml") + if args.strict and missing_primary_count: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..c7b23ec --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +pythonpath = . +testpaths = tests diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c24df7d --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest>=7,<9 diff --git a/requirements.txt b/requirements.txt index d66c2b2..67a7069 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,14 @@ -torch -torchvision +torch==2.0.0 +torchvision==0.15.1 mlconfig +numpy +pandas +scikit-learn +scipy +scikit-image +Pillow +PyYAML +medmnist +tqdm +timm==0.6.13 +# Optional (Task 8 LPIPS): pip install lpips diff --git a/revision_experiments/__init__.py b/revision_experiments/__init__.py new file mode 100644 index 0000000..740e42a --- /dev/null +++ b/revision_experiments/__init__.py @@ -0,0 +1 @@ +"""Configuration-driven experiment helpers for the DataCook revision.""" diff --git a/revision_experiments/common.py b/revision_experiments/common.py new file mode 100644 index 0000000..b74136a --- /dev/null +++ b/revision_experiments/common.py @@ -0,0 +1,244 @@ +"""Shared configuration, artifact and CSV utilities.""" + +from __future__ import annotations + +import json +import logging +import copy +import subprocess +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import torch +import yaml + + +BASE_METADATA = [ + "dataset", + "modality", + "task_type", + "method", + "surrogate_model", + "victim_model", + "seed", + "epsilon", + "setting", + "attack_type", + "attack_strength", +] + + +def setup_logging(name: str, verbose: bool = False) -> logging.Logger: + logging.basicConfig( + level=logging.DEBUG if verbose else logging.INFO, + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + return logging.getLogger(name) + + +def load_config(path: str | Path) -> tuple[dict[str, Any], Path]: + config_path = Path(path).expanduser().resolve() + if not config_path.is_file(): + raise FileNotFoundError(f"Configuration file does not exist: {config_path}") + with config_path.open("r", encoding="utf-8") as handle: + config = yaml.safe_load(handle) or {} + if not isinstance(config, dict): + raise ValueError("The top-level YAML value must be a mapping.") + return config, config_path.parent + + +def resolve_path(path: str | Path, base_dir: Path) -> Path: + candidate = Path(path).expanduser() + return candidate.resolve() if candidate.is_absolute() else (base_dir / candidate).resolve() + + +def _artifact_parts(spec: str | Mapping[str, Any], default_key: str | None) -> tuple[str, str | None]: + if isinstance(spec, str): + if "::" in spec: + path, key = spec.rsplit("::", 1) + return path, key + return spec, default_key + if not isinstance(spec, Mapping) or "path" not in spec: + raise TypeError("Artifact specs must be a path string or a mapping containing `path`.") + return str(spec["path"]), spec.get("key", default_key) + + +def load_array( + spec: str | Mapping[str, Any], + base_dir: Path, + *, + default_key: str | None = None, +) -> np.ndarray: + path_text, key = _artifact_parts(spec, default_key) + path = resolve_path(path_text, base_dir) + if not path.is_file(): + raise FileNotFoundError(f"Required artifact does not exist: {path}") + suffix = path.suffix.lower() + if suffix == ".npy": + return np.load(path, allow_pickle=False) + if suffix == ".npz": + with np.load(path, allow_pickle=False) as archive: + if key is None: + if len(archive.files) != 1: + raise KeyError(f"Artifact {path} has multiple arrays; specify `key` or use path::key.") + key = archive.files[0] + if key not in archive: + raise KeyError(f"Array {key!r} not found in {path}; available: {archive.files}.") + return np.asarray(archive[key]) + if suffix == ".csv": + frame = pd.read_csv(path) + if key is None: + if frame.shape[1] != 1: + raise KeyError(f"CSV {path} has multiple columns; specify `key`.") + key = str(frame.columns[0]) + if key not in frame: + raise KeyError(f"Column {key!r} not found in {path}.") + return frame[key].to_numpy() + if suffix in {".pt", ".pth"}: + try: + value = torch.load(path, map_location="cpu", weights_only=False) + except TypeError: # PyTorch versions before weights_only was introduced + value = torch.load(path, map_location="cpu") + if key is not None: + if not isinstance(value, Mapping) or key not in value: + raise KeyError(f"Key {key!r} not found in tensor artifact {path}.") + value = value[key] + if torch.is_tensor(value): + return value.detach().cpu().numpy() + return np.asarray(value) + raise ValueError(f"Unsupported artifact format {suffix!r} for {path}.") + + +def load_prediction_artifact( + spec: str | Mapping[str, Any], base_dir: Path +) -> dict[str, np.ndarray | None]: + """Load y_true plus y_pred/y_prob from a safe NPZ artifact.""" + if isinstance(spec, str): + path = resolve_path(spec, base_dir) + keys: Mapping[str, str] = {} + else: + path = resolve_path(str(spec["path"]), base_dir) + keys = spec.get("keys", {}) + if path.suffix.lower() != ".npz": + raise ValueError(f"Prediction artifacts must be .npz files: {path}") + if not path.is_file(): + raise FileNotFoundError(f"Prediction artifact does not exist: {path}") + true_key = keys.get("y_true", "y_true") + pred_key = keys.get("y_pred", "y_pred") + prob_key = keys.get("y_prob", "y_prob") + with np.load(path, allow_pickle=False) as archive: + if true_key not in archive: + raise KeyError(f"{path} is missing required array {true_key!r}.") + if pred_key not in archive and prob_key not in archive: + raise KeyError(f"{path} must contain {pred_key!r} or {prob_key!r}.") + return { + "y_true": np.asarray(archive[true_key]), + "y_pred": np.asarray(archive[pred_key]) if pred_key in archive else None, + "y_prob": np.asarray(archive[prob_key]) if prob_key in archive else None, + } + + +def metadata(record: Mapping[str, Any], *, setting: str = "", attack_type: str = "", attack_strength: Any = "") -> dict[str, Any]: + source = record.get("metadata", record) + row = {field: source.get(field, "") for field in BASE_METADATA} + row["setting"] = setting or row["setting"] + row["attack_type"] = attack_type or row["attack_type"] + row["attack_strength"] = attack_strength if attack_strength != "" else row["attack_strength"] + missing = [field for field in BASE_METADATA[:8] if row[field] == ""] + if missing: + raise ValueError(f"Experiment metadata is missing required fields: {missing}.") + return row + + +def write_rows(rows: Iterable[Mapping[str, Any]], output: str | Path, base_dir: Path | None = None) -> Path: + rows = list(rows) + if not rows: + raise ValueError(f"Refusing to create an empty result file: {output}") + path = Path(output).expanduser() + if not path.is_absolute(): + # Result paths are conventionally relative to the repository/current cwd, + # while artifact paths are relative to the config file. + path = (base_dir or Path.cwd()) / path + path = path.resolve() + path.parent.mkdir(parents=True, exist_ok=True) + frame = pd.DataFrame(rows) + ordered = [column for column in BASE_METADATA if column in frame.columns] + ordered += [column for column in frame.columns if column not in ordered] + frame[ordered].to_csv(path, index=False) + return path + + +def json_value(value: Any) -> str: + if isinstance(value, np.ndarray): + value = value.tolist() + elif isinstance(value, np.generic): + value = value.item() + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + +def require_records(config: Mapping[str, Any], key: str = "experiments") -> list[dict[str, Any]]: + records = config.get(key) + if not isinstance(records, list) or not records: + raise ValueError(f"Configuration must contain a non-empty `{key}` list.") + expanded = [] + defaults = config.get("metadata_defaults", {}) + for original in records: + record = copy.deepcopy(original) + record_meta = {**defaults, **record.get("metadata", {})} + seeds = record.pop("seeds", config.get("seeds", [record_meta.get("seed", "")])) + epsilons = record.pop("epsilons", [record_meta.get("epsilon", "")]) + if not isinstance(seeds, list): + seeds = [seeds] + if not isinstance(epsilons, list): + epsilons = [epsilons] + for seed in seeds: + for epsilon in epsilons: + item = copy.deepcopy(record) + item["metadata"] = {**record_meta, "seed": seed, "epsilon": epsilon} + context = { + "seed": seed, + "epsilon": epsilon, + "epsilon_pixel": float(epsilon) * 255 if epsilon != "" else "", + } + expanded.append(_format_templates(item, context)) + return expanded + + +class _KeepUnknown(dict): + def __missing__(self, key): + return "{" + key + "}" + + +def _format_templates(value: Any, context: Mapping[str, Any]) -> Any: + if isinstance(value, str): + return value.format_map(_KeepUnknown(context)) + if isinstance(value, list): + return [_format_templates(item, context) for item in value] + if isinstance(value, dict): + return {key: _format_templates(item, context) for key, item in value.items()} + return value + + +def run_prepare_commands(config: Mapping[str, Any], base_dir: Path, logger: logging.Logger) -> None: + """Run explicitly configured artifact-generation commands without a shell.""" + commands = list(config.get("prepare_commands", [])) + # Per-record commands are expanded with the same seed/epsilon templates as + # artifact paths, enabling actual multi-seed and budget generation runs. + if isinstance(config.get("experiments"), list) and config.get("experiments"): + for record in require_records(config): + commands.extend(record.get("prepare_commands", [])) + for index, item in enumerate(commands): + if isinstance(item, list): + command, cwd = item, Path.cwd() + elif isinstance(item, Mapping): + command = item.get("command") + cwd = resolve_path(item.get("cwd", "."), base_dir) + else: + raise TypeError("prepare_commands entries must be argument lists or mappings.") + if not isinstance(command, list) or not command: + raise ValueError(f"prepare_commands[{index}].command must be a non-empty argument list.") + logger.info("Preparing artifacts: %s", " ".join(map(str, command))) + subprocess.run([str(part) for part in command], cwd=cwd, check=True) diff --git a/revision_experiments/modeling.py b/revision_experiments/modeling.py new file mode 100644 index 0000000..09b35ab --- /dev/null +++ b/revision_experiments/modeling.py @@ -0,0 +1,121 @@ +"""Adapters for loading DataCook/PyTorch models and producing probabilities.""" + +from __future__ import annotations + +import importlib +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from torch.utils.data import DataLoader, TensorDataset + +from .common import load_array, resolve_path + + +def import_object(dotted_path: str) -> Any: + module_name, separator, attribute = dotted_path.rpartition(".") + if not separator: + raise ValueError(f"Expected a dotted import path, got {dotted_path!r}.") + module = importlib.import_module(module_name) + return getattr(module, attribute) + + +def load_model(spec: Mapping[str, Any], base_dir: Path, device: str | torch.device) -> torch.nn.Module: + """Load a model through a custom factory or an existing mlconfig YAML.""" + if "factory" in spec: + factory = import_object(str(spec["factory"])) + model = factory(**dict(spec.get("kwargs", {}))) + elif "config_path" in spec: + import dataset # noqa: F401 - registers mlconfig dataset + import models # noqa: F401 - registers mlconfig models + import mlconfig + + config_path = resolve_path(str(spec["config_path"]), base_dir) + config = mlconfig.load(str(config_path)) + model = config.model() + else: + raise ValueError("Model spec requires `factory` or `config_path`.") + if not isinstance(model, torch.nn.Module): + raise TypeError("Model factory did not return torch.nn.Module.") + + checkpoint_text = spec.get("checkpoint") + if checkpoint_text: + checkpoint_path = resolve_path(str(checkpoint_text), base_dir) + if not checkpoint_path.is_file(): + raise FileNotFoundError(f"Model checkpoint does not exist: {checkpoint_path}") + try: + checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + except TypeError: # PyTorch versions before weights_only was introduced + checkpoint = torch.load(checkpoint_path, map_location="cpu") + if isinstance(checkpoint, Mapping): + state = checkpoint.get("model_state_dict", checkpoint.get("state_dict", checkpoint)) + else: + state = checkpoint + if not isinstance(state, Mapping): + raise ValueError(f"Could not find a state dict in {checkpoint_path}.") + # Transparently handle checkpoints saved from DataParallel. + if state and all(str(key).startswith("module.") for key in state): + state = {str(key)[7:]: value for key, value in state.items()} + model.load_state_dict(state, strict=bool(spec.get("strict", True))) + return model.to(device).eval() + + +def probabilities_from_logits(logits: torch.Tensor, task_type: str) -> torch.Tensor: + task = task_type.replace("-", "").replace("_", "").lower() + if task == "multilabel": + return torch.sigmoid(logits) + if task == "binary" and (logits.ndim == 1 or logits.shape[-1] == 1): + return torch.sigmoid(logits.reshape(-1)) + if task in {"binary", "multiclass"}: + return torch.softmax(logits, dim=-1) + raise ValueError(f"Unsupported task_type {task_type!r}.") + + +def predict_probabilities( + model: torch.nn.Module, + inputs: np.ndarray | torch.Tensor, + *, + task_type: str, + device: str | torch.device, + batch_size: int = 128, +) -> np.ndarray: + tensor = torch.as_tensor(inputs).float() + loader = DataLoader(TensorDataset(tensor), batch_size=batch_size, shuffle=False) + probabilities = [] + model.eval() + with torch.no_grad(): + for (batch,) in loader: + logits = model(batch.to(device)) + probabilities.append(probabilities_from_logits(logits, task_type).cpu()) + return torch.cat(probabilities).numpy() + + +def load_xy_artifact(spec: Mapping[str, Any], base_dir: Path) -> tuple[np.ndarray, np.ndarray]: + if "inputs" not in spec or "labels" not in spec: + raise ValueError("Data spec requires `inputs` and `labels` artifact specs.") + return load_array(spec["inputs"], base_dir), load_array(spec["labels"], base_dir) + + +def build_cooker(spec: Mapping[str, Any], base_dir: Path) -> Callable[[torch.Tensor], torch.Tensor]: + """Build an authorized cooker from a callable or saved additive perturbation.""" + if "callable" in spec: + factory = import_object(str(spec["callable"])) + kwargs = dict(spec.get("kwargs", {})) + candidate = factory(**kwargs) + return candidate if callable(candidate) else factory + if "perturbation" in spec: + perturbation = torch.as_tensor(load_array(spec["perturbation"], base_dir)).float() + clamp = spec.get("clamp", [0.0, 1.0]) + + def add_perturbation(batch: torch.Tensor) -> torch.Tensor: + noise = perturbation.to(batch.device) + if noise.ndim == batch.ndim - 1: + noise = noise.unsqueeze(0) + if noise.shape[0] not in (1, batch.shape[0]): + raise ValueError("Per-sample perturbation count must match the timed batch; use batch_size accordingly.") + return torch.clamp(batch + noise, float(clamp[0]), float(clamp[1])) + + return add_perturbation + raise ValueError("Cooker spec requires `callable` or `perturbation`.") diff --git a/run_adaptive_attacks.py b/run_adaptive_attacks.py new file mode 100644 index 0000000..ec45dc2 --- /dev/null +++ b/run_adaptive_attacks.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Task 3: evaluate purification, augmentation and fine-tuning attacks.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import torch +from sklearn.model_selection import StratifiedShuffleSplit +from torch.utils.data import DataLoader, TensorDataset + +from attacks import fine_tune_model, gaussian_smooth, jpeg_compress, median_filter, strong_augment +from metrics import compute_classification_metrics +from revision_experiments.common import load_config, load_prediction_artifact, metadata, require_records, setup_logging, write_rows +from revision_experiments.modeling import load_model, load_xy_artifact, predict_probabilities + + +def _apply_attack(images: np.ndarray, attack_type: str, strength: Any, seed: int) -> np.ndarray: + attack_type = {"gaussian": "gaussian_smoothing", "smoothing": "gaussian_smoothing", "augmentation": "strong_augmentation"}.get(attack_type, attack_type) + if attack_type == "jpeg": + return np.asarray(jpeg_compress(images, quality=int(strength))) + if attack_type == "gaussian_smoothing": + params = strength if isinstance(strength, dict) else {"sigma": float(strength), "kernel_size": 5} + return np.asarray(gaussian_smooth(images, **params)) + if attack_type == "strong_augmentation": + return np.asarray(strong_augment(images, strength=int(strength), seed=seed)) + if attack_type == "median_filter": + return np.asarray(median_filter(images, kernel_size=int(strength))) + raise ValueError(f"Unsupported inference attack type {attack_type!r}.") + + +def _select_raw_subset( + inputs: np.ndarray, + labels: np.ndarray, + subset_size: int | None, + seed: int, +) -> tuple[np.ndarray, np.ndarray]: + """Select a deterministic class-stratified raw subset for an attack budget.""" + if len(inputs) != len(labels): + raise ValueError("Raw subset inputs and labels must have the same length.") + if subset_size is None: + return inputs, labels + subset_size = int(subset_size) + if subset_size <= 0 or subset_size > len(inputs): + raise ValueError( + f"subset_size must be in [1, {len(inputs)}], got {subset_size}." + ) + targets = np.asarray(labels) + if targets.ndim == 2 and targets.shape[1] == 1: + targets = targets.reshape(-1) + if targets.ndim != 1: + rng = np.random.default_rng(seed) + indices = np.sort(rng.choice(len(inputs), size=subset_size, replace=False)) + else: + classes = np.unique(targets) + if subset_size < len(classes): + raise ValueError( + f"subset_size={subset_size} cannot cover all {len(classes)} classes." + ) + splitter = StratifiedShuffleSplit( + n_splits=1, train_size=subset_size, random_state=seed + ) + try: + indices, _ = next(splitter.split(np.zeros(len(targets)), targets)) + except ValueError as exc: + raise ValueError( + f"Could not draw a stratified raw subset of size {subset_size}: {exc}" + ) from exc + indices = np.sort(indices) + return np.asarray(inputs)[indices], np.asarray(labels)[indices] + + +def _attacked_metrics(record: dict[str, Any], attack: dict[str, Any], base_dir, device: str) -> dict[str, float]: + task_type = record["metadata"]["task_type"] + threshold = float(record.get("threshold", 0.5)) + if "predictions" in attack: + artifact = load_prediction_artifact(attack["predictions"], base_dir) + return compute_classification_metrics(artifact["y_true"], artifact["y_pred"], artifact["y_prob"], task_type, threshold) + + model = load_model(attack.get("model", record.get("model", {})), base_dir, device) + attack_type = attack["type"] + if attack_type in {"finetune", "robust_finetune"}: + subset_x, subset_y = load_xy_artifact(attack["raw_subset"], base_dir) + seed = int(record["metadata"]["seed"]) + subset_x, subset_y = _select_raw_subset( + subset_x, subset_y, attack.get("subset_size"), seed + ) + generator = torch.Generator().manual_seed(seed) + loader = DataLoader( + TensorDataset(torch.as_tensor(subset_x).float(), torch.as_tensor(subset_y)), + batch_size=int(attack.get("batch_size", 64)), shuffle=True, generator=generator, + ) + input_transform = None + if attack_type == "robust_finetune": + strength = int(attack.get("augmentation_strength", 2)) + input_transform = lambda batch: strong_augment(batch, strength=strength) + model = fine_tune_model( + model, loader, task_type=task_type, epochs=int(attack.get("epochs", 5)), + learning_rate=float(attack.get("learning_rate", 1e-4)), device=device, + input_transform=input_transform, + seed=seed, + ) + eval_x, eval_y = load_xy_artifact(attack["eval_data"], base_dir) + else: + eval_x, eval_y = load_xy_artifact(attack.get("eval_data", record.get("eval_data", {})), base_dir) + eval_x = _apply_attack(eval_x, attack_type, attack["strength"], int(record["metadata"]["seed"])) + prob = predict_probabilities(model, eval_x, task_type=task_type, device=device, batch_size=int(attack.get("batch_size", 128))) + return compute_classification_metrics(eval_y, y_prob=prob, task_type=task_type, threshold=threshold) + + +def _result_path(value: str | Path) -> Path: + path = Path(value).expanduser() + return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve() + + +def _merge_results(new_output: Path, merge_with: str | Path, combined_output: str | Path) -> Path: + """Merge old inference attacks and new fine-tuning attacks without changing either input.""" + existing_path = _result_path(merge_with) + if not existing_path.is_file(): + raise FileNotFoundError(f"Existing adaptive-attack result does not exist: {existing_path}") + frames = [pd.read_csv(existing_path), pd.read_csv(new_output)] + columns = list(frames[0].columns) + [ + column for column in frames[1].columns if column not in frames[0].columns + ] + combined = pd.concat( + [frame.reindex(columns=columns) for frame in frames], ignore_index=True + ) + combined = combined.drop_duplicates().reset_index(drop=True) + destination = _result_path(combined_output) + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_suffix(destination.suffix + ".tmp") + combined.to_csv(temporary, index=False) + temporary.replace(destination) + return destination + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("adaptive-attacks", args.verbose) + config, base_dir = load_config(args.config) + rows = [] + for record in require_records(config): + baseline = {} + for setting in ("clean", "authorized_cooked", "unauthorized_raw"): + artifact = load_prediction_artifact(record["predictions"][setting], base_dir) + baseline[setting] = compute_classification_metrics( + artifact["y_true"], artifact["y_pred"], artifact["y_prob"], + record["metadata"]["task_type"], float(record.get("threshold", 0.5)), + ) + for attack in record.get("attacks", []): + if "type" not in attack: + raise ValueError("Every adaptive attack requires `type`.") + if "strength" in attack: + strength = attack["strength"] + else: + strength = f"epochs={attack.get('epochs', 5)},lr={attack.get('learning_rate', 1e-4):g}" + if attack["type"] == "robust_finetune": + strength += f",aug={attack.get('augmentation_strength', 2)}" + if "subset_size" in attack: + strength += f",n={int(attack['subset_size'])}" + logger.info("Running %s strength=%s", attack["type"], strength) + attacked = _attacked_metrics(record, attack, base_dir, args.device) + metric_names = attack.get("metric_names", record.get("metric_names")) or sorted(set(baseline["clean"]) & set(attacked)) + for metric in metric_names: + clean = baseline["clean"][metric] + authorized = baseline["authorized_cooked"][metric] + unauthorized = baseline["unauthorized_raw"][metric] + after = attacked[metric] + denominator = clean - unauthorized + raw_recovery = (after - unauthorized) / denominator if denominator != 0 else float("nan") + recovery = float(np.clip(raw_recovery, 0.0, 1.0)) if np.isfinite(raw_recovery) else float("nan") + rows.append({ + **metadata(record, setting="adaptive_attack", attack_type=attack["type"], attack_strength=strength), + "metric": metric, "clean": clean, "authorized_cooked": authorized, + "unauthorized_raw": unauthorized, "attacked_unauthorized_raw": after, + "cp_before_attack": unauthorized - clean, "cp_after_attack": after - clean, + "recovery_rate": recovery, "recovery_rate_unclipped": raw_recovery, + "raw_subset_size": attack.get("subset_size", "full") + if attack["type"] in {"finetune", "robust_finetune"} else "", + }) + if not rows: + raise ValueError("No attacks were configured; no results were written.") + output = config.get("output", "results/attacks/adaptive_attack_results.csv") + output_path = write_rows(rows, output) + logger.info("Wrote %s", output_path) + merge_with = config.get("merge_with") + combined_output = config.get("combined_output") + if bool(merge_with) != bool(combined_output): + raise ValueError("`merge_with` and `combined_output` must be configured together.") + if merge_with and combined_output: + logger.info("Wrote merged adaptive attacks to %s", _merge_results(output_path, merge_with, combined_output)) + + +if __name__ == "__main__": + main() diff --git a/run_datacook_pipeline.py b/run_datacook_pipeline.py new file mode 100644 index 0000000..2b6d77c --- /dev/null +++ b/run_datacook_pipeline.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Resumable one-click DataCook main experiment and revision evaluation pipeline.""" + +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import logging +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +import yaml + + +PROJECT_ROOT = Path(__file__).resolve().parent +STAGES = ("perturbation", "clean_victim", "protected", "metrics") + + +def _path(value: str) -> Path: + candidate = Path(value).expanduser() + return candidate.resolve() if candidate.is_absolute() else (PROJECT_ROOT / candidate).resolve() + + +def _slug(value: Any) -> str: + return re.sub(r"[^A-Za-z0-9_.=-]+", "-", str(value)).strip("-") + + +def _write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, ensure_ascii=False), encoding="utf-8") + temporary.replace(path) + + +def _complete(paths: list[Path]) -> bool: + return all(path.is_file() if path.suffix else path.is_dir() for path in paths) + + +def _run(command: list[str], log_path: Path, logger: logging.Logger, dry_run: bool) -> None: + rendered = " ".join(command) + logger.info("Command: %s", rendered) + if dry_run: + return + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as log_handle: + log_handle.write(f"\n[{dt.datetime.now().isoformat()}] {rendered}\n") + process = subprocess.Popen( + command, + cwd=PROJECT_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + assert process.stdout is not None + for line in process.stdout: + print(line, end="") + log_handle.write(line) + log_handle.flush() + return_code = process.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, command) + + +def _main_command( + python: str, + *, + model: str, + config_path: Path, + exp_name: Path, + dataset_name: str, + train_type: str, + train_path: Path, + test_type: str, + test_path: Path, + recover_type: str, + recover_path: Path, + prediction_dir: Path, + role: str, + seed: int, + train_batch_size: int, + eval_batch_size: int, + workers: int, + checkpoint: Path, + high_resolution: bool = False, +) -> list[str]: + command = [ + python, "-u", "main.py", + "--version", model, + "--exp_name", str(exp_name), + "--config_path", str(config_path), + "--train_data_type", train_type, + "--train_data_path", str(train_path), + "--test_data_type", test_type, + "--test_data_path", str(test_path), + "--recover_test_data_type", recover_type, + "--recover_test_data_path", str(recover_path), + "--train_batch_size", str(train_batch_size), + "--eval_batch_size", str(eval_batch_size), + "--num_of_workers", str(workers), + "--seed", str(seed), + "--deterministic", + "--train", + "--export_predictions_dir", str(prediction_dir), + "--evaluation_role", role, + ] + if checkpoint.is_file(): + command.append("--load_model") + if high_resolution: + command.append("--high_resolution") + return command + + +def _run_one(config: dict[str, Any], seed: int, epsilon: float, args, logger) -> dict[str, Path]: + dataset_cfg = config["dataset"] + experiment_cfg = config.get("experiment", {}) + model_cfg = config["models"] + runtime_cfg = config.get("runtime", {}) + dataset_name = dataset_cfg["name"] + high_resolution = bool(dataset_cfg.get("high_resolution", False)) + poison_name = dataset_cfg.get("poison_name", "Poison" + dataset_name) + surrogate = model_cfg.get("surrogate", "resnet18") + victim = model_cfg.get("victim", surrogate) + epsilon_scale = experiment_cfg.get("epsilon_scale", "pixel") + normalized_epsilon = epsilon / 255.0 if epsilon_scale == "pixel" else epsilon + step_size = experiment_cfg.get("step_size") + if step_size is None: + step_size = epsilon * float(experiment_cfg.get("step_size_ratio", 0.2)) + + epsilon_label = f"{epsilon:g}" + run_name = _slug( + f"{dataset_name}-{experiment_cfg.get('attack_type', 'datacook')}-" + f"sur={surrogate}-vic={victim}-seed={seed}-eps={epsilon_label}" + ) + experiment_root = _path(config.get("experiment_root", "experiments/revision_pipeline")) + result_root = _path(config.get("result_root", "results/revision_pipeline")) + run_dir = experiment_root / run_name + prediction_dir = run_dir / "predictions" + log_dir = run_dir / "pipeline_logs" + state_path = run_dir / "pipeline_state.json" + is_3d = str(dataset_cfg.get("modality", "")).lower() in { + "3d", "3d_image", "3d_volume", "volume" + } or len(dataset_cfg.get("noise_shape", [])) == 5 + task_type = str(dataset_cfg.get("task_type", "multiclass")) + normalized_task = task_type.replace("-", "").replace("_", "").lower() + is_multilabel = normalized_task == "multilabel" + cooked_as_npz = is_3d or is_multilabel + cooked_train_dir = run_dir / surrogate / "ae" + cooked_test_dir = run_dir / surrogate / "ae_test" + cooked_train = cooked_train_dir / "perturb_data.npz" if cooked_as_npz else cooked_train_dir + cooked_test = cooked_test_dir / "perturb_data.npz" if cooked_as_npz else cooked_test_dir + python = str(runtime_cfg.get("python", sys.executable)) + config_path = _path(dataset_cfg["config_path"]) + for model_name in {surrogate, victim}: + model_config = config_path / f"{model_name}.yaml" + if not model_config.is_file(): + raise FileNotFoundError(f"Model config does not exist: {model_config}") + raw_data_path = _path(dataset_cfg.get("data_path", "../datasets")) + noise_shape = dataset_cfg.get("noise_shape") + test_noise_shape = dataset_cfg.get("test_noise_shape") + if not noise_shape or not test_noise_shape: + raise ValueError("dataset.noise_shape and dataset.test_noise_shape are required for the current generator.") + expected_size = 224 if high_resolution else None + if expected_size is not None and ( + len(noise_shape) != 4 + or len(test_noise_shape) != 4 + or list(noise_shape[-2:]) != [expected_size, expected_size] + or list(test_noise_shape[-2:]) != [expected_size, expected_size] + ): + raise ValueError( + "High-resolution pipelines require 4D noise_shape/test_noise_shape " + "ending in [224, 224]." + ) + num_classes = int(dataset_cfg["num_classes"]) + train_batch_size = int(runtime_cfg.get("train_batch_size", 128)) + eval_batch_size = int(runtime_cfg.get("eval_batch_size", train_batch_size)) + workers = int(runtime_cfg.get("num_workers", 8)) + + state = { + "run_name": run_name, + "dataset": dataset_name, + "seed": seed, + "epsilon": epsilon, + "epsilon_normalized": normalized_epsilon, + "updated_at": dt.datetime.now().isoformat(), + "stages": {}, + } + if state_path.is_file(): + state.update(json.loads(state_path.read_text(encoding="utf-8"))) + logger.info("===== Run %s =====", run_name) + + selected = set(args.only or STAGES) + + def stage(name: str, command: list[str], expected: list[Path]) -> None: + if name not in selected: + logger.info("Skipping unselected stage %s", name) + return + if not args.force and _complete(expected): + logger.info("Stage %s already complete; reusing artifacts.", name) + state["stages"][name] = {"status": "reused", "updated_at": dt.datetime.now().isoformat()} + if not args.dry_run: + _write_json(state_path, state) + return + logger.info("Starting stage %s", name) + state["stages"][name] = {"status": "running", "updated_at": dt.datetime.now().isoformat()} + if not args.dry_run: + _write_json(state_path, state) + try: + _run(command, log_dir / f"{name}.log", logger, args.dry_run) + if not args.dry_run and not _complete(expected): + missing = [str(path) for path in expected if not (path.is_file() if path.suffix else path.is_dir())] + raise RuntimeError(f"Stage {name} exited successfully but outputs are missing: {missing}") + state["stages"][name] = {"status": "complete", "updated_at": dt.datetime.now().isoformat()} + except Exception as exc: + state["stages"][name] = {"status": "failed", "error": str(exc), "updated_at": dt.datetime.now().isoformat()} + if not args.dry_run: + _write_json(state_path, state) + raise + if not args.dry_run: + _write_json(state_path, state) + + perturb_command = [ + python, "-u", "perturbation.py", + "--seed", str(seed), + "--version", surrogate, + "--exp_name", str(run_dir), + "--config_path", str(config_path), + "--train_data_type", dataset_name, + "--train_data_path", str(raw_data_path), + "--test_data_type", dataset_name, + "--test_data_path", str(raw_data_path), + "--recover_test_data_type", dataset_name, + "--recover_test_data_path", str(raw_data_path), + "--noise_shape", *map(str, noise_shape), + "--test_noise_shape", *map(str, test_noise_shape), + "--epsilon", str(epsilon), + "--epsilon_scale", epsilon_scale, + "--step_size", str(step_size), + "--num_steps", str(experiment_cfg.get("num_steps", 20)), + "--attack_type", experiment_cfg.get("attack_type", "datacook"), + "--universal_stop_error", str(experiment_cfg.get("universal_stop_error", 0.01)), + "--train_batch_size", str(train_batch_size), + "--eval_batch_size", str(eval_batch_size), + "--num_of_workers", str(workers), + "--deterministic", + "--export_predictions_dir", str(prediction_dir), + ] + if high_resolution: + perturb_command.append("--high_resolution") + stage( + "perturbation", + perturb_command, + [run_dir / "perturbation.pt", run_dir / "test_perturbation.pt", cooked_train, cooked_test, prediction_dir / "clean.npz"], + ) + + if victim != surrogate: + clean_exp = run_dir / "clean_victim" + clean_checkpoint = clean_exp / victim / "checkpoints" / f"{victim}.pth" + clean_command = _main_command( + python, model=victim, config_path=config_path, exp_name=clean_exp, + dataset_name=dataset_name, train_type=dataset_name, train_path=raw_data_path, + test_type=dataset_name, test_path=raw_data_path, recover_type=dataset_name, + recover_path=raw_data_path, prediction_dir=prediction_dir, role="clean", + seed=seed, train_batch_size=train_batch_size, eval_batch_size=eval_batch_size, + workers=workers, checkpoint=clean_checkpoint, high_resolution=high_resolution, + ) + stage("clean_victim", clean_command, [clean_checkpoint, prediction_dir / "clean.npz"]) + else: + logger.info("Victim equals surrogate; clean predictions are reused from perturbation stage.") + + protected_exp = run_dir / "protected" + protected_checkpoint = protected_exp / victim / "checkpoints" / f"{victim}.pth" + protected_command = _main_command( + python, model=victim, config_path=config_path, exp_name=protected_exp, + dataset_name=dataset_name, train_type=poison_name, train_path=cooked_train, + test_type=dataset_name, test_path=raw_data_path, recover_type=poison_name, + recover_path=cooked_test, prediction_dir=prediction_dir, role="protected", + seed=seed, train_batch_size=train_batch_size, eval_batch_size=eval_batch_size, + workers=workers, checkpoint=protected_checkpoint, high_resolution=high_resolution, + ) + stage( + "protected", + protected_command, + [protected_checkpoint, prediction_dir / "authorized_cooked.npz", prediction_dir / "unauthorized_raw.npz"], + ) + + metrics_dir = result_root / run_name / "metrics" + metrics_config = run_dir / "generated_metrics.yaml" + metrics_payload = { + "experiments": [{ + "metadata": { + "dataset": dataset_name, + "modality": dataset_cfg.get( + "modality", "high_resolution" if high_resolution else ("3d_volume" if is_3d else "2d_image") + ), + "task_type": task_type, + "method": experiment_cfg.get("attack_type", "datacook"), + "surrogate_model": surrogate, + "victim_model": victim, + "seed": seed, + "epsilon": normalized_epsilon, + }, + "num_classes": num_classes, + "threshold": experiment_cfg.get("threshold", 0.5), + "metric_names": dataset_cfg.get("metric_names"), + "predictions": { + "clean": str(prediction_dir / "clean.npz"), + "authorized_cooked": str(prediction_dir / "authorized_cooked.npz"), + "unauthorized_raw": str(prediction_dir / "unauthorized_raw.npz"), + }, + }], + "outputs": { + "main_metrics": str(metrics_dir / "main_metrics.csv"), + "cp_pp": str(metrics_dir / "cp_pp_summary.csv"), + }, + } + if not args.dry_run: + metrics_config.parent.mkdir(parents=True, exist_ok=True) + metrics_config.write_text(yaml.safe_dump(metrics_payload, sort_keys=False), encoding="utf-8") + metrics_command = [python, "run_unified_evaluation.py", "--config", str(metrics_config)] + stage("metrics", metrics_command, [metrics_dir / "main_metrics.csv", metrics_dir / "cp_pp_summary.csv"]) + logger.info("Completed %s; results: %s", run_name, metrics_dir) + return { + "main_metrics": metrics_dir / "main_metrics.csv", + "cp_pp": metrics_dir / "cp_pp_summary.csv", + "result_root": result_root, + } + + +def _concatenate_csv(inputs: list[Path], output: Path) -> None: + existing = [path for path in inputs if path.is_file()] + if not existing: + return + output.parent.mkdir(parents=True, exist_ok=True) + fieldnames = None + with output.open("w", encoding="utf-8", newline="") as output_handle: + writer = None + for path in existing: + with path.open("r", encoding="utf-8", newline="") as input_handle: + reader = csv.DictReader(input_handle) + if fieldnames is None: + fieldnames = reader.fieldnames + writer = csv.DictWriter(output_handle, fieldnames=fieldnames) + writer.writeheader() + elif reader.fieldnames != fieldnames: + raise ValueError(f"Cannot concatenate CSV files with different schemas: {path}") + assert writer is not None + writer.writerows(reader) + + +def _discover_run_results(result_root: Path, filename: str) -> list[Path]: + """Find per-run results so summaries include every dataset/config run. + + A pipeline invocation used to summarize only the seed/epsilon combinations + from the current config. That caused a later dataset invocation to replace + the shared summary with results from that dataset alone. Scanning the + per-run directories makes sequential multi-dataset execution resumable and + keeps one cumulative summary. + """ + return sorted( + path for path in result_root.glob(f"*/metrics/{filename}") + if path.is_file() + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--dry-run", action="store_true", help="Print commands without running or writing state.") + parser.add_argument("--force", action="store_true", help="Rerun selected stages even if outputs exist.") + parser.add_argument("--only", nargs="+", choices=STAGES, help="Run only selected stages.") + parser.add_argument("--seeds", nargs="+", type=int, help="Override the seed list in the YAML config.") + parser.add_argument("--epsilons", nargs="+", type=float, help="Override the epsilon list in the YAML config.") + args = parser.parse_args() + config_path = _path(args.config) + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("Pipeline config must be a YAML mapping.") + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logger = logging.getLogger("datacook-pipeline") + if not args.dry_run: + python = str(config.get("runtime", {}).get("python", sys.executable)) + preflight = subprocess.run( + [python, "-c", ( + "import torch, torchvision, mlconfig, medmnist, sklearn, skimage, pandas, yaml, timm; " + "print('PyTorch:', torch.__version__); " + "print('timm:', timm.__version__); " + "print('CUDA available:', torch.cuda.is_available()); " + "print('Device count:', torch.cuda.device_count())" + )], + cwd=PROJECT_ROOT, + text=True, + capture_output=True, + ) + if preflight.stdout: + logger.info("Environment preflight:\n%s", preflight.stdout.rstrip()) + if preflight.returncode: + raise RuntimeError( + "Environment preflight failed. Install requirements in the configured Python environment.\n" + + preflight.stderr + ) + seeds = args.seeds if args.seeds is not None else config.get("seeds", [0]) + epsilons = ( + args.epsilons if args.epsilons is not None + else config.get("epsilons", [config.get("experiment", {}).get("epsilon", 8)]) + ) + completed_runs = [] + for seed in seeds: + for epsilon in epsilons: + completed_runs.append(_run_one(config, int(seed), float(epsilon), args, logger)) + + should_summarize = not args.dry_run and (args.only is None or "metrics" in args.only) + if should_summarize and completed_runs: + result_root = completed_runs[0]["result_root"] + summary_dir = result_root / "summary" + combined_main = summary_dir / "main_metrics.csv" + combined_cp = summary_dir / "cp_pp_summary.csv" + _concatenate_csv(_discover_run_results(result_root, "main_metrics.csv"), combined_main) + _concatenate_csv(_discover_run_results(result_root, "cp_pp_summary.csv"), combined_cp) + if combined_cp.is_file(): + statistics_config = summary_dir / "statistics.yaml" + aggregated_output = summary_dir / "aggregated_results.csv" + statistics_config.write_text(yaml.safe_dump({ + "inputs": [{ + "path": str(combined_cp), + "experiment_type": "main", + "value_columns": ["cp", "pp"], + }], + "aggregated_output": str(aggregated_output), + }, sort_keys=False), encoding="utf-8") + python = str(config.get("runtime", {}).get("python", sys.executable)) + _run( + [python, "-m", "analysis.aggregate_results", "--config", str(statistics_config)], + summary_dir / "aggregate.log", + logger, + False, + ) + logger.info("Combined results written to %s", summary_dir) + + +if __name__ == "__main__": + main() diff --git a/run_datacook_suite.py b/run_datacook_suite.py new file mode 100755 index 0000000..b4eae65 --- /dev/null +++ b/run_datacook_suite.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Run several resumable DataCook pipeline configurations sequentially.""" + +from __future__ import annotations + +import argparse +import logging +import subprocess +import sys +from pathlib import Path + +import yaml + + +PROJECT_ROOT = Path(__file__).resolve().parent +STAGES = ("perturbation", "clean_victim", "protected", "metrics") + + +def _resolve(path: str | Path, base: Path) -> Path: + candidate = Path(path).expanduser() + return candidate.resolve() if candidate.is_absolute() else (base / candidate).resolve() + + +def load_pipeline_configs(suite_path: Path) -> list[Path]: + payload = yaml.safe_load(suite_path.read_text(encoding="utf-8")) or {} + if not isinstance(payload, dict): + raise ValueError("Suite config must be a YAML mapping.") + entries = payload.get("pipelines") + if not isinstance(entries, list) or not entries: + raise ValueError("Suite config requires a non-empty `pipelines` list.") + configs = [_resolve(entry, suite_path.parent) for entry in entries] + missing = [str(path) for path in configs if not path.is_file()] + if missing: + raise FileNotFoundError("Pipeline config(s) do not exist: " + ", ".join(missing)) + return configs + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, help="Suite YAML containing a `pipelines` list.") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--force", action="store_true") + parser.add_argument("--only", nargs="+", choices=STAGES) + parser.add_argument("--seeds", nargs="+", type=int, help="Override seeds in every pipeline config.") + parser.add_argument("--epsilons", nargs="+", type=float, help="Override epsilons in every pipeline config.") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") + logger = logging.getLogger("datacook-suite") + suite_path = _resolve(args.config, PROJECT_ROOT) + configs = load_pipeline_configs(suite_path) + logger.info("Suite contains %d dataset pipelines.", len(configs)) + + for index, pipeline_config in enumerate(configs, start=1): + command = [ + sys.executable, + "-u", + "run_datacook_pipeline.py", + "--config", + str(pipeline_config), + ] + if args.dry_run: + command.append("--dry-run") + if args.force: + command.append("--force") + if args.only: + command.extend(["--only", *args.only]) + if args.seeds is not None: + command.extend(["--seeds", *map(str, args.seeds)]) + if args.epsilons is not None: + command.extend(["--epsilons", *map(str, args.epsilons)]) + logger.info("===== Dataset pipeline %d/%d: %s =====", index, len(configs), pipeline_config.name) + logger.info("Command: %s", " ".join(command)) + subprocess.run(command, cwd=PROJECT_ROOT, check=True) + + logger.info("All %d dataset pipelines completed successfully.", len(configs)) + + +if __name__ == "__main__": + main() diff --git a/run_epsilon_ablation.py b/run_epsilon_ablation.py new file mode 100644 index 0000000..3927ee8 --- /dev/null +++ b/run_epsilon_ablation.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Task 4: evaluate configurable perturbation-budget runs.""" + +from __future__ import annotations + +import argparse + +from metrics import compute_classification_metrics, compute_cp_pp, compute_image_quality_metrics +from revision_experiments.common import load_array, load_config, load_prediction_artifact, metadata, require_records, setup_logging, write_rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("epsilon-ablation", args.verbose) + config, base_dir = load_config(args.config) + rows = [] + for record in require_records(config): + task = record["metadata"]["task_type"] + values = {} + for setting in ("clean", "authorized_cooked", "unauthorized_raw"): + artifact = load_prediction_artifact(record["predictions"][setting], base_dir) + values[setting] = compute_classification_metrics( + artifact["y_true"], artifact["y_pred"], artifact["y_prob"], task, float(record.get("threshold", 0.5)), + ) + metric_names = record.get("metric_names") or sorted(set.intersection(*(set(item) for item in values.values()))) + cp_pp = compute_cp_pp( + values["clean"], values["authorized_cooked"], values["unauthorized_raw"], metric_names, + record.get("random_guess_metrics"), task_type=task, num_classes=record.get("num_classes"), + label_prevalence=record.get("label_prevalence"), + ) + quality = {name: float("nan") for name in ("ssim_mean", "psnr_mean", "lpips_mean", "l2_norm_mean", "linf_norm_mean")} + if "images" in record: + raw = load_array(record["images"]["raw"], base_dir) + cooked = load_array(record["images"]["cooked"], base_dir) + full_quality = compute_image_quality_metrics(raw, cooked, compute_lpips=bool(record.get("compute_lpips", True))) + quality.update({name: full_quality[name] for name in quality}) + elif record["metadata"].get("modality", "").lower() not in {"tabular", "table"}: + logger.warning("No image artifacts for %s epsilon=%s; quality columns are NaN.", record["metadata"]["dataset"], record["metadata"]["epsilon"]) + for metric, result in cp_pp.items(): + aliases = {name[:-5] if name.endswith("_mean") else name: value for name, value in quality.items()} + rows.append({**metadata(record, setting="epsilon_ablation"), "metric": metric, **result, **quality, **aliases}) + output = config.get("output", "results/ablations/epsilon_ablation.csv") + logger.info("Wrote %s", write_rows(rows, output)) + + +if __name__ == "__main__": + main() diff --git a/run_image_quality.py b/run_image_quality.py new file mode 100644 index 0000000..94c5484 --- /dev/null +++ b/run_image_quality.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Task 8: compute image quality and imperceptibility metrics.""" + +from __future__ import annotations + +import argparse + +from metrics import compute_image_quality_metrics +from revision_experiments.common import load_array, load_config, metadata, require_records, setup_logging, write_rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("image-quality", args.verbose) + config, base_dir = load_config(args.config) + rows = [] + for record in require_records(config): + images = record.get("images", {}) + if "raw" not in images or "cooked" not in images: + raise ValueError("Each image-quality experiment requires images.raw and images.cooked.") + raw = load_array(images["raw"], base_dir) + cooked = load_array(images["cooked"], base_dir) + logger.info("Computing image quality for %s (%d samples)", record["metadata"]["dataset"], len(raw)) + values = compute_image_quality_metrics( + raw, cooked, data_range=record.get("data_range"), + compute_lpips=bool(record.get("compute_lpips", True)), device=record.get("device"), + ) + rows.append({**metadata(record, setting="image_quality"), **values}) + output = config.get("output", "results/image_quality/quality_metrics.csv") + logger.info("Wrote %s", write_rows(rows, output)) + + +if __name__ == "__main__": + main() diff --git a/run_pseudo_label_analysis.py b/run_pseudo_label_analysis.py new file mode 100644 index 0000000..f8f8a60 --- /dev/null +++ b/run_pseudo_label_analysis.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Task 5: sample-level pseudo-label export and quality/CP/PP analysis.""" + +from __future__ import annotations + +import argparse +import warnings + +import numpy as np +from scipy.stats import pearsonr, spearmanr + +from metrics import compute_classification_metrics, compute_cp_pp +from revision_experiments.common import json_value, load_config, load_prediction_artifact, metadata, require_records, resolve_path, setup_logging, write_rows + + +def _load_pseudo(spec, base_dir, task_type: str, threshold: float): + path = resolve_path(spec["path"] if isinstance(spec, dict) else spec, base_dir) + if not path.is_file() or path.suffix.lower() != ".npz": + raise FileNotFoundError(f"Pseudo-label artifact must be an existing .npz file: {path}") + keys = spec.get("keys", {}) if isinstance(spec, dict) else {} + with np.load(path, allow_pickle=False) as archive: + true = np.asarray(archive[keys.get("y_true", "y_true")]) + sample_ids = np.asarray(archive[keys.get("sample_id", "sample_id")]) if keys.get("sample_id", "sample_id") in archive else np.arange(len(true)) + prob_key, logits_key = keys.get("y_prob", "y_prob"), keys.get("logits", "logits") + if prob_key in archive: + prob = np.asarray(archive[prob_key], dtype=float) + elif logits_key in archive: + logits = np.asarray(archive[logits_key], dtype=float) + task = task_type.replace("-", "").replace("_", "").lower() + if task == "multilabel" or (task == "binary" and (logits.ndim == 1 or logits.shape[-1] == 1)): + prob = 1.0 / (1.0 + np.exp(-np.clip(logits, -50, 50))) + else: + shifted = logits - logits.max(axis=-1, keepdims=True) + exp = np.exp(shifted) + prob = exp / exp.sum(axis=-1, keepdims=True) + else: + raise KeyError(f"{path} must contain probabilities or logits.") + + if not np.all(np.isfinite(prob)) or np.any(prob < 0) or np.any(prob > 1): + raise ValueError(f"Pseudo-label probabilities in {path} must be finite and in [0,1].") + task = task_type.replace("-", "").replace("_", "").lower() + if task == "binary": + if prob.ndim == 2 and prob.shape[1] == 2: + prob = prob[:, 1] + prob = prob.reshape(-1) + pseudo = (prob >= threshold).astype(int) + confidence = np.maximum(prob, 1.0 - prob) + true = true.reshape(-1).astype(int) + correct = pseudo == true + elif task == "multiclass": + pseudo = np.argmax(prob, axis=1) + confidence = np.max(prob, axis=1) + true = true.reshape(-1).astype(int) + correct = pseudo == true + elif task == "multilabel": + pseudo = (prob >= threshold).astype(int) + confidence = np.mean(np.maximum(prob, 1.0 - prob), axis=1) + true = np.asarray(true).astype(int) + if true.shape != pseudo.shape: + raise ValueError("Multilabel pseudo y_true and probabilities must have the same shape.") + correct = np.all(pseudo == true, axis=1) + else: + raise ValueError(f"Unsupported task_type {task_type!r}.") + return sample_ids, true, pseudo, confidence, correct + + +def _by_class(true: np.ndarray, pseudo: np.ndarray, task_type: str) -> dict[str, float]: + task = task_type.replace("-", "").replace("_", "").lower() + output = {} + if task == "multilabel": + for index in range(true.shape[1]): + output[str(index)] = float(np.mean(true[:, index] == pseudo[:, index])) + else: + for label in np.unique(true): + mask = true == label + output[str(int(label))] = float(np.mean(pseudo[mask] == true[mask])) + return output + + +def _cp_pp(record, base_dir): + if "predictions" not in record: + supplied = record.get("cp_pp", {}) + return supplied.get("cp_acc", np.nan), supplied.get("pp_acc", np.nan), supplied.get("cp_auc", np.nan), supplied.get("pp_auc", np.nan) + task = record["metadata"]["task_type"] + metrics = {} + for setting in ("clean", "authorized_cooked", "unauthorized_raw"): + artifact = load_prediction_artifact(record["predictions"][setting], base_dir) + metrics[setting] = compute_classification_metrics(artifact["y_true"], artifact["y_pred"], artifact["y_prob"], task, float(record.get("threshold", 0.5))) + acc_name = "subset_acc" if task.replace("-", "").replace("_", "").lower() == "multilabel" else "acc" + auc_name = "auc" if task.replace("-", "").replace("_", "").lower() == "binary" else "macro_auc" + result = compute_cp_pp(metrics["clean"], metrics["authorized_cooked"], metrics["unauthorized_raw"], [acc_name, auc_name], task_type=task, num_classes=record.get("num_classes"), label_prevalence=record.get("label_prevalence")) + return result[acc_name]["cp"], result[acc_name]["pp"], result[auc_name]["cp"], result[auc_name]["pp"] + + +def _correlation(x, y, method): + mask = np.isfinite(x) & np.isfinite(y) + if mask.sum() < 2 or np.unique(x[mask]).size < 2 or np.unique(y[mask]).size < 2: + return float("nan") + return float((pearsonr if method == "pearson" else spearmanr)(x[mask], y[mask]).statistic) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("pseudo-label-analysis", args.verbose) + config, base_dir = load_config(args.config) + sample_rows, quality_rows = [], [] + for record in require_records(config): + task = record["metadata"]["task_type"] + ids, true, pseudo, confidence, correct = _load_pseudo(record["pseudo_labels"], base_dir, task, float(record.get("threshold", 0.5))) + meta = metadata(record, setting="pseudo_labels") + for sample_id, target, predicted, conf, is_correct in zip(ids, true, pseudo, confidence, correct): + sample_rows.append({ + **meta, "sample_id": sample_id, "true_label": json_value(target), + "pseudo_label": json_value(predicted), "pseudo_confidence": float(conf), + "surrogate_correct": bool(is_correct), + }) + cp_acc, pp_acc, cp_auc, pp_auc = _cp_pp(record, base_dir) + quality_rows.append({ + **meta, "pseudo_acc": float(np.mean(correct)), "mean_confidence": float(np.mean(confidence)), + "pseudo_acc_by_class": json_value(_by_class(true, pseudo, task)), + "cp_acc": cp_acc, "pp_acc": pp_acc, "cp_auc": cp_auc, "pp_auc": pp_auc, + }) + + group_fields = ["dataset", "modality", "task_type", "method", "surrogate_model"] + for row in quality_rows: + peers = [candidate for candidate in quality_rows if all(candidate[field] == row[field] for field in group_fields)] + pseudo_acc = np.asarray([item["pseudo_acc"] for item in peers]) + confidence = np.asarray([item["mean_confidence"] for item in peers]) + for outcome in ("cp_acc", "pp_acc", "cp_auc", "pp_auc"): + target = np.asarray([item[outcome] for item in peers], dtype=float) + row[f"pearson_pseudo_acc_{outcome}"] = _correlation(pseudo_acc, target, "pearson") + row[f"spearman_pseudo_acc_{outcome}"] = _correlation(pseudo_acc, target, "spearman") + row[f"pearson_confidence_{outcome}"] = _correlation(confidence, target, "pearson") + row[f"spearman_confidence_{outcome}"] = _correlation(confidence, target, "spearman") + row["pearson_cp"] = row["pearson_pseudo_acc_cp_acc"] + row["spearman_cp"] = row["spearman_pseudo_acc_cp_acc"] + row["pearson_pp"] = row["pearson_pseudo_acc_pp_acc"] + row["spearman_pp"] = row["spearman_pseudo_acc_pp_acc"] + if len(peers) < 2: + warnings.warn(f"Only one run is available for {row['dataset']}; correlations are NaN.", RuntimeWarning) + + outputs = config.get("outputs", {}) + samples_path = write_rows(sample_rows, outputs.get("samples", "results/analysis/pseudo_label_samples.csv")) + quality_path = write_rows(quality_rows, outputs.get("quality", "results/analysis/pseudo_label_quality.csv")) + logger.info("Wrote %s and %s", samples_path, quality_path) + + +if __name__ == "__main__": + main() diff --git a/run_revision_experiments.py b/run_revision_experiments.py new file mode 100644 index 0000000..8286ccd --- /dev/null +++ b/run_revision_experiments.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Master entry point for all DataCook revision experiments.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +from revision_experiments.common import load_config, resolve_path, run_prepare_commands, setup_logging + + +TASK_COMMANDS = { + "metrics": [["run_unified_evaluation.py"]], + "attacks": [["run_adaptive_attacks.py"]], + "finetune_attacks": [["run_adaptive_attacks.py"]], + "finetune_budget_attacks": [["run_adaptive_attacks.py"]], + "epsilon": [["run_epsilon_ablation.py"]], + "pseudo": [["run_pseudo_label_analysis.py"]], + "statistics": [["-m", "analysis.aggregate_results"], ["-m", "analysis.statistical_tests"]], + "runtime": [["run_runtime_analysis.py"]], + "image_quality": [["run_image_quality.py"]], + "final_analysis": [["-m", "analysis.finalize_revision_results"]], +} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", default="configs/revision.yaml") + parser.add_argument("--tasks", nargs="*", choices=TASK_COMMANDS, help="Run only these tasks; enabled flags are ignored.") + parser.add_argument("--prepare", action="store_true", help="Run each task config's explicit prepare_commands first.") + parser.add_argument("--continue-on-error", action="store_true") + parser.add_argument("--device") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("revision-master", args.verbose) + config, base_dir = load_config(args.config) + project_root = Path(__file__).resolve().parent + requested = set(args.tasks or []) + failures = [] + for task_name, commands in TASK_COMMANDS.items(): + task_config = config.get("tasks", {}).get(task_name, {}) + if args.tasks is None and not task_config.get("enabled", False): + logger.info("Skipping disabled task %s", task_name) + continue + if requested and task_name not in requested: + continue + if "config" not in task_config: + raise ValueError(f"Task {task_name!r} has no config path in the master config.") + task_config_path = resolve_path(task_config["config"], base_dir) + try: + if args.prepare: + child_config, child_base = load_config(task_config_path) + run_prepare_commands(child_config, child_base, logger) + for command in commands: + if command[0] == "-m": + full = [sys.executable, *command, "--config", str(task_config_path)] + else: + full = [sys.executable, str(project_root / command[0]), "--config", str(task_config_path)] + if args.verbose: + full.append("--verbose") + if args.device and command[0] in {"run_adaptive_attacks.py", "run_runtime_analysis.py"}: + full.extend(["--device", args.device]) + logger.info("Running task %s: %s", task_name, " ".join(full)) + subprocess.run(full, cwd=project_root, check=True) + except subprocess.CalledProcessError as exc: + failures.append((task_name, exc.returncode)) + logger.error("Task %s failed with exit code %s", task_name, exc.returncode) + if not args.continue_on_error: + raise + if failures: + raise SystemExit("Revision tasks failed: " + ", ".join(f"{name}({code})" for name, code in failures)) + logger.info("All selected revision tasks completed.") + + +if __name__ == "__main__": + main() diff --git a/run_runtime_analysis.py b/run_runtime_analysis.py new file mode 100644 index 0000000..d169f28 --- /dev/null +++ b/run_runtime_analysis.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Task 7: robust cooking and inference latency measurements.""" + +from __future__ import annotations + +import argparse +import time +from collections.abc import Mapping +from typing import Any + +import numpy as np +import torch +from torch.utils.data import DataLoader, TensorDataset + +from revision_experiments.common import load_array, load_config, metadata, require_records, setup_logging, write_rows +from revision_experiments.modeling import build_cooker, load_model + + +SECONDS_TO_MICROSECONDS = 1_000_000.0 +LATENCY_UNIT = "microseconds_per_sample" +PER_SAMPLE_LATENCY_COLUMNS = ( + "cooking_time_per_sample", + "raw_inference_time_per_sample", + "cooked_inference_time_per_sample", + "authorized_cooking_latency_per_sample", +) +MACRO_AVERAGE_COLUMNS = (*PER_SAMPLE_LATENCY_COLUMNS, "overhead_ratio", "gpu_memory_mb") + + +def _sync(device: torch.device) -> None: + if device.type == "cuda": + torch.cuda.synchronize(device) + + +def _time_inference(model, inputs: torch.Tensor, batch_size: int, device: torch.device, repeats: int, warmup: int) -> float: + loader = DataLoader(TensorDataset(inputs), batch_size=batch_size, shuffle=False) + model.eval() + elapsed = [] + with torch.no_grad(): + for run in range(warmup + repeats): + _sync(device) + start = time.perf_counter() + for (batch,) in loader: + model(batch.to(device, non_blocking=True)) + _sync(device) + if run >= warmup: + elapsed.append(time.perf_counter() - start) + return float(np.median(elapsed)) / len(inputs) + + +def _time_cooking(cooker, inputs: torch.Tensor, device: torch.device, repeats: int, warmup: int): + elapsed, output = [], None + batch = inputs.to(device) + with torch.no_grad(): + for run in range(warmup + repeats): + _sync(device) + start = time.perf_counter() + output = cooker(batch) + _sync(device) + if run >= warmup: + elapsed.append(time.perf_counter() - start) + if not torch.is_tensor(output): + output = torch.as_tensor(output, device=device) + return float(np.median(elapsed)), output.detach().cpu() + + +def _shared_value(rows: list[Mapping[str, Any]], key: str, fallback: Any = "mixed") -> Any: + """Return a shared metadata value, or ``fallback`` when rows differ.""" + values = [row.get(key, "") for row in rows] + return values[0] if values and all(value == values[0] for value in values) else fallback + + +def _append_macro_average(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Append an equal-dataset macro average to a multi-dataset runtime table.""" + if len(rows) < 2: + return rows + + average = { + "dataset": "Average", + "modality": _shared_value(rows, "modality"), + "task_type": _shared_value(rows, "task_type"), + "method": _shared_value(rows, "method"), + "surrogate_model": _shared_value(rows, "surrogate_model"), + "victim_model": _shared_value(rows, "victim_model"), + "seed": _shared_value(rows, "seed"), + "epsilon": _shared_value(rows, "epsilon"), + "setting": "runtime", + "attack_type": "", + "attack_strength": "", + "resolution": _shared_value(rows, "resolution"), + "num_samples": sum(int(row["num_samples"]) for row in rows), + "batch_size": _shared_value(rows, "batch_size"), + "device": _shared_value(rows, "device"), + "cooking_time_total_seconds": sum(float(row["cooking_time_total_seconds"]) for row in rows), + "latency_unit": LATENCY_UNIT, + "aggregation": "macro_average", + "repeats": _shared_value(rows, "repeats"), + "warmup": _shared_value(rows, "warmup"), + } + for column in MACRO_AVERAGE_COLUMNS: + average[column] = float(np.mean([float(row[column]) for row in rows])) + return [*rows, average] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("runtime-analysis", args.verbose) + config, base_dir = load_config(args.config) + device = torch.device(args.device) + rows = [] + for record in require_records(config): + inputs = torch.as_tensor(load_array(record["data"]["raw_inputs"], base_dir)).float() + if len(inputs) == 0: + raise ValueError("Runtime analysis requires at least one sample.") + batch_size = int(record.get("batch_size", 128)) + repeats, warmup = int(record.get("repeats", 5)), int(record.get("warmup", 1)) + cooker = build_cooker(record["cooker"], base_dir) + model = load_model(record["model"], base_dir, device) + if device.type == "cuda": + torch.cuda.reset_peak_memory_stats(device) + + cooking_total, generated_cooked = _time_cooking(cooker, inputs, device, repeats, warmup) + if "cooked_inputs" in record["data"]: + cooked = torch.as_tensor(load_array(record["data"]["cooked_inputs"], base_dir)).float() + else: + cooked = generated_cooked + if cooked.shape != inputs.shape: + raise ValueError("Cooked and raw inputs have different shapes in runtime analysis.") + raw_latency = _time_inference(model, inputs, batch_size, device, repeats, warmup) + cooked_latency = _time_inference(model, cooked, batch_size, device, repeats, warmup) + cooking_latency_seconds = cooking_total / len(inputs) + overhead = ( + (cooking_latency_seconds + cooked_latency) / raw_latency + if raw_latency > 0 else float("nan") + ) + memory = float(torch.cuda.max_memory_allocated(device) / 1024**2) if device.type == "cuda" else float("nan") + shape = "x".join(str(value) for value in inputs.shape[2:]) if inputs.ndim >= 4 else str(inputs.shape[-1]) + rows.append({ + **metadata(record, setting="runtime"), "resolution": record.get("resolution", shape), + "num_samples": len(inputs), "batch_size": batch_size, "device": str(device), + "cooking_time_total_seconds": cooking_total, + "cooking_time_per_sample": cooking_latency_seconds * SECONDS_TO_MICROSECONDS, + "raw_inference_time_per_sample": raw_latency * SECONDS_TO_MICROSECONDS, + "cooked_inference_time_per_sample": cooked_latency * SECONDS_TO_MICROSECONDS, + "authorized_cooking_latency_per_sample": cooking_latency_seconds * SECONDS_TO_MICROSECONDS, + "overhead_ratio": overhead, "gpu_memory_mb": memory, + "latency_unit": LATENCY_UNIT, "aggregation": "dataset", + "repeats": repeats, "warmup": warmup, + }) + logger.info( + "Timed %s: %.3f microseconds/sample cooking", + record["metadata"]["dataset"], + cooking_latency_seconds * SECONDS_TO_MICROSECONDS, + ) + if bool(config.get("include_average", True)): + rows = _append_macro_average(rows) + output = config.get("output", "results/runtime/runtime_summary.csv") + logger.info("Wrote %s", write_rows(rows, output)) + + +if __name__ == "__main__": + main() diff --git a/run_tabular_epsilon_ablation.py b/run_tabular_epsilon_ablation.py new file mode 100644 index 0000000..8b0281a --- /dev/null +++ b/run_tabular_epsilon_ablation.py @@ -0,0 +1,1009 @@ +#!/usr/bin/env python3 +"""Self-contained DataCook epsilon ablation for tabular classification. + +Binary, multiclass and multilabel datasets share the same experiment protocol. +Epsilon is measured in standardized-feature L-infinity units and must not be +interpreted as an image pixel budget. +""" + +from __future__ import annotations + +import argparse +import copy +import logging +import random +import time +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import torch +import torch.nn as nn +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler +from torch.utils.data import DataLoader, TensorDataset + +from metrics import compute_classification_metrics, compute_cp_pp +from revision_experiments.common import load_config, write_rows + + +PROJECT_ROOT = Path(__file__).resolve().parent +SETTINGS = ("clean", "authorized_cooked", "unauthorized_raw") + + +class TabularNet(nn.Module): + """Small MLP matching the architecture used by the legacy tabular scripts.""" + + def __init__( + self, + input_dim: int, + output_dim: int = 2, + hidden_dims: tuple[int, ...] = (64, 32), + dropout: float = 0.3, + ): + super().__init__() + layers: list[nn.Module] = [] + previous = input_dim + for hidden in hidden_dims: + layers.extend((nn.Linear(previous, hidden), nn.ReLU())) + if dropout > 0: + layers.append(nn.Dropout(dropout)) + previous = hidden + layers.append(nn.Linear(previous, output_dim)) + self.net = nn.Sequential(*layers) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return self.net(inputs) + + +def _seed_everything(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def _resolve(path: str | Path, base_dir: Path | None = None) -> Path: + candidate = Path(path).expanduser() + if candidate.is_absolute(): + return candidate.resolve() + return ((base_dir or PROJECT_ROOT) / candidate).resolve() + + +def _normalise_task_type(value: str) -> str: + task_type = value.lower().replace("-", "").replace("_", "") + aliases = { + "binary": "binary", + "binaryclass": "binary", + "multiclass": "multiclass", + "multilabel": "multilabel", + } + if task_type not in aliases: + raise ValueError( + f"Unsupported tabular task_type {value!r}; expected binary, " + "multiclass, or multilabel." + ) + return aliases[task_type] + + +def load_tabular_split( + data_path: Path, + *, + features: list[str], + targets: list[str], + task_type: str, + seed: int, + test_size: float, + categorical_mappings: dict[str, dict[Any, Any]] | None = None, + target_mapping: dict[Any, Any] | None = None, + stratify_split: bool = True, + preprocess_on_full_data: bool = False, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Load, split and standardize a tabular dataset.""" + if not data_path.is_file(): + raise FileNotFoundError( + f"Tabular dataset does not exist: {data_path}. Pass --data-path " + "or update dataset.data_path in the YAML configuration." + ) + frame = pd.read_csv(data_path) + missing = [ + column for column in [*features, *targets] + if column not in frame.columns + ] + if missing: + raise ValueError( + f"{data_path} is missing required columns: {missing}. " + f"Available columns: {list(frame.columns)}" + ) + + for column, mapping in (categorical_mappings or {}).items(): + if column not in frame: + raise ValueError(f"Categorical mapping references unknown column {column!r}.") + frame[column] = frame[column].map(mapping) + + inputs = frame[features].apply(pd.to_numeric, errors="coerce") + if task_type == "multilabel": + labels = ( + frame[targets] + .apply(pd.to_numeric, errors="raise") + .astype(int) + .to_numpy() + ) + if not set(np.unique(labels)).issubset({0, 1}): + raise ValueError("Multilabel targets must contain only 0 and 1.") + stratify = None + else: + if len(targets) != 1: + raise ValueError( + f"{task_type} requires exactly one target column, got {targets}." + ) + target_values = frame[targets[0]] + if target_mapping: + target_values = target_values.map(target_mapping) + if target_values.isna().any(): + unknown = sorted(frame.loc[target_values.isna(), targets[0]].astype(str).unique()) + raise ValueError( + f"Target mapping for {targets[0]!r} did not cover values: {unknown}." + ) + labels = pd.to_numeric(target_values, errors="raise").astype(int).to_numpy() + unique_labels = sorted(np.unique(labels).tolist()) + expected = list(range(len(unique_labels))) + if unique_labels != expected: + raise ValueError( + f"{targets[0]} labels must be contiguous integers starting at 0; " + f"found {unique_labels}." + ) + if task_type == "binary" and unique_labels != [0, 1]: + raise ValueError(f"{targets[0]} must contain binary labels 0 and 1.") + stratify = labels if stratify_split else None + + if not 0 < test_size < 1: + raise ValueError("test_size must be in (0,1).") + + if preprocess_on_full_data: + # Historical compatibility: the original tabular scripts imputed and + # standardized the complete dataset before the train/test split. + full_medians = inputs.median(numeric_only=True) + inputs = inputs.fillna(full_medians) + if inputs.isna().any().any(): + raise ValueError( + f"{data_path} contains feature columns that cannot be converted to numbers." + ) + scaler = StandardScaler() + # Match the historical scripts: StandardScaler receives pandas' native + # float64 array, and conversion to torch/float32 happens only afterwards. + scaled = scaler.fit_transform(inputs.to_numpy()).astype(np.float32) + x_train, x_test, y_train, y_test = train_test_split( + scaled, + labels, + test_size=test_size, + random_state=seed, + stratify=stratify, + ) + else: + x_train, x_test, y_train, y_test = train_test_split( + inputs, + labels, + test_size=test_size, + random_state=seed, + stratify=stratify, + ) + train_medians = x_train.median(numeric_only=True) + x_train = x_train.fillna(train_medians) + x_test = x_test.fillna(train_medians) + if x_train.isna().any().any() or x_test.isna().any().any(): + raise ValueError( + f"{data_path} contains feature columns that cannot be converted to numbers." + ) + scaler = StandardScaler() + x_train = scaler.fit_transform(x_train.to_numpy(dtype=np.float32)).astype(np.float32) + x_test = scaler.transform(x_test.to_numpy(dtype=np.float32)).astype(np.float32) + all_scaled = np.concatenate((x_train, x_test), axis=0) + return ( + torch.from_numpy(x_train), + torch.from_numpy( + y_train.astype(np.float32) + if task_type == "multilabel" + else y_train.astype(np.int64) + ), + torch.from_numpy(x_test), + torch.from_numpy( + y_test.astype(np.float32) + if task_type == "multilabel" + else y_test.astype(np.int64) + ), + torch.from_numpy(all_scaled.min(axis=0).astype(np.float32)), + torch.from_numpy(all_scaled.max(axis=0).astype(np.float32)), + ) + + +def _train_model( + model: nn.Module, + inputs: torch.Tensor, + labels: torch.Tensor, + *, + device: torch.device, + epochs: int, + batch_size: int, + learning_rate: float, + seed: int, + task_type: str, + class_weights: torch.Tensor | None = None, + shuffle: bool = True, + validation_inputs: torch.Tensor | None = None, + validation_labels: torch.Tensor | None = None, + select_best_on_validation: bool = False, + use_global_data_order: bool = False, +) -> nn.Module: + if epochs <= 0 or batch_size <= 0 or learning_rate <= 0: + raise ValueError("epochs, batch_size and learning_rate must be positive.") + generator = None if use_global_data_order else torch.Generator().manual_seed(seed) + loader = DataLoader( + TensorDataset(inputs, labels), + batch_size=batch_size, + shuffle=shuffle, + generator=generator, + ) + model = model.to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) + if task_type == "multilabel": + criterion: nn.Module = nn.BCEWithLogitsLoss( + pos_weight=class_weights.to(device) if class_weights is not None else None + ) + else: + criterion = nn.CrossEntropyLoss( + weight=class_weights.to(device) if class_weights is not None else None + ) + if select_best_on_validation and ( + validation_inputs is None or validation_labels is None + ): + raise ValueError( + "select_best_on_validation requires validation inputs and labels." + ) + best_score = float("-inf") + best_state: dict[str, torch.Tensor] | None = None + for _ in range(epochs): + model.train() + for batch_inputs, batch_labels in loader: + optimizer.zero_grad(set_to_none=True) + logits = model(batch_inputs.to(device)) + labels_device = batch_labels.to(device) + if task_type != "multilabel": + labels_device = labels_device.reshape(-1) + loss = criterion(logits, labels_device) + loss.backward() + optimizer.step() + if select_best_on_validation: + model.eval() + with torch.no_grad(): + logits = model(validation_inputs.to(device)) + if task_type == "multilabel": + predictions = (logits >= 0).to(torch.int64).cpu() + truth = validation_labels.to(torch.int64).cpu() + score = float((predictions == truth).all(dim=1).float().mean()) + else: + predictions = logits.argmax(dim=1).cpu() + truth = validation_labels.reshape(-1).to(torch.int64).cpu() + score = float((predictions == truth).float().mean()) + if score > best_score: + best_score = score + best_state = copy.deepcopy(model.state_dict()) + if best_state is not None: + model.load_state_dict(best_state) + return model.eval() + + +def _probabilities( + model: nn.Module, + inputs: torch.Tensor, + *, + device: torch.device, + batch_size: int, + task_type: str = "binary", +) -> np.ndarray: + loader = DataLoader(TensorDataset(inputs), batch_size=batch_size, shuffle=False) + batches = [] + model.eval() + with torch.no_grad(): + for (batch,) in loader: + logits = model(batch.to(device)) + probability = ( + torch.sigmoid(logits) + if task_type == "multilabel" + else torch.softmax(logits, dim=1) + ) + batches.append(probability.cpu()) + return torch.cat(batches).numpy() + + +def cook_tabular_inputs( + model: nn.Module, + inputs: torch.Tensor, + *, + epsilon: float, + attackable_indices: list[int], + x_min: torch.Tensor, + x_max: torch.Tensor, + num_steps: int, + device: torch.device, + step_size_ratio: float | None = None, + step_size: float | None = None, + task_type: str = "binary", +) -> tuple[torch.Tensor, float]: + """Generate DataCook error-minimizing perturbations for a whole split.""" + if epsilon <= 0: + raise ValueError("Tabular epsilon must be positive.") + if num_steps <= 0: + raise ValueError("num_steps must be positive.") + if step_size is not None and step_size_ratio is not None: + raise ValueError("Configure either step_size or step_size_ratio, not both.") + if step_size is None and step_size_ratio is None: + raise ValueError("Either step_size or step_size_ratio must be configured.") + if step_size is not None and step_size <= 0: + raise ValueError("step_size must be positive.") + if step_size_ratio is not None and step_size_ratio <= 0: + raise ValueError("step_size_ratio must be positive.") + if not attackable_indices: + raise ValueError("At least one attackable feature is required.") + if min(attackable_indices) < 0 or max(attackable_indices) >= inputs.shape[1]: + raise ValueError("An attackable feature index is outside the input feature range.") + + original = inputs.to(device) + lower = x_min.to(device) + upper = x_max.to(device) + mask = torch.zeros_like(original) + mask[:, attackable_indices] = 1.0 + model.eval() + with torch.no_grad(): + logits = model(original) + pseudo_labels = ( + (logits >= 0).to(logits.dtype) + if task_type == "multilabel" + else logits.argmax(dim=1) + ) + + cooked = original.clone().detach() + effective_step_size = ( + float(step_size) + if step_size is not None + else float(epsilon) * float(step_size_ratio) + ) + if device.type == "cuda": + torch.cuda.synchronize(device) + started = time.perf_counter() + for _ in range(num_steps): + cooked.requires_grad_(True) + logits = model(cooked) + loss = ( + nn.functional.binary_cross_entropy_with_logits(logits, pseudo_labels) + if task_type == "multilabel" + else nn.functional.cross_entropy(logits, pseudo_labels) + ) + gradient = torch.autograd.grad(loss, cooked, only_inputs=True)[0] + with torch.no_grad(): + # DataCook uses the error-minimizing direction (-sign(gradient)). + cooked = cooked - effective_step_size * gradient.sign() * mask + delta = torch.clamp(cooked - original, -float(epsilon), float(epsilon)) + cooked = torch.maximum(torch.minimum(original + delta, upper), lower) + if device.type == "cuda": + torch.cuda.synchronize(device) + elapsed = time.perf_counter() - started + return cooked.detach().cpu(), elapsed + + +def _atomic_npz(path: Path, **arrays: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + ".tmp.npz") + np.savez_compressed(temporary, **arrays) + temporary.replace(path) + + +def _state_dict_cpu(model: nn.Module) -> dict[str, torch.Tensor]: + return { + name: value.detach().cpu() + for name, value in model.state_dict().items() + } + + +def _metadata( + dataset: str, + seed: int, + epsilon: float, + setting: str, + task_type: str, +) -> dict[str, Any]: + return { + "dataset": dataset, + "modality": "tabular", + "task_type": task_type, + "method": "datacook", + "surrogate_model": "mlp", + "victim_model": "mlp", + "seed": seed, + "epsilon": epsilon, + "epsilon_scale": "standardized_feature_linf", + "setting": setting, + "attack_type": "", + "attack_strength": "", + } + + +def _run_name(dataset: str, seed: int, epsilon: float) -> str: + return f"{dataset}-datacook-sur=mlp-vic=mlp-seed={seed}-eps={epsilon:g}" + + +def _read_rows(path: Path) -> list[dict[str, Any]]: + return pd.read_csv(path).to_dict(orient="records") + + +def run_ablation(config: dict[str, Any], base_dir: Path, args) -> dict[str, Path]: + dataset_cfg = config.get("dataset", {}) + experiment_cfg = config.get("experiment", {}) + model_cfg = config.get("model", {}) + output_cfg = config.get("outputs", {}) + dataset_name = str(dataset_cfg.get("name", "Diabetes")) + task_type = _normalise_task_type(dataset_cfg.get("task_type", "binary")) + features = list(dataset_cfg.get("features", [])) + configured_targets = dataset_cfg.get( + "targets", [dataset_cfg.get("target", "Outcome")] + ) + targets = ( + [str(configured_targets)] + if isinstance(configured_targets, str) + else [str(value) for value in configured_targets] + ) + if not features: + raise ValueError("dataset.features must be a non-empty list.") + if not targets: + raise ValueError("dataset.target or dataset.targets must be configured.") + data_path = _resolve( + args.data_path or dataset_cfg.get("data_path", "TableData/dataset/diabetes.csv"), + base_dir if not args.data_path else PROJECT_ROOT, + ) + seeds = list(args.seeds or experiment_cfg.get("seeds", [0, 1, 2])) + epsilons = list(args.epsilons or experiment_cfg.get("epsilons", [0.1, 0.25, 0.5, 1.0, 2.0])) + if not seeds or not epsilons: + raise ValueError("At least one seed and epsilon are required.") + if len(set(seeds)) != len(seeds) or len(set(epsilons)) != len(epsilons): + raise ValueError("Seeds and epsilons must not contain duplicates.") + + attackable_features = list(experiment_cfg.get( + "attackable_features", ["BMI", "DiabetesPedigreeFunction"] + )) + missing_attackable = [name for name in attackable_features if name not in features] + if missing_attackable: + raise ValueError(f"Unknown attackable features: {missing_attackable}") + attackable_indices = [features.index(name) for name in attackable_features] + device = torch.device(args.device) + if device.type == "cuda" and not torch.cuda.is_available(): + raise RuntimeError("CUDA was requested but is not available.") + + artifact_root = _resolve( + getattr(args, "artifact_root", None) + or output_cfg.get("artifact_root", "artifacts/revision_pipeline_tabular") + ) + result_root = _resolve( + getattr(args, "result_root", None) + or output_cfg.get("result_root", "results/revision_pipeline_tabular") + ) + requested = len(seeds) * len(epsilons) + logging.info( + "Tabular preflight: dataset=%s path=%s rows_expected_runs=%d device=%s", + dataset_name, data_path, requested, device, + ) + if not data_path.is_file(): + raise FileNotFoundError( + f"Missing tabular dataset: {data_path}. " + "Set TABULAR_DATA in the job or pass --data-path." + ) + if args.dry_run: + frame = pd.read_csv(data_path, nrows=5) + missing = [ + column for column in [*features, *targets] + if column not in frame.columns + ] + if missing: + raise ValueError(f"Dataset preflight failed; missing columns: {missing}") + logging.info("Dry run complete; no training or output was performed.") + return {} + + hidden_dims = tuple(int(value) for value in model_cfg.get("hidden_dims", [64, 32])) + dropout = float(model_cfg.get("dropout", 0.3)) + configured_baseline_checkpoint = ( + getattr(args, "baseline_checkpoint", None) + or model_cfg.get("baseline_checkpoint") + ) + baseline_checkpoint = ( + _resolve( + configured_baseline_checkpoint, + PROJECT_ROOT if getattr(args, "baseline_checkpoint", None) else base_dir, + ) + if configured_baseline_checkpoint else None + ) + require_baseline_checkpoint = bool( + model_cfg.get("require_baseline_checkpoint", False) + ) + if baseline_checkpoint is not None and not baseline_checkpoint.is_file(): + raise FileNotFoundError( + f"Configured baseline checkpoint does not exist: {baseline_checkpoint}" + ) + if require_baseline_checkpoint and baseline_checkpoint is None: + raise ValueError( + "model.require_baseline_checkpoint=true but no baseline checkpoint " + "was configured or passed with --baseline-checkpoint." + ) + baseline_epochs = int(experiment_cfg.get("baseline_epochs", 100)) + protected_epochs = int(experiment_cfg.get("protected_epochs", 100)) + batch_size = int(experiment_cfg.get("batch_size", 32)) + learning_rate = float(experiment_cfg.get("learning_rate", 1.0e-3)) + num_steps = int(experiment_cfg.get("num_steps", 20)) + configured_step_size = experiment_cfg.get("step_size") + configured_step_size_ratio = experiment_cfg.get("step_size_ratio") + if configured_step_size is not None and configured_step_size_ratio is not None: + raise ValueError( + "experiment.step_size and experiment.step_size_ratio are mutually exclusive." + ) + if configured_step_size is None and configured_step_size_ratio is None: + configured_step_size_ratio = 0.25 + step_size = ( + float(configured_step_size) + if configured_step_size is not None else None + ) + step_size_ratio = ( + float(configured_step_size_ratio) + if configured_step_size_ratio is not None else None + ) + test_size = float(dataset_cfg.get("test_size", 0.2)) + categorical_mappings = dict(dataset_cfg.get("categorical_mappings", {})) + target_mapping = dataset_cfg.get("target_mapping") + class_balance = bool(experiment_cfg.get("class_balance", False)) + max_positive_weight = float(experiment_cfg.get("max_positive_weight", 100.0)) + split_seed = experiment_cfg.get("split_seed") + stratify_split = bool(experiment_cfg.get("stratify_split", True)) + preprocess_on_full_data = bool( + experiment_cfg.get("preprocess_on_full_data", False) + ) + baseline_select_best_on_test = bool( + experiment_cfg.get("baseline_select_best_on_test", False) + ) + baseline_shuffle = bool(experiment_cfg.get("baseline_shuffle", True)) + protected_shuffle = bool(experiment_cfg.get("protected_shuffle", True)) + configured_victim_seed = experiment_cfg.get("victim_seed") + configured_protected_seed = experiment_cfg.get("protected_seed") + victim_seed_offset = int(experiment_cfg.get("victim_seed_offset", 100_000)) + protected_seed_offset = int(experiment_cfg.get("protected_seed_offset", 0)) + reset_rng_each_epsilon = bool( + experiment_cfg.get("reset_rng_each_epsilon", False) + ) + victim_initialization_count = int( + experiment_cfg.get("victim_initialization_count", 1) + ) + victim_deepcopy_template = bool( + experiment_cfg.get("victim_deepcopy_template", False) + ) + historical_loader_rng_draws = int( + experiment_cfg.get("historical_loader_rng_draws_before_training", 0) + ) + if historical_loader_rng_draws < 0: + raise ValueError( + "experiment.historical_loader_rng_draws_before_training must be non-negative." + ) + if victim_initialization_count <= 0: + raise ValueError("experiment.victim_initialization_count must be positive.") + baseline_global_data_order = bool( + experiment_cfg.get("baseline_global_data_order", False) + ) + protected_global_data_order = bool( + experiment_cfg.get("protected_global_data_order", False) + ) + + metric_rows: list[dict[str, Any]] = [] + cp_rows: list[dict[str, Any]] = [] + epsilon_rows: list[dict[str, Any]] = [] + runtime_rows: list[dict[str, Any]] = [] + for seed in seeds: + seed = int(seed) + split = load_tabular_split( + data_path, + features=features, + targets=targets, + task_type=task_type, + seed=int(split_seed) if split_seed is not None else seed, + test_size=test_size, + categorical_mappings=categorical_mappings, + target_mapping=target_mapping, + stratify_split=stratify_split, + preprocess_on_full_data=preprocess_on_full_data, + ) + x_train, y_train, x_test, y_test, x_min, x_max = split + if task_type == "multilabel": + num_outputs = len(targets) + else: + observed_classes = int(torch.max(torch.cat((y_train, y_test))).item()) + 1 + num_outputs = int(dataset_cfg.get("num_classes", observed_classes)) + if num_outputs < observed_classes: + raise ValueError( + f"dataset.num_classes={num_outputs} is smaller than the " + f"{observed_classes} observed classes." + ) + + class_weights = None + if class_balance: + if task_type == "multilabel": + positives = y_train.sum(dim=0) + negatives = y_train.shape[0] - positives + if torch.any(positives == 0): + raise ValueError( + f"{dataset_name} has a training label with no positive samples." + ) + class_weights = torch.clamp( + negatives / positives, + max=max_positive_weight, + ).to(torch.float32) + else: + counts = torch.bincount(y_train, minlength=num_outputs).to(torch.float32) + if torch.any(counts == 0): + raise ValueError( + f"{dataset_name} has a training class with no samples." + ) + class_weights = y_train.numel() / (num_outputs * counts) + + baseline_dir = artifact_root / f"{dataset_name}-seed={seed}" + baseline_path = baseline_dir / "clean_model.pth" + _seed_everything(seed) + baseline = TabularNet( + len(features), + output_dim=num_outputs, + hidden_dims=hidden_dims, + dropout=dropout, + ) + if baseline_checkpoint is not None: + checkpoint = torch.load(baseline_checkpoint, map_location="cpu") + state_dict = ( + checkpoint["state_dict"] + if isinstance(checkpoint, dict) and "state_dict" in checkpoint + else checkpoint + ) + baseline.load_state_dict(state_dict) + baseline = baseline.to(device).eval() + logging.info("Loaded historical baseline: %s", baseline_checkpoint) + baseline_dir.mkdir(parents=True, exist_ok=True) + torch.save(_state_dict_cpu(baseline), baseline_path) + elif baseline_path.is_file() and not args.force: + baseline.load_state_dict(torch.load(baseline_path, map_location="cpu")) + baseline = baseline.to(device).eval() + logging.info("Reusing baseline: %s", baseline_path) + else: + baseline = _train_model( + baseline, x_train, y_train, device=device, + epochs=baseline_epochs, batch_size=batch_size, + learning_rate=learning_rate, seed=seed, + task_type=task_type, class_weights=class_weights, + shuffle=baseline_shuffle, + validation_inputs=x_test, + validation_labels=y_test, + select_best_on_validation=baseline_select_best_on_test, + use_global_data_order=baseline_global_data_order, + ) + baseline_dir.mkdir(parents=True, exist_ok=True) + torch.save(_state_dict_cpu(baseline), baseline_path) + clean_prob = _probabilities( + baseline, + x_test, + device=device, + batch_size=batch_size, + task_type=task_type, + ) + clean_metrics = compute_classification_metrics( + y_test.numpy(), y_prob=clean_prob, task_type=task_type + ) + + victim_seed = ( + int(configured_victim_seed) + if configured_victim_seed is not None else seed + victim_seed_offset + ) + protected_seed = ( + int(configured_protected_seed) + if configured_protected_seed is not None else seed + protected_seed_offset + ) + _seed_everything(victim_seed) + victim_template = None + for _ in range(victim_initialization_count): + victim_template = TabularNet( + len(features), + output_dim=num_outputs, + hidden_dims=hidden_dims, + dropout=dropout, + ) + assert victim_template is not None + victim_initial_state = copy.deepcopy(victim_template.state_dict()) + paired_cpu_rng_state = torch.random.get_rng_state() + paired_cuda_rng_states = ( + torch.cuda.get_rng_state_all() if device.type == "cuda" else None + ) + + for epsilon in epsilons: + epsilon = float(epsilon) + if reset_rng_each_epsilon: + # Epsilon is the only changing factor within a seed. Restoring + # the post-initialization RNG state prevents budget order from + # changing dropout masks or shuffled minibatches. + torch.random.set_rng_state(paired_cpu_rng_state) + if paired_cuda_rng_states is not None: + torch.cuda.set_rng_state_all(paired_cuda_rng_states) + effective_step_size = ( + step_size + if step_size is not None + else epsilon * float(step_size_ratio) + ) + attack_parameters = { + "num_steps": num_steps, + "step_size": effective_step_size, + "step_size_ratio": ( + step_size_ratio if step_size is None else float("nan") + ), + } + run_name = _run_name(dataset_name, seed, epsilon) + run_artifacts = artifact_root / run_name + run_results = result_root / run_name / "metrics" + per_run_files = { + "main": run_results / "main_metrics.csv", + "cp": run_results / "cp_pp_summary.csv", + "epsilon": run_results / "epsilon_ablation.csv", + "runtime": run_results / "runtime_summary.csv", + } + prediction_files = { + setting: run_artifacts / "predictions" / f"{setting}.npz" + for setting in SETTINGS + } + ready = all(path.is_file() for path in [ + *per_run_files.values(), *prediction_files.values(), + ]) + if ready and not args.force: + logging.info("Reusing completed tabular run: %s", run_name) + metric_rows.extend(_read_rows(per_run_files["main"])) + cp_rows.extend(_read_rows(per_run_files["cp"])) + epsilon_rows.extend(_read_rows(per_run_files["epsilon"])) + runtime_rows.extend(_read_rows(per_run_files["runtime"])) + continue + + logging.info("Running %s", run_name) + cooked_train, train_cooking_time = cook_tabular_inputs( + baseline, x_train, epsilon=epsilon, + attackable_indices=attackable_indices, + x_min=x_min, x_max=x_max, num_steps=num_steps, + step_size_ratio=step_size_ratio, step_size=step_size, + device=device, + task_type=task_type, + ) + cooked_test, test_cooking_time = cook_tabular_inputs( + baseline, x_test, epsilon=epsilon, + attackable_indices=attackable_indices, + x_min=x_min, x_max=x_max, num_steps=num_steps, + step_size_ratio=step_size_ratio, step_size=step_size, + device=device, + task_type=task_type, + ) + _atomic_npz( + run_artifacts / "cooked_train.npz", + raw=x_train.numpy(), cooked=cooked_train.numpy(), + labels=y_train.numpy(), + ) + _atomic_npz( + run_artifacts / "cooked_test.npz", + raw=x_test.numpy(), cooked=cooked_test.numpy(), + labels=y_test.numpy(), + ) + + if victim_deepcopy_template: + # Historical PrivacyProtector uses copy.deepcopy(raw_model). + # Unlike constructing and then loading, this does not advance + # the global RNG before shuffled batches and dropout training. + victim = copy.deepcopy(victim_template) + else: + victim = TabularNet( + len(features), + output_dim=num_outputs, + hidden_dims=hidden_dims, + dropout=dropout, + ) + victim.load_state_dict(victim_initial_state) + # Constructing a DataLoader iterator advances the global torch RNG + # even with shuffle=False. The old scripts iterated the raw train + # and test loaders once to obtain pseudo-labels before training. + for _ in range(historical_loader_rng_draws): + torch.empty((), dtype=torch.int64).random_() + if device.type == "cuda": + torch.cuda.synchronize(device) + training_started = time.perf_counter() + victim = _train_model( + victim, cooked_train, y_train, device=device, + epochs=protected_epochs, batch_size=batch_size, + learning_rate=learning_rate, seed=protected_seed, + task_type=task_type, class_weights=class_weights, + shuffle=protected_shuffle, + use_global_data_order=protected_global_data_order, + ) + if device.type == "cuda": + torch.cuda.synchronize(device) + protected_training_time = time.perf_counter() - training_started + torch.save(_state_dict_cpu(victim), run_artifacts / "protected_model.pth") + + inference_times: dict[str, float] = {} + probabilities: dict[str, np.ndarray] = {"clean": clean_prob} + for setting, inputs in ( + ("authorized_cooked", cooked_test), + ("unauthorized_raw", x_test), + ): + if device.type == "cuda": + torch.cuda.synchronize(device) + started = time.perf_counter() + probabilities[setting] = _probabilities( + victim, + inputs, + device=device, + batch_size=batch_size, + task_type=task_type, + ) + if device.type == "cuda": + torch.cuda.synchronize(device) + inference_times[setting] = ( + time.perf_counter() - started + ) / len(inputs) + for setting, probability in probabilities.items(): + predictions = ( + (probability >= 0.5).astype(np.int64) + if task_type == "multilabel" + else np.argmax(probability, axis=1) + ) + _atomic_npz( + prediction_files[setting], + y_true=y_test.numpy(), + y_pred=predictions, + y_prob=probability, + ) + + all_metrics = {"clean": clean_metrics} + for setting in ("authorized_cooked", "unauthorized_raw"): + all_metrics[setting] = compute_classification_metrics( + y_test.numpy(), y_prob=probabilities[setting], + task_type=task_type, + ) + requested_metrics = sorted( + set.intersection(*(set(values) for values in all_metrics.values())) + ) + cp_pp = compute_cp_pp( + all_metrics["clean"], + all_metrics["authorized_cooked"], + all_metrics["unauthorized_raw"], + requested_metrics, + task_type=task_type, + num_classes=num_outputs, + label_prevalence=( + y_test.to(torch.float32).mean(dim=0).tolist() + if task_type == "multilabel" + else None + ), + ) + run_metric_rows = [] + for setting, values in all_metrics.items(): + run_metric_rows.extend({ + **_metadata( + dataset_name, seed, epsilon, setting, task_type + ), + **attack_parameters, + "metric": metric, + "value": value, + } for metric, value in values.items()) + run_cp_rows = [{ + **_metadata( + dataset_name, seed, epsilon, "cp_pp", task_type + ), + **attack_parameters, + "metric": metric, + **values, + } for metric, values in cp_pp.items()] + run_epsilon_rows = [{ + **_metadata( + dataset_name, + seed, + epsilon, + "epsilon_ablation", + task_type, + ), + **attack_parameters, + "metric": metric, + **values, + } for metric, values in cp_pp.items()] + run_runtime_rows = [{ + **_metadata( + dataset_name, seed, epsilon, "runtime", task_type + ), + "resolution": f"{len(features)}_features", + "num_samples": len(x_test), + "num_train_samples": len(x_train), + "num_test_samples": len(x_test), + "batch_size": batch_size, + "device": str(device), + "attackable_features": ",".join(attackable_features), + **attack_parameters, + "train_cooking_time_total": train_cooking_time, + "train_cooking_time_per_sample": train_cooking_time / len(x_train), + "test_cooking_time_total": test_cooking_time, + "test_cooking_time_per_sample": test_cooking_time / len(x_test), + "cooking_time_total": test_cooking_time, + "cooking_time_per_sample": test_cooking_time / len(x_test), + "authorized_cooking_latency_per_sample": test_cooking_time / len(x_test), + "raw_inference_time_per_sample": inference_times["unauthorized_raw"], + "cooked_inference_time_per_sample": inference_times["authorized_cooked"], + "overhead_ratio": ( + test_cooking_time / len(x_test) + + inference_times["authorized_cooked"] + ) / inference_times["unauthorized_raw"] + if inference_times["unauthorized_raw"] > 0 else float("nan"), + "protected_training_time": protected_training_time, + "gpu_memory_mb": ( + float(torch.cuda.max_memory_allocated(device) / 1024**2) + if device.type == "cuda" else float("nan") + ), + "repeats": 1, + "warmup": 0, + }] + write_rows(run_metric_rows, per_run_files["main"]) + write_rows(run_cp_rows, per_run_files["cp"]) + write_rows(run_epsilon_rows, per_run_files["epsilon"]) + write_rows(run_runtime_rows, per_run_files["runtime"]) + metric_rows.extend(run_metric_rows) + cp_rows.extend(run_cp_rows) + epsilon_rows.extend(run_epsilon_rows) + runtime_rows.extend(run_runtime_rows) + + outputs = { + "main_metrics": result_root / "summary/main_metrics.csv", + "cp_pp": result_root / "summary/cp_pp_summary.csv", + "epsilon": result_root / "epsilon_ablation.csv", + "runtime": result_root / "runtime_summary.csv", + } + write_rows(metric_rows, outputs["main_metrics"]) + write_rows(cp_rows, outputs["cp_pp"]) + write_rows(epsilon_rows, outputs["epsilon"]) + write_rows(runtime_rows, outputs["runtime"]) + logging.info("Completed %d tabular runs. Results: %s", requested, result_root) + return outputs + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", + default="configs/revision/tabular_epsilon_diabetes.yaml", + ) + parser.add_argument("--data-path", help="Override dataset.data_path.") + parser.add_argument("--artifact-root", help="Override outputs.artifact_root.") + parser.add_argument("--result-root", help="Override outputs.result_root.") + parser.add_argument( + "--baseline-checkpoint", + help="Load a historical surrogate checkpoint instead of retraining it.", + ) + parser.add_argument("--seeds", nargs="+", type=int) + parser.add_argument("--epsilons", nargs="+", type=float) + parser.add_argument( + "--device", default="cuda" if torch.cuda.is_available() else "cpu" + ) + parser.add_argument("--force", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(asctime)s | %(levelname)s | tabular-epsilon | %(message)s", + ) + config, base_dir = load_config(args.config) + run_ablation(config, base_dir, args) + + +if __name__ == "__main__": + main() diff --git a/run_unified_evaluation.py b/run_unified_evaluation.py new file mode 100644 index 0000000..505e6e6 --- /dev/null +++ b/run_unified_evaluation.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Task 1/2: unified metrics and generalized CP/PP from prediction artifacts.""" + +from __future__ import annotations + +import argparse + +from metrics import compute_classification_metrics, compute_cp_pp +from revision_experiments.common import ( + load_config, + load_prediction_artifact, + metadata, + require_records, + setup_logging, + write_rows, +) + + +SETTINGS = ("clean", "authorized_cooked", "unauthorized_raw") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--verbose", action="store_true") + args = parser.parse_args() + logger = setup_logging("unified-evaluation", args.verbose) + config, base_dir = load_config(args.config) + metric_rows, cp_rows = [], [] + + for index, record in enumerate(require_records(config)): + task_type = record.get("task_type", record.get("metadata", {}).get("task_type")) + predictions = record.get("predictions", {}) + missing = [setting for setting in SETTINGS if setting not in predictions] + if missing: + raise ValueError(f"Experiment {index} is missing prediction settings: {missing}.") + logger.info("Evaluating %s (%s)", record.get("metadata", {}).get("dataset", index), task_type) + all_metrics = {} + clean_y_true = None + for setting in SETTINGS: + artifact = load_prediction_artifact(predictions[setting], base_dir) + if setting == "clean": + clean_y_true = artifact["y_true"] + values = compute_classification_metrics( + artifact["y_true"], artifact["y_pred"], artifact["y_prob"], + task_type=task_type, threshold=float(record.get("threshold", 0.5)), + ) + all_metrics[setting] = values + meta = metadata(record, setting=setting) + metric_rows.extend({**meta, "metric": name, "value": value} for name, value in values.items()) + + requested = record.get("metric_names") or sorted(set.intersection(*(set(values) for values in all_metrics.values()))) + label_prevalence = record.get("label_prevalence") + normalized_task = str(task_type).replace("-", "").replace("_", "").lower() + if label_prevalence is None and normalized_task == "multilabel": + import numpy as np + + truth = np.asarray(clean_y_true) + if truth.ndim != 2: + raise ValueError("Multi-label clean y_true must have shape (N,L).") + label_prevalence = truth.mean(axis=0).tolist() + cp_pp = compute_cp_pp( + all_metrics["clean"], all_metrics["authorized_cooked"], all_metrics["unauthorized_raw"], requested, + record.get("random_guess_metrics"), task_type=task_type, + num_classes=record.get("num_classes"), label_prevalence=label_prevalence, + ) + base_meta = metadata(record, setting="cp_pp") + cp_rows.extend({**base_meta, "metric": name, **values} for name, values in cp_pp.items()) + + outputs = config.get("outputs", {}) + metrics_path = write_rows(metric_rows, outputs.get("main_metrics", "results/metrics/main_metrics.csv")) + cp_path = write_rows(cp_rows, outputs.get("cp_pp", "results/metrics/cp_pp_summary.csv")) + logger.info("Wrote %s and %s", metrics_path, cp_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/.DS_Store b/scripts/.DS_Store deleted file mode 100644 index 9c38f2bffdad30ba545b84382021e8951c67ca01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJx{|h5Iv`rC?7*d286P(B9&i|mUKiys+cKl5lFO3rKV*K|BZnS@sD_Swn|Lw zZXks2D*GjOXWz?`_nF@XmQLA7^&_SUaQXFH^qm*of@4FO@?uwRglBZ_MG@p1&G&hWo(T=< z%GYr70gmF2@NuzVZHWcy>~kGxa!o6`r!|x*{Ho8Vs2a$Ma0q4w83i%}{8}^zv!%~z z^$Jcd65+9p25Y-$pk6{Zbm>)VL#tvBTj{U0&v}vS^2s zk;<6IRW|N~Vx&6!sic$1ENZU`r~S5T1=i3nKLBaW7uQ))!bye1g7!)PPbN(u2M4*KF5r^1(aeMRr(C4&WGwD0Le%Y@gzo>J6TIXul+b8QFXR;1_gF14@IxfzD zGvEw31J1xt44`LI=9h*(I|I&uGw{g(&xeRXOa-%{9vztU3ILqJtO8yCb%BuvU@Djm zVSxlo1zKt-SBzlku*VXY3T8t~m!OrgPpRBrUV>JKJ!-gwWazUq;0z2IIJ4z~{r?f4 z%;+V5nBpsEz!~^w41`&|sn>X^(p$g19(!%Xc*3ACE>nR(*H#SFR*VJDk!y9DKT?N3 X70ia*g?3E``j3DT;*&G*3k-Y!*>5tM diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..716156c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,18 @@ +# Historical launchers + +This directory contains the original per-dataset shell launchers for DataCook, +DataCook (Adv), and the comparison methods. They are retained temporarily for +historical traceability and baseline migration. + +For current DataCook experiments, use the validated YAML-driven entry points +documented in the repository root: + +```bash +python run_datacook_pipeline.py --config configs/pipelines/bloodmnist.yaml +python run_datacook_suite.py --config configs/pipelines/medmnist_three.yaml +``` + +The historical scripts are not the source of truth for the accepted-paper +experiments. Do not mix their arguments or output layouts with the unified +pipeline. They can be removed after every comparison-method launcher needed for +reproduction has been migrated and validated against its historical output. diff --git a/summarize_tabular_legacy_reproduction.py b/summarize_tabular_legacy_reproduction.py new file mode 100755 index 0000000..961f8aa --- /dev/null +++ b/summarize_tabular_legacy_reproduction.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Compare strict legacy tabular reproduction results with the old table.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import pandas as pd + + +TARGETS = { + "Diabetes": {"metric": "acc", "cp": -18.18, "pp": 1.31}, + "HepatitisC": {"metric": "acc", "cp": -31.71, "pp": 0.00}, + "SystemicDiseases": {"metric": "subset_acc", "cp": -50.22, "pp": -0.45}, +} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--result-root", + default="results/revision_pipeline_tabular_legacy_exact", + ) + parser.add_argument("--tolerance", type=float, default=1.0) + parser.add_argument("--strict", action="store_true") + args = parser.parse_args() + + root = Path(args.result_root).expanduser().resolve() + paths = sorted((root / "datasets").glob("*/epsilon_ablation.csv")) + if not paths: + raise FileNotFoundError(f"No epsilon_ablation.csv files found under {root}") + frame = pd.concat((pd.read_csv(path) for path in paths), ignore_index=True) + + rows = [] + for dataset, target in TARGETS.items(): + selected = frame[ + (frame["dataset"] == dataset) + & (frame["seed"] == 42) + & (frame["epsilon"] == 5.0) + & (frame["metric"] == target["metric"]) + ] + if len(selected) != 1: + raise ValueError( + f"Expected exactly one seed=42, epsilon=5 row for {dataset}/" + f"{target['metric']}, found {len(selected)}." + ) + row = selected.iloc[0] + observed_cp = 100.0 * float(row["cp"]) + observed_pp = 100.0 * float(row["pp"]) + cp_error = observed_cp - target["cp"] + pp_error = observed_pp - target["pp"] + rows.append({ + "dataset": dataset, + "metric": target["metric"], + "clean": 100.0 * float(row["clean"]), + "authorized_cooked": 100.0 * float(row["authorized_cooked"]), + "unauthorized_raw": 100.0 * float(row["unauthorized_raw"]), + "observed_cp": observed_cp, + "target_cp": target["cp"], + "cp_error": cp_error, + "observed_pp": observed_pp, + "target_pp": target["pp"], + "pp_error": pp_error, + "within_tolerance": ( + abs(cp_error) <= args.tolerance + and abs(pp_error) <= args.tolerance + ), + }) + + output = pd.DataFrame(rows) + output_path = root / "target_reproduction.csv" + output_path.parent.mkdir(parents=True, exist_ok=True) + output.to_csv(output_path, index=False) + print(output.to_string(index=False, float_format=lambda value: f"{value:.2f}")) + print(f"Wrote {output_path}") + if args.strict and not bool(output["within_tolerance"].all()): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/test_3d_pipeline.py b/tests/test_3d_pipeline.py new file mode 100644 index 0000000..2905e62 --- /dev/null +++ b/tests/test_3d_pipeline.py @@ -0,0 +1,74 @@ +import logging +from pathlib import Path + +import numpy as np + +from volume_io import save_cooked_volume_npz +from run_datacook_suite import load_pipeline_configs + + +def test_3d_cooked_splits_are_exported_as_npz(tmp_path): + output = tmp_path / "ae" / "perturb_data.npz" + save_cooked_volume_npz( + output, + np.full((2, 1, 8, 8, 8), 0.5, dtype=np.float32), + np.asarray([[0], [1]], dtype=np.int64), + ) + + assert output.is_file() + with np.load(output) as archive: + assert archive["perturb_images"].shape == (2, 1, 8, 8, 8) + np.testing.assert_array_equal(archive["perturb_labels"], [0, 1]) + + +def test_perturbation_uses_3d_patch_and_npz_contract(): + source = Path("perturbation.py").read_text(encoding="utf-8") + assert "_patch_noise_to_3d_img" in source + assert "save_cooked_volume_npz(output_path, volume_images, volume_labels)" in source + + +def test_3d_smoke_config_uses_single_seed_and_epsilon(): + import yaml + + config = yaml.safe_load( + Path("configs/pipelines/organmnist3d_smoke.yaml").read_text(encoding="utf-8") + ) + assert config["dataset"]["noise_shape"] == [971, 1, 28, 28, 28] + assert config["dataset"]["test_noise_shape"] == [610, 1, 28, 28, 28] + assert config["models"] == { + "surrogate": "resnet18_3d", + "victim": "resnet18_3d", + } + assert config["seeds"] == [0] + assert config["epsilons"] == [8] + + +def test_four_dataset_3d_suite_has_aligned_formal_settings(): + import yaml + + suite = Path("configs/pipelines/medmnist3d_four.yaml").resolve() + paths = load_pipeline_configs(suite) + assert [path.name for path in paths] == [ + "organmnist3d.yaml", + "nodulemnist3d.yaml", + "fracturemnist3d.yaml", + "synapsemnist3d.yaml", + ] + expected = { + "OrganMNIST3D": (971, 610, 11, "multiclass"), + "NoduleMNIST3D": (1158, 310, 2, "binary"), + "FractureMNIST3D": (1027, 240, 3, "multiclass"), + "SynapseMNIST3D": (1230, 352, 2, "binary"), + } + for path in paths: + config = yaml.safe_load(path.read_text(encoding="utf-8")) + dataset = config["dataset"] + train_count, test_count, classes, task = expected[dataset["name"]] + assert dataset["noise_shape"] == [train_count, 1, 28, 28, 28] + assert dataset["test_noise_shape"] == [test_count, 1, 28, 28, 28] + assert dataset["num_classes"] == classes + assert dataset["task_type"] == task + assert config["seeds"] == [0, 1, 2] + assert config["epsilons"] == [8, 16, 32] + assert config["models"]["surrogate"] == "resnet18_3d" + assert config["models"]["victim"] == "resnet18_3d" diff --git a/tests/test_checkpoint_compatibility.py b/tests/test_checkpoint_compatibility.py new file mode 100644 index 0000000..889043a --- /dev/null +++ b/tests/test_checkpoint_compatibility.py @@ -0,0 +1,17 @@ +import torch + +import util + + +def test_checkpoint_suffix_and_resume_compatibility(tmp_path): + model = torch.nn.Linear(2, 2) + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1) + base = tmp_path / "model" + util.save_model(str(base), 0, model, optimizer, scheduler, ENV={"global_step": 1}) + assert (tmp_path / "model.pth").is_file() + restored = torch.nn.Linear(2, 2) + restored_optimizer = torch.optim.SGD(restored.parameters(), lr=0.1) + restored_scheduler = torch.optim.lr_scheduler.StepLR(restored_optimizer, step_size=1) + checkpoint = util.load_model(str(base), restored, restored_optimizer, restored_scheduler) + assert checkpoint["epoch"] == 1 diff --git a/tests/test_datacook_suite.py b/tests/test_datacook_suite.py new file mode 100644 index 0000000..9294b6a --- /dev/null +++ b/tests/test_datacook_suite.py @@ -0,0 +1,52 @@ +from pathlib import Path + +import yaml + +from run_datacook_pipeline import _discover_run_results +from run_datacook_suite import load_pipeline_configs + + +def test_three_dataset_suite_configs_are_valid(): + suite = Path("configs/pipelines/medmnist_three.yaml").resolve() + configs = load_pipeline_configs(suite) + assert [path.name for path in configs] == [ + "bloodmnist.yaml", + "breastmnist.yaml", + "retinamnist.yaml", + ] + expected = { + "BloodMNIST": (8, "multiclass", [11959, 3, 28, 28], [3421, 3, 28, 28]), + "BreastMNIST": (2, "binary", [546, 3, 28, 28], [156, 3, 28, 28]), + "RetinaMNIST": (5, "multiclass", [1080, 3, 28, 28], [400, 3, 28, 28]), + } + for path in configs: + config = yaml.safe_load(path.read_text(encoding="utf-8")) + dataset = config["dataset"] + assert ( + dataset["num_classes"], + dataset["task_type"], + dataset["noise_shape"], + dataset["test_noise_shape"], + ) == expected[dataset["name"]] + assert (Path(dataset["config_path"]) / "resnet18.yaml").is_file() + + +def test_additional_suite_contains_only_new_datasets(): + suite = Path("configs/pipelines/medmnist_additional_two.yaml").resolve() + assert [path.name for path in load_pipeline_configs(suite)] == [ + "breastmnist.yaml", + "retinamnist.yaml", + ] + + +def test_result_discovery_combines_dataset_run_directories(tmp_path): + expected = [] + for run_name in ("BloodMNIST-seed=0", "BreastMNIST-seed=0", "RetinaMNIST-seed=0"): + path = tmp_path / run_name / "metrics" / "main_metrics.csv" + path.parent.mkdir(parents=True) + path.write_text("dataset,metric,value\n", encoding="utf-8") + expected.append(path) + summary = tmp_path / "summary" / "main_metrics.csv" + summary.parent.mkdir() + summary.write_text("do,not,include\n", encoding="utf-8") + assert _discover_run_results(tmp_path, "main_metrics.csv") == sorted(expected) diff --git a/tests/test_final_analysis.py b/tests/test_final_analysis.py new file mode 100644 index 0000000..d60c244 --- /dev/null +++ b/tests/test_final_analysis.py @@ -0,0 +1,76 @@ +import numpy as np +import pandas as pd + +from analysis.finalize_revision_results import ( + build_budget_all, + build_epsilon_ablation, + build_significance, +) + + +def _attack_rows(strength="epochs=10,lr=0.0001"): + rows = [] + for seed in (0, 1, 2): + for attack_type, after in (("finetune", 0.7), ("robust_finetune", 0.65)): + rows.append({ + "dataset": "Toy", "modality": "2d_image", "task_type": "multiclass", + "method": "datacook", "surrogate_model": "r18", "victim_model": "r18", + "seed": seed, "epsilon": 8 / 255, "setting": "adaptive_attack", + "attack_type": attack_type, "attack_strength": strength, "metric": "acc", + "clean": 0.8, "authorized_cooked": 0.8, "unauthorized_raw": 0.5, + "attacked_unauthorized_raw": after, "cp_before_attack": -0.3, + "cp_after_attack": after - 0.8, "recovery_rate": (after - 0.5) / 0.3, + "recovery_rate_unclipped": (after - 0.5) / 0.3, + }) + return pd.DataFrame(rows) + + +def test_epsilon_ablation_merges_quality_by_run(): + cp = pd.DataFrame([ + {"dataset": "Toy", "modality": "2d_image", "task_type": "multiclass", + "method": "datacook", "surrogate_model": "r18", "victim_model": "r18", + "seed": 0, "epsilon": 8 / 255, "metric": "acc", "cp": -0.2, "pp": 0.0}, + ]) + quality = pd.DataFrame([ + {"dataset": "Toy", "modality": "2d_image", "task_type": "multiclass", + "method": "datacook", "surrogate_model": "r18", "victim_model": "r18", + "seed": 0, "epsilon": 8 / 255, "ssim_mean": 0.95, "ssim_std": 0.01}, + ]) + result = build_epsilon_ablation(cp, quality) + assert result.loc[0, "epsilon_pixel"] == 8 + assert result.loc[0, "ssim_mean"] == 0.95 + + +def test_budget_all_reuses_full_and_adds_fixed_budget(): + full = _attack_rows() + budget = _attack_rows("epochs=10,lr=0.0001,n=25") + budget["raw_subset_size"] = 25 + result = build_budget_all(full, budget, {"Toy": 60}) + assert set(result["raw_subset_size"]) == {25, 60} + assert set(result["raw_subset_budget"]) == {"25", "full"} + assert len(result) == 12 + + +def test_significance_reports_effect_and_holm_value(): + attacks = _attack_rows() + cp = pd.DataFrame([ + {"dataset": "Toy", "epsilon": 8 / 255, "metric": "acc", "seed": seed, + "clean": 0.8, "unauthorized_raw": 0.5} + for seed in (0, 1, 2) + ]) + budget_all = build_budget_all(attacks, pd.DataFrame(), {"Toy": 60}) + result = build_significance(cp, attacks, budget_all) + assert not result.empty + assert result["p_value"].notna().all() + assert result["p_value_holm"].between(0, 1).all() + before_after = result[result["comparison"].eq("attack_before_vs_after")] + assert np.all(before_after["mean_difference_b_minus_a"] > 0) + + +def test_summary_preserves_identical_infinite_psnr(): + from analysis.finalize_revision_results import _summary_table + + frame = pd.DataFrame({"dataset": ["Breast", "Breast", "Breast"], "psnr_mean": [np.inf] * 3}) + result = _summary_table(frame, ["dataset"], ["psnr_mean"]) + assert np.isposinf(result.loc[0, "psnr_mean_mean"]) + assert result.loc[0, "psnr_mean_std"] == 0.0 diff --git a/tests/test_high_resolution_pipeline.py b/tests/test_high_resolution_pipeline.py new file mode 100644 index 0000000..d708cef --- /dev/null +++ b/tests/test_high_resolution_pipeline.py @@ -0,0 +1,68 @@ +from pathlib import Path + +import yaml + +from prepare_revision_posthoc import load_run_specs +from run_datacook_pipeline import _main_command +from run_datacook_suite import load_pipeline_configs + + +def test_224_suite_uses_three_datasets_and_safe_defaults(): + suite = Path("configs/pipelines/medmnist_224_three.yaml").resolve() + configs = load_pipeline_configs(suite) + assert [path.name for path in configs] == [ + "breastmnist224.yaml", + "pneumoniamnist224.yaml", + "retinamnist224.yaml", + ] + expected_counts = { + "BreastMNIST": (546, 156), + "PneumoniaMNIST": (4708, 624), + "RetinaMNIST": (1080, 400), + } + for path in configs: + config = yaml.safe_load(path.read_text(encoding="utf-8")) + dataset = config["dataset"] + train_count, test_count = expected_counts[dataset["name"]] + assert dataset["high_resolution"] is True + assert dataset["modality"] == "high_resolution" + assert dataset["noise_shape"] == [train_count, 3, 224, 224] + assert dataset["test_noise_shape"] == [test_count, 3, 224, 224] + assert config["models"] == { + "surrogate": "resnet18_224", + "victim": "resnet18_224", + } + assert config["seeds"] == [0, 1, 2] + assert config["epsilons"] == [8] + + +def test_224_flag_is_forwarded_to_training_command(tmp_path): + command = _main_command( + "python", + model="resnet18_224", + config_path=tmp_path, + exp_name=tmp_path / "exp", + dataset_name="BreastMNIST", + train_type="BreastMNIST", + train_path=tmp_path, + test_type="BreastMNIST", + test_path=tmp_path, + recover_type="BreastMNIST", + recover_path=tmp_path, + prediction_dir=tmp_path / "predictions", + role="clean", + seed=0, + train_batch_size=8, + eval_batch_size=8, + workers=0, + checkpoint=tmp_path / "missing.pth", + high_resolution=True, + ) + assert "--high_resolution" in command + + +def test_posthoc_specs_keep_224_resolution_metadata(): + specs = load_run_specs(Path("configs/pipelines/medmnist_224_three.yaml").resolve()) + assert len(specs) == 9 + assert all(spec.high_resolution for spec in specs) + assert {spec.modality for spec in specs} == {"high_resolution"} diff --git a/tests/test_merge_result_shards.py b/tests/test_merge_result_shards.py new file mode 100644 index 0000000..b68f74c --- /dev/null +++ b/tests/test_merge_result_shards.py @@ -0,0 +1,29 @@ +import pandas as pd +import pytest + +from analysis.merge_result_shards import merge_shards + + +def _row(seed, value): + return { + "dataset": "Toy", "method": "datacook", "seed": seed, "epsilon": 8 / 255, + "setting": "adaptive_attack", "attack_type": "finetune", + "attack_strength": "epochs=10", "metric": "acc", + "attacked_unauthorized_raw": value, + } + + +def test_merge_shards_combines_non_overlapping_rows(tmp_path): + first, second = tmp_path / "first.csv", tmp_path / "second.csv" + pd.DataFrame([_row(0, 0.6)]).to_csv(first, index=False) + pd.DataFrame([_row(1, 0.7)]).to_csv(second, index=False) + result = merge_shards([first, second]) + assert result["seed"].tolist() == [0, 1] + + +def test_merge_shards_rejects_conflicting_identity(tmp_path): + first, second = tmp_path / "first.csv", tmp_path / "second.csv" + pd.DataFrame([_row(0, 0.6)]).to_csv(first, index=False) + pd.DataFrame([_row(0, 0.9)]).to_csv(second, index=False) + with pytest.raises(ValueError, match="Conflicting result rows"): + merge_shards([first, second]) diff --git a/tests/test_multilabel_pipeline.py b/tests/test_multilabel_pipeline.py new file mode 100644 index 0000000..326e31a --- /dev/null +++ b/tests/test_multilabel_pipeline.py @@ -0,0 +1,100 @@ +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +import yaml + +from evaluator import Evaluator +from toolbox import PerturbationTool +from volume_io import save_cooked_array_npz + + +def test_multilabel_cooked_npz_preserves_label_matrix(tmp_path): + output = tmp_path / "perturb_data.npz" + images = np.random.default_rng(0).random((3, 3, 8, 8), dtype=np.float32) + labels = np.asarray([[1, 0], [0, 1], [1, 1]], dtype=np.int64) + save_cooked_array_npz(output, images, labels) + + # dataset.py is available in the project/server environment. Lightweight + # analysis environments may omit its mlconfig runtime dependency. + pytest.importorskip("mlconfig") + from dataset import Data3DFolderWithLabel + + dataset = Data3DFolderWithLabel(output) + image, label, prediction_label = dataset[1] + assert image.shape == (3, 8, 8) + assert image.dtype == torch.float32 + assert label.tolist() == [0, 1] + assert prediction_label.tolist() == [0, 1] + + +def _multilabel_config(): + return SimpleNamespace( + log_frequency=100, + num_classes=2, + task_type="multilabel", + grad_clip=5.0, + ) + + +def test_bce_trainer_uses_multilabel_targets(): + pytest.importorskip("mlconfig") + from trainer import Trainer + + config = _multilabel_config() + criterion = torch.nn.BCEWithLogitsLoss() + model = torch.nn.Linear(2, 2) + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + images = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) + labels = torch.tensor([[1, 0], [0, 1]]) + trainer = Trainer(criterion, {}, None, config, "ChestMNIST") + payload = trainer.train_batch(images, labels, model, optimizer) + assert 0.0 <= payload["acc"] <= 1.0 + + +def test_evaluator_exports_sigmoid_multilabel_predictions(): + config = _multilabel_config() + model = torch.nn.Linear(2, 2) + images = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) + labels = torch.tensor([[1, 0], [0, 1]]) + evaluator = Evaluator({}, None, config, "ChestMNIST") + eval_payload = evaluator.eval_batch(images, labels, model) + assert 0.0 <= eval_payload["acc"] <= 1.0 + assert evaluator.y_true[0].shape == (2, 2) + assert evaluator.y_prob[0].shape == (2, 2) + assert torch.all((evaluator.y_prob[0] >= 0) & (evaluator.y_prob[0] <= 1)) + + +def test_datacook_generates_per_label_pseudo_targets_for_bce(): + model = torch.nn.Linear(2, 3) + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + tool = PerturbationTool(epsilon=0.1, num_steps=1, step_size=0.01) + images = torch.rand(4, 2) + labels = torch.zeros(4, 3) + perturb_img, delta = tool.datacook_v1( + images, + labels, + model, + optimizer, + torch.nn.BCEWithLogitsLoss(), + random_noise=torch.zeros_like(images), + ) + assert perturb_img.shape == images.shape + assert delta.shape == images.shape + + +def test_chestmnist_pipeline_contract(): + config = yaml.safe_load(Path("configs/pipelines/chestmnist.yaml").read_text()) + dataset = config["dataset"] + assert dataset["task_type"] == "multilabel" + assert dataset["num_classes"] == 14 + assert dataset["noise_shape"] == [78468, 3, 28, 28] + assert dataset["test_noise_shape"] == [22433, 3, 28, 28] + + model_config = yaml.safe_load( + Path("configs/MedMNIST/ChestMNIST/resnet18.yaml").read_text() + ) + assert model_config["criterion"]["name"] == "BCEWithLogitsLoss" + assert model_config["task_type"] == "multilabel" diff --git a/tests/test_perturbation_regressions.py b/tests/test_perturbation_regressions.py new file mode 100644 index 0000000..253e091 --- /dev/null +++ b/tests/test_perturbation_regressions.py @@ -0,0 +1,17 @@ +from pathlib import Path + + +def test_samplewise_eval_uses_defined_eval_target(): + source = Path("perturbation.py").read_text(encoding="utf-8") + function = source.split("def samplewise_perturbation_eval", 1)[1] + function = function.split("def main", 1)[0] + assert "_ensure_labels_shape(eval_target, labels)" in function + assert "_ensure_labels_shape(loader_key, labels)" not in function + + +def test_surrogate_resume_infers_device_from_model(): + source = Path("perturbation.py").read_text(encoding="utf-8") + train_function = source.split("def train(args, config", 1)[1] + train_function = train_function.split("def check_ssim", 1)[0] + assert "model_device = next(model.parameters()).device" in train_function + assert "map_location=model_device" in train_function diff --git a/tests/test_prediction_export.py b/tests/test_prediction_export.py new file mode 100644 index 0000000..9a2a33c --- /dev/null +++ b/tests/test_prediction_export.py @@ -0,0 +1,21 @@ +from types import SimpleNamespace + +import numpy as np +import torch + +from evaluator import Evaluator + + +def test_evaluator_prediction_export_and_reset(tmp_path): + config = SimpleNamespace(log_frequency=100, num_classes=2, task_type="multiclass") + evaluator = Evaluator({}, None, config, "Toy") + evaluator.y_true = [torch.tensor([0, 1])] + evaluator.y_prob = [torch.tensor([[0.9, 0.1], [0.2, 0.8]])] + output = tmp_path / "clean.npz" + evaluator.save_predictions(str(output)) + with np.load(output) as artifact: + assert artifact["y_true"].tolist() == [0, 1] + assert artifact["y_prob"].shape == (2, 2) + evaluator._reset_stats() + assert evaluator.y_true == [] + assert evaluator.recover_y_prob == [] diff --git a/tests/test_prepare_revision_posthoc.py b/tests/test_prepare_revision_posthoc.py new file mode 100644 index 0000000..0d3db72 --- /dev/null +++ b/tests/test_prepare_revision_posthoc.py @@ -0,0 +1,282 @@ +import sys +import types +from dataclasses import replace +from pathlib import Path + +import numpy as np +import torch +import yaml +from PIL import Image + +from prepare_revision_posthoc import ( + RunSpec, + _cooked_index, + _load_official_split, + export_cooked_test, + generate_configs, + load_run_specs, +) + + +def test_suite_expands_to_45_expected_runs(): + specs = load_run_specs(Path("configs/pipelines/medmnist_three.yaml").resolve()) + assert len(specs) == 45 + assert {spec.dataset for spec in specs} == {"BloodMNIST", "BreastMNIST", "RetinaMNIST"} + assert {spec.seed for spec in specs} == {0, 1, 2} + assert {spec.epsilon_pixel for spec in specs} == {2.0, 4.0, 8.0, 12.0, 16.0} + + +def test_suite_cli_seed_override_discovers_runs_outside_base_yaml(): + specs = load_run_specs( + Path("configs/pipelines/medmnist_three.yaml").resolve(), + selected_seeds={3, 4, 5, 6, 7}, + ) + assert len(specs) == 75 + assert {spec.seed for spec in specs} == {3, 4, 5, 6, 7} + assert {spec.epsilon_pixel for spec in specs} == {2.0, 4.0, 8.0, 12.0, 16.0} + + +def test_suite_cli_seed_and_epsilon_overrides_match_attack_extension(): + specs = load_run_specs( + Path("configs/pipelines/medmnist_three.yaml").resolve(), + selected_datasets={"RetinaMNIST"}, + selected_seeds=set(range(8)), + selected_epsilons={12.0}, + ) + assert len(specs) == 8 + assert {spec.seed for spec in specs} == set(range(8)) + assert {spec.epsilon_pixel for spec in specs} == {12.0} + + +def test_extra_2d_suite_expands_to_120_expected_runs(): + specs = load_run_specs(Path("configs/pipelines/medmnist_extra_three.yaml").resolve()) + assert len(specs) == 120 + assert {spec.dataset for spec in specs} == { + "PneumoniaMNIST", "DermaMNIST", "OrganCMNIST", + } + assert {spec.seed for spec in specs} == set(range(8)) + assert {spec.epsilon_pixel for spec in specs} == {2.0, 4.0, 8.0, 12.0, 16.0} + counts = {spec.dataset: (spec.test_count, spec.num_classes) for spec in specs} + assert counts == { + "PneumoniaMNIST": (624, 2), + "DermaMNIST": (2005, 7), + "OrganCMNIST": (8216, 11), + } + + +def test_3d_suite_expands_to_36_expected_runs(): + specs = load_run_specs(Path("configs/pipelines/medmnist3d_four.yaml").resolve()) + assert len(specs) == 36 + assert {spec.dataset for spec in specs} == { + "OrganMNIST3D", "NoduleMNIST3D", + "FractureMNIST3D", "SynapseMNIST3D", + } + assert {spec.modality for spec in specs} == {"3d_volume"} + assert {spec.seed for spec in specs} == {0, 1, 2} + assert {spec.epsilon_pixel for spec in specs} == {8.0, 16.0, 32.0} + + +def test_official_3d_split_uses_volume_transform(monkeypatch, tmp_path): + constructor_kwargs = {} + + class FakeVolumeDataset: + def __init__(self, **kwargs): + constructor_kwargs.update(kwargs) + self.transform = kwargs["transform"] + + def __iter__(self): + for label in (0, 1): + volume = np.full((1, 4, 4, 4), label, dtype=np.uint8) + yield self.transform(volume), np.array([label]) + + fake_medmnist = types.ModuleType("medmnist") + fake_medmnist.INFO = { + "toymnist3d": {"python_class": "FakeVolumeDataset"}, + } + fake_medmnist.FakeVolumeDataset = FakeVolumeDataset + monkeypatch.setitem(sys.modules, "medmnist", fake_medmnist) + spec = RunSpec( + dataset="ToyMNIST3D", modality="3d_volume", task_type="binary", + num_classes=2, surrogate="resnet18_3d", victim="resnet18_3d", + method="datacook", seed=0, epsilon_pixel=8.0, epsilon=8 / 255, + run_name="ToyMNIST3D-run", run_dir=tmp_path, model_config=tmp_path / "model.yaml", + data_path=tmp_path, test_count=2, + ) + + images, labels = _load_official_split(spec, "test") + + assert images.shape == (2, 1, 4, 4, 4) + assert images.dtype == np.float32 + assert labels.tolist() == [0, 1] + assert constructor_kwargs["as_rgb"] is False + assert constructor_kwargs["transform"].__class__.__name__ == "Transform3D" + + +def test_cooked_export_restores_global_order_and_checks_labels(tmp_path): + raw = np.zeros((3, 3, 2, 2), dtype=np.float32) + labels = np.array([1, 0, 1], dtype=np.int64) + cooked_dir = tmp_path / "ae_test" + values = {0: 32, 1: 96, 2: 224} + for index in (2, 0, 1): + folder = cooked_dir / str(labels[index]) + folder.mkdir(parents=True, exist_ok=True) + image = np.full((2, 2, 3), values[index], dtype=np.uint8) + Image.fromarray(image).save(folder / f"{index % 2}_{index}.png") + + output = tmp_path / "cooked.npz" + export_cooked_test(cooked_dir, raw, labels, output, force=False, logger=_NullLogger()) + with np.load(output, allow_pickle=False) as artifact: + assert artifact["labels"].tolist() == [1, 0, 1] + assert np.allclose(artifact["images"][:, 0, 0, 0], np.array([32, 96, 224]) / 255) + + +def test_cooked_export_recovers_from_saved_test_perturbation(tmp_path): + raw = np.full((2, 3, 2, 2), 0.5, dtype=np.float32) + labels = np.array([0, 1], dtype=np.int64) + perturbation = torch.from_numpy(np.stack([ + np.full((3, 2, 2), 0.01, dtype=np.float32), + np.full((3, 2, 2), -0.02, dtype=np.float32), + ])) + perturbation_path = tmp_path / "test_perturbation.pt" + torch.save(perturbation, perturbation_path) + cooked_dir = tmp_path / "empty_ae_test" + cooked_dir.mkdir() + output = tmp_path / "recovered.npz" + + export_cooked_test( + cooked_dir, + raw, + labels, + output, + force=False, + logger=_NullLogger(), + perturbation_path=perturbation_path, + ) + + expected = np.floor(np.clip(raw + perturbation.numpy(), 0, 1) * 255 + 0.5) / 255 + with np.load(output, allow_pickle=False) as artifact: + np.testing.assert_allclose(artifact["images"], expected, atol=1e-7) + np.testing.assert_array_equal(artifact["labels"], labels) + + +def test_generated_configs_include_only_task_ready_runs(tmp_path): + run_dir = tmp_path / "experiments" / "Toy-run" + config_dir = tmp_path / "configs" / "revision" / "generated" + artifact_root = tmp_path / "artifacts" + model_config = tmp_path / "model.yaml" + model_config.write_text("model: {}\n", encoding="utf-8") + spec = RunSpec( + dataset="ToyMNIST", modality="2d_image", task_type="multiclass", num_classes=3, + surrogate="resnet18", victim="resnet18", method="datacook", seed=0, + epsilon_pixel=8.0, epsilon=8 / 255, run_name="Toy-run", run_dir=run_dir, + model_config=model_config, data_path=tmp_path, test_count=2, + ) + for path in spec.predictions.values(): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + spec.pseudo_labels.parent.mkdir(parents=True, exist_ok=True) + spec.pseudo_labels.touch() + spec.checkpoint.parent.mkdir(parents=True, exist_ok=True) + spec.checkpoint.touch() + spec.test_perturbation.touch() + raw_path = artifact_root / spec.dataset / "raw_test.npz" + raw_val_path = artifact_root / spec.dataset / "raw_val.npz" + cooked_path = artifact_root / spec.run_name / "cooked_test.npz" + raw_path.parent.mkdir(parents=True) + cooked_path.parent.mkdir(parents=True) + arrays = {"images": np.zeros((2, 3, 2, 2), np.float32), "labels": np.array([0, 1])} + np.savez_compressed(raw_path, **arrays) + validation_arrays = { + "images": np.zeros((60, 3, 2, 2), np.float32), + "labels": np.tile(np.arange(3), 20), + } + np.savez_compressed(raw_val_path, **validation_arrays) + np.savez_compressed(cooked_path, **arrays) + + counts = generate_configs( + [spec], artifact_root, config_dir, + attacks_epsilon=8, runtime_seed=0, runtime_epsilon=8, + ) + assert counts == { + "pseudo": 1, "image_quality": 1, "runtime": 1, + "attacks": 1, "finetune_attacks": 1, "finetune_budget_attacks": 1, + } + attacks = yaml.safe_load((config_dir / "attacks.yaml").read_text(encoding="utf-8")) + assert len(attacks["experiments"][0]["attacks"]) == 11 + finetune = yaml.safe_load((config_dir / "attacks_finetune.yaml").read_text(encoding="utf-8")) + assert [attack["type"] for attack in finetune["experiments"][0]["attacks"]] == [ + "finetune", "finetune", "robust_finetune", "robust_finetune", + ] + assert finetune["output"].endswith("adaptive_finetune_results.csv") + assert finetune["combined_output"].endswith("adaptive_attack_results_all.csv") + budget = yaml.safe_load((config_dir / "attacks_finetune_budget.yaml").read_text(encoding="utf-8")) + assert [(attack["type"], attack["subset_size"], attack["epochs"]) for attack in budget["experiments"][0]["attacks"]] == [ + ("finetune", 25, 10), ("robust_finetune", 25, 10), + ("finetune", 50, 10), ("robust_finetune", 50, 10), + ] + final_analysis = yaml.safe_load((config_dir / "final_analysis.yaml").read_text(encoding="utf-8")) + assert final_analysis["full_subset_sizes"] == {"ToyMNIST": 60} + master = yaml.safe_load((config_dir.parent / "generated.yaml").read_text(encoding="utf-8")) + assert master["tasks"]["pseudo"]["enabled"] is True + assert master["tasks"]["finetune_attacks"]["enabled"] is False + assert master["tasks"]["finetune_budget_attacks"]["enabled"] is False + assert master["tasks"]["epsilon"]["enabled"] is False + + suffix_dir = tmp_path / "configs" / "revision" / "extension" / "generated" + generate_configs( + [spec], artifact_root, suffix_dir, + attacks_epsilon=8, runtime_seed=0, runtime_epsilon=8, + output_suffix="seed 3-7/eps8", finetune_epochs=(10,), budget_sizes=(25,), + ) + suffix_attacks = yaml.safe_load((suffix_dir / "attacks_finetune.yaml").read_text(encoding="utf-8")) + assert suffix_attacks["output"].endswith("adaptive_finetune_results_seed-3-7-eps8.csv") + assert suffix_attacks["merge_with"].endswith("adaptive_attack_results_seed-3-7-eps8.csv") + assert [attack["epochs"] for attack in suffix_attacks["experiments"][0]["attacks"]] == [10, 10] + suffix_budget = yaml.safe_load((suffix_dir / "attacks_finetune_budget.yaml").read_text(encoding="utf-8")) + assert {attack["subset_size"] for attack in suffix_budget["experiments"][0]["attacks"]} == {25} + + selected_dir = tmp_path / "configs" / "revision" / "3d" / "generated" + volume_spec = replace(spec, modality="3d_volume") + selected_counts = generate_configs( + [volume_spec], artifact_root, selected_dir, + attacks_epsilon=8, runtime_seed=0, runtime_epsilon=8, + tasks={"pseudo", "runtime"}, + result_root="results/revision_pipeline_3d", + ) + assert selected_counts == { + "pseudo": 1, "image_quality": 0, "runtime": 1, + "attacks": 0, "finetune_attacks": 0, "finetune_budget_attacks": 0, + } + pseudo = yaml.safe_load((selected_dir / "pseudo.yaml").read_text(encoding="utf-8")) + runtime = yaml.safe_load((selected_dir / "runtime.yaml").read_text(encoding="utf-8")) + assert pseudo["outputs"]["quality"] == ( + "results/revision_pipeline_3d/posthoc/pseudo_label_quality.csv" + ) + assert runtime["output"] == "results/revision_pipeline_3d/posthoc/runtime_summary.csv" + assert runtime["include_average"] is True + assert runtime["experiments"][0]["batch_size"] == 16 + selected_master = yaml.safe_load( + (selected_dir.parent / "generated.yaml").read_text(encoding="utf-8") + ) + assert selected_master["tasks"]["pseudo"]["enabled"] is True + assert selected_master["tasks"]["runtime"]["enabled"] is True + assert selected_master["tasks"]["image_quality"]["enabled"] is False + assert selected_master["tasks"]["attacks"]["enabled"] is False + + +def test_cooked_index_requires_global_suffix(tmp_path): + assert _cooked_index(tmp_path / "17_123.png") == 123 + try: + _cooked_index(tmp_path / "sample.png") + except ValueError as error: + assert "global sample index" in str(error) + else: + raise AssertionError("Expected filename validation to fail.") + + +class _NullLogger: + def info(self, *_args, **_kwargs): + pass + + def warning(self, *_args, **_kwargs): + pass diff --git a/tests/test_revision_attacks.py b/tests/test_revision_attacks.py new file mode 100644 index 0000000..ecab8d9 --- /dev/null +++ b/tests/test_revision_attacks.py @@ -0,0 +1,51 @@ +import numpy as np +import pandas as pd + +from attacks.jpeg import jpeg_compress +from attacks.smoothing import gaussian_smooth +from attacks.denoise import median_filter +from attacks.augmentation import strong_augment +from run_adaptive_attacks import _merge_results, _select_raw_subset + + +def test_image_attacks_preserve_shape_and_range(): + images = np.random.default_rng(0).random((2, 3, 12, 12), dtype=np.float32) + for attacked in ( + jpeg_compress(images, quality=75), + gaussian_smooth(images, sigma=1.0, kernel_size=3), + median_filter(images, kernel_size=3), + strong_augment(images, strength=2, seed=0), + ): + assert attacked.shape == images.shape + assert attacked.min() >= 0 + assert attacked.max() <= 1 + + +def test_merge_results_preserves_inputs_and_deduplicates(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + old = pd.DataFrame([{"attack_type": "jpeg", "metric": "accuracy", "value": 0.5}]) + new = pd.DataFrame([ + {"attack_type": "finetune", "metric": "accuracy", "value": 0.7}, + {"attack_type": "finetune", "metric": "accuracy", "value": 0.7}, + ]) + old.to_csv("old.csv", index=False) + new.to_csv("new.csv", index=False) + + output = _merge_results(tmp_path / "new.csv", "old.csv", "all.csv") + + combined = pd.read_csv(output) + assert combined["attack_type"].tolist() == ["jpeg", "finetune"] + assert len(pd.read_csv("old.csv")) == 1 + assert len(pd.read_csv("new.csv")) == 2 + + +def test_raw_budget_subset_is_reproducible_and_stratified(): + inputs = np.arange(60).reshape(20, 3) + labels = np.repeat(np.arange(4), 5) + first_x, first_y = _select_raw_subset(inputs, labels, subset_size=12, seed=7) + second_x, second_y = _select_raw_subset(inputs, labels, subset_size=12, seed=7) + + assert np.array_equal(first_x, second_x) + assert np.array_equal(first_y, second_y) + assert len(first_x) == 12 + assert set(first_y) == {0, 1, 2, 3} diff --git a/tests/test_revision_metrics.py b/tests/test_revision_metrics.py new file mode 100644 index 0000000..c414623 --- /dev/null +++ b/tests/test_revision_metrics.py @@ -0,0 +1,134 @@ +import math +import sys +from types import SimpleNamespace + +import numpy as np +import pytest + +from metrics.classification_metrics import compute_classification_metrics +from metrics.cp_pp_metrics import compute_cp_pp +from metrics.image_quality import _lpips_values, compute_image_quality_metrics + + +def test_binary_metrics_and_ece(): + truth = np.array([0, 0, 1, 1]) + prob = np.array([0.1, 0.4, 0.6, 0.9]) + values = compute_classification_metrics(truth, y_prob=prob, task_type="binary") + assert values["acc"] == 1.0 + assert values["auc"] == 1.0 + assert values["sensitivity"] == 1.0 + assert values["specificity"] == 1.0 + assert 0 <= values["ece"] <= 1 + + +def test_multiclass_metrics_with_missing_class_in_truth(): + truth = np.array([0, 0, 1, 1]) + prob = np.array([[.8, .1, .1], [.7, .2, .1], [.1, .8, .1], [.2, .7, .1]]) + values = compute_classification_metrics(truth, y_prob=prob, task_type="multiclass") + assert values["acc"] == 1.0 + assert values["macro_auc"] == 1.0 + assert values["macro_specificity"] == 1.0 + + +def test_multiclass_rejects_non_probability_scores(): + with pytest.raises(ValueError, match="sum to 1"): + compute_classification_metrics( + np.array([0, 1]), y_prob=np.array([[.8, .8], [.2, .2]]), task_type="multiclass" + ) + + +def test_multilabel_metrics(): + truth = np.array([[1, 0], [0, 1], [1, 1]]) + prob = np.array([[.9, .1], [.2, .8], [.8, .7]]) + values = compute_classification_metrics(truth, y_prob=prob, task_type="multi-label") + assert values["subset_acc"] == 1.0 + assert values["hamming_loss"] == 0.0 + assert values["micro_f1"] == 1.0 + + +def test_cp_pp_random_baseline(): + clean = {"acc": .8, "auc": .9} + auth = {"acc": .82, "auc": .91} + unauth = {"acc": .55, "auc": .6} + result = compute_cp_pp(clean, auth, unauth, ["acc", "auc"], task_type="binary") + assert math.isclose(result["acc"]["cp"], -.25) + assert math.isclose(result["acc"]["pp"], .02) + assert result["auc"]["random_guess"] == .5 + + +def test_image_quality_identical_without_lpips(): + images = np.zeros((2, 1, 8, 8), dtype=np.float32) + values = compute_image_quality_metrics(images, images, compute_lpips=False) + assert values["ssim_mean"] == 1.0 + assert math.isinf(values["psnr_mean"]) + assert values["l2_norm_mean"] == 0.0 + assert values["linf_norm_mean"] == 0.0 + + +def test_psnr_uses_aggregate_mse_when_some_images_are_identical(): + raw = np.zeros((2, 1, 8, 8), dtype=np.float32) + cooked = raw.copy() + cooked[1] = 0.1 + values = compute_image_quality_metrics(raw, cooked, compute_lpips=False) + expected = 10 * math.log10(1.0 / 0.005) + assert math.isfinite(values["psnr_mean"]) + assert math.isclose(values["psnr_mean"], expected, rel_tol=1e-6) + assert values["psnr_std"] == 0.0 + + +def test_3d_volume_quality_without_lpips(): + raw = np.zeros((2, 1, 8, 8, 8), dtype=np.float32) + cooked = raw.copy() + cooked[1, :, 2:6, 2:6, 2:6] = 0.1 + values = compute_image_quality_metrics(raw, cooked, compute_lpips=False) + assert 0 < values["ssim_mean"] < 1 + assert math.isfinite(values["psnr_mean"]) + assert values["l2_norm_mean"] > 0 + assert math.isclose(values["linf_norm_mean"], 0.05, rel_tol=1e-6) + assert math.isnan(values["lpips_mean"]) + + +def test_3d_volume_rejects_lpips(): + volumes = np.zeros((1, 1, 8, 8, 8), dtype=np.float32) + with pytest.raises(ValueError, match="LPIPS is a 2D image metric"): + compute_image_quality_metrics(volumes, volumes, compute_lpips=True) + + +def test_lpips_resizes_medmnist_and_uses_bounded_batches(monkeypatch): + observed_shapes = [] + + class FakeLPIPS: + def to(self, _device): + return self + + def eval(self): + return self + + def __call__(self, raw, cooked): + observed_shapes.append(tuple(raw.shape)) + return (raw - cooked).square().mean(dim=(1, 2, 3), keepdim=True) + + monkeypatch.setitem(sys.modules, "lpips", SimpleNamespace(LPIPS=lambda net: FakeLPIPS())) + raw = np.zeros((130, 3, 28, 28), dtype=np.float32) + cooked = np.full_like(raw, 0.01) + values = _lpips_values(raw, cooked, device="cpu", batch_size=64) + + assert len(values) == 130 + assert observed_shapes == [(64, 3, 64, 64), (64, 3, 64, 64), (2, 3, 64, 64)] + + +def test_lpips_runtime_error_is_not_silently_converted_to_nan(monkeypatch): + class FailingLPIPS: + def to(self, _device): + return self + + def eval(self): + return self + + def __call__(self, _raw, _cooked): + raise RuntimeError("forward failed") + + monkeypatch.setitem(sys.modules, "lpips", SimpleNamespace(LPIPS=lambda net: FailingLPIPS())) + images = np.zeros((1, 3, 28, 28), dtype=np.float32) + with pytest.raises(RuntimeError, match="forward failed"): + _lpips_values(images, images, device="cpu") diff --git a/tests/test_runtime_analysis.py b/tests/test_runtime_analysis.py new file mode 100644 index 0000000..fd3030b --- /dev/null +++ b/tests/test_runtime_analysis.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest + +from run_runtime_analysis import LATENCY_UNIT, _append_macro_average + + +def _row(dataset: str, latency: float, samples: int) -> dict: + return { + "dataset": dataset, + "modality": "3d_volume", + "task_type": "multiclass", + "method": "datacook", + "surrogate_model": "resnet18_3d", + "victim_model": "resnet18_3d", + "seed": 0, + "epsilon": 32 / 255, + "setting": "runtime", + "attack_type": "", + "attack_strength": "", + "resolution": "28x28x28", + "num_samples": samples, + "batch_size": 16, + "device": "cuda", + "cooking_time_total_seconds": latency * samples / 1_000_000, + "cooking_time_per_sample": latency, + "raw_inference_time_per_sample": latency * 10, + "cooked_inference_time_per_sample": latency * 11, + "authorized_cooking_latency_per_sample": latency, + "overhead_ratio": 1.2, + "gpu_memory_mb": latency * 2, + "latency_unit": LATENCY_UNIT, + "aggregation": "dataset", + "repeats": 5, + "warmup": 1, + } + + +def test_runtime_macro_average_is_equal_weighted_across_datasets() -> None: + rows = [_row("small", 10.0, 10), _row("large", 30.0, 100)] + + output = _append_macro_average(rows) + + assert len(output) == 3 + average = output[-1] + assert average["dataset"] == "Average" + assert average["aggregation"] == "macro_average" + assert average["latency_unit"] == LATENCY_UNIT + assert average["num_samples"] == 110 + assert average["cooking_time_per_sample"] == pytest.approx(20.0) + assert average["raw_inference_time_per_sample"] == pytest.approx(200.0) + assert average["cooked_inference_time_per_sample"] == pytest.approx(220.0) + assert average["gpu_memory_mb"] == pytest.approx(40.0) + + +def test_single_runtime_record_does_not_add_duplicate_average() -> None: + rows = [_row("only", 10.0, 10)] + assert _append_macro_average(rows) == rows diff --git a/tests/test_tabular_epsilon_ablation.py b/tests/test_tabular_epsilon_ablation.py new file mode 100644 index 0000000..ddf59b2 --- /dev/null +++ b/tests/test_tabular_epsilon_ablation.py @@ -0,0 +1,246 @@ +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import torch + +from run_tabular_epsilon_ablation import ( + TabularNet, + cook_tabular_inputs, + load_tabular_split, + run_ablation, +) + + +FEATURES = [ + "Pregnancies", "Glucose", "BloodPressure", "SkinThickness", + "Insulin", "BMI", "DiabetesPedigreeFunction", "Age", +] + + +def test_tabular_cooker_respects_mask_and_linf_budget(): + torch.manual_seed(0) + model = TabularNet(8, dropout=0) + inputs = torch.randn(12, 8) + cooked, elapsed = cook_tabular_inputs( + model, + inputs, + epsilon=0.2, + attackable_indices=[5, 6], + x_min=inputs.min(dim=0).values, + x_max=inputs.max(dim=0).values, + num_steps=2, + step_size_ratio=0.5, + device=torch.device("cpu"), + ) + delta = cooked - inputs + assert elapsed >= 0 + assert float(delta.abs().max()) <= 0.2 + 1e-6 + np.testing.assert_allclose(delta[:, :5].numpy(), 0, atol=1e-7) + np.testing.assert_allclose(delta[:, 7:].numpy(), 0, atol=1e-7) + + +def test_multilabel_tabular_cooker_respects_budget(): + torch.manual_seed(0) + model = TabularNet(5, output_dim=3, dropout=0) + inputs = torch.randn(12, 5) + cooked, _ = cook_tabular_inputs( + model, + inputs, + epsilon=0.15, + attackable_indices=[1, 3], + x_min=inputs.min(dim=0).values, + x_max=inputs.max(dim=0).values, + num_steps=2, + step_size_ratio=0.5, + device=torch.device("cpu"), + task_type="multilabel", + ) + delta = cooked - inputs + assert float(delta.abs().max()) <= 0.15 + 1e-6 + np.testing.assert_allclose(delta[:, [0, 2, 4]].numpy(), 0, atol=1e-7) + + +def test_tabular_cooker_supports_absolute_step_size(): + torch.manual_seed(0) + model = TabularNet(8, dropout=0) + inputs = torch.randn(12, 8) + cooked, _ = cook_tabular_inputs( + model, + inputs, + epsilon=5.0, + attackable_indices=[5, 6], + x_min=torch.full((8,), -100.0), + x_max=torch.full((8,), 100.0), + num_steps=1, + step_size=0.05, + device=torch.device("cpu"), + ) + delta = cooked - inputs + assert float(delta.abs().max()) <= 0.05 + 1e-6 + assert float(delta[:, [5, 6]].abs().max()) >= 0.05 - 1e-6 + np.testing.assert_allclose(delta[:, :5].numpy(), 0, atol=1e-7) + np.testing.assert_allclose(delta[:, 7:].numpy(), 0, atol=1e-7) + + +def test_tabular_loader_supports_categorical_multiclass(tmp_path): + frame = pd.DataFrame({ + "Age": np.arange(30), + "Sex": np.tile(["m", "f"], 15), + "Category": np.repeat(["healthy", "a", "b"], 10), + }) + path = tmp_path / "multiclass.csv" + frame.to_csv(path, index=False) + split = load_tabular_split( + path, + features=["Age", "Sex"], + targets=["Category"], + task_type="multiclass", + seed=0, + test_size=0.2, + categorical_mappings={"Sex": {"m": 0, "f": 1}}, + target_mapping={"healthy": 0, "a": 1, "b": 2}, + ) + x_train, y_train, x_test, y_test, _, _ = split + assert x_train.shape == (24, 2) + assert x_test.shape == (6, 2) + assert y_train.dtype == torch.int64 + assert set(torch.cat((y_train, y_test)).tolist()) == {0, 1, 2} + + +def test_small_tabular_ablation_exports_unified_results(tmp_path): + rng = np.random.default_rng(7) + rows = 80 + frame = pd.DataFrame({ + feature: rng.normal(size=rows) + for feature in FEATURES + }) + frame["Outcome"] = np.tile([0, 1], rows // 2) + data_path = tmp_path / "diabetes.csv" + frame.to_csv(data_path, index=False) + artifact_root = tmp_path / "artifacts" + result_root = tmp_path / "results" + config = { + "dataset": { + "name": "Diabetes", + "data_path": str(data_path), + "features": FEATURES, + "target": "Outcome", + "test_size": 0.25, + }, + "model": {"hidden_dims": [8], "dropout": 0}, + "experiment": { + "seeds": [0], + "epsilons": [0.1], + "attackable_features": ["BMI", "DiabetesPedigreeFunction"], + "num_steps": 1, + "step_size": 0.05, + "baseline_epochs": 2, + "protected_epochs": 2, + "batch_size": 16, + "learning_rate": 0.01, + }, + "outputs": { + "artifact_root": str(artifact_root), + "result_root": str(result_root), + }, + } + args = SimpleNamespace( + data_path=None, + seeds=None, + epsilons=None, + device="cpu", + force=False, + dry_run=False, + ) + + outputs = run_ablation(config, tmp_path, args) + + assert set(outputs) == {"main_metrics", "cp_pp", "epsilon", "runtime"} + assert all(path.is_file() for path in outputs.values()) + epsilon = pd.read_csv(outputs["epsilon"]) + assert set(epsilon["metric"]) >= {"acc", "auc", "balanced_acc"} + assert set(epsilon["epsilon_scale"]) == {"standardized_feature_linf"} + assert set(epsilon["num_steps"]) == {1} + assert set(epsilon["step_size"]) == {0.05} + assert epsilon["step_size_ratio"].isna().all() + runtime = pd.read_csv(outputs["runtime"]) + assert len(runtime) == 1 + assert runtime.loc[0, "num_samples"] == 20 + assert runtime.loc[0, "cooking_time_per_sample"] >= 0 + assert runtime.loc[0, "overhead_ratio"] > 0 + with np.load( + artifact_root + / "Diabetes-datacook-sur=mlp-vic=mlp-seed=0-eps=0.1" + / "predictions" + / "authorized_cooked.npz" + ) as artifact: + assert artifact["y_true"].shape == (20,) + assert artifact["y_prob"].shape == (20, 2) + + +def test_small_multilabel_ablation_exports_multilabel_predictions(tmp_path): + rng = np.random.default_rng(11) + rows = 60 + features = ["F0", "F1", "F2", "F3"] + frame = pd.DataFrame({ + feature: rng.normal(size=rows) + for feature in features + }) + frame["L0"] = np.tile([0, 1], rows // 2) + frame["L1"] = np.tile([0, 0, 1, 1], rows // 4) + frame["L2"] = np.tile([0, 0, 0, 1, 1, 1], rows // 6) + data_path = tmp_path / "multilabel.csv" + frame.to_csv(data_path, index=False) + artifact_root = tmp_path / "artifacts" + result_root = tmp_path / "results" + config = { + "dataset": { + "name": "ToyMultilabel", + "task_type": "multilabel", + "data_path": str(data_path), + "features": features, + "targets": ["L0", "L1", "L2"], + "test_size": 0.25, + }, + "model": {"hidden_dims": [8], "dropout": 0}, + "experiment": { + "seeds": [0], + "epsilons": [0.1], + "attackable_features": ["F1", "F3"], + "num_steps": 1, + "step_size_ratio": 0.5, + "baseline_epochs": 2, + "protected_epochs": 2, + "batch_size": 16, + "learning_rate": 0.01, + "class_balance": True, + }, + "outputs": { + "artifact_root": str(artifact_root), + "result_root": str(result_root), + }, + } + args = SimpleNamespace( + data_path=None, + seeds=None, + epsilons=None, + device="cpu", + force=False, + dry_run=False, + ) + + outputs = run_ablation(config, tmp_path, args) + + epsilon = pd.read_csv(outputs["epsilon"]) + assert set(epsilon["metric"]) >= {"subset_acc", "macro_auc", "micro_f1"} + assert set(epsilon["task_type"]) == {"multilabel"} + with np.load( + artifact_root + / "ToyMultilabel-datacook-sur=mlp-vic=mlp-seed=0-eps=0.1" + / "predictions" + / "authorized_cooked.npz" + ) as artifact: + assert artifact["y_true"].shape == (15, 3) + assert artifact["y_pred"].shape == (15, 3) + assert artifact["y_prob"].shape == (15, 3) diff --git a/tests/test_tabular_legacy_statistics.py b/tests/test_tabular_legacy_statistics.py new file mode 100644 index 0000000..e2d5248 --- /dev/null +++ b/tests/test_tabular_legacy_statistics.py @@ -0,0 +1,25 @@ +import pandas as pd + +from analysis.tabular_legacy_statistics import build_tests + + +def test_tabular_legacy_statistics_pairs_seeds_and_applies_holm(): + rows = [] + for epsilon in (0.1, 0.25, 0.5, 1.0, 2.0, 5.0): + for seed in range(8): + rows.append({ + "dataset": "Diabetes", + "task_type": "binary", + "epsilon": epsilon, + "metric": "acc", + "seed": seed, + "clean": 0.8, + "authorized_cooked": 0.79, + "unauthorized_raw": 0.6, + }) + result = build_tests(pd.DataFrame(rows), {}) + assert len(result) == 12 + assert set(result["num_pairs"]) == {8} + assert set(result["num_nonzero"]) == {8} + assert result["p_value_holm"].notna().all() + assert result["significant_holm"].all() diff --git a/toolbox.py b/toolbox.py index 535eec3..52feff3 100644 --- a/toolbox.py +++ b/toolbox.py @@ -25,6 +25,24 @@ def __init__(self, seed=0, epsilon=0.03137254901, num_steps=20, step_size=0.0078 self.patchsize = patchsize np.random.seed(seed) + @staticmethod + def _classification_loss(criterion, model, images, labels, optimizer): + """Run ordinary CE/BCE criteria while preserving custom criteria.""" + if isinstance(criterion, torch.nn.BCEWithLogitsLoss): + logits = model(images) + targets = labels.float() + if logits.shape != targets.shape: + raise ValueError( + f'Multi-label target/logit shapes must match; got {targets.shape} and {logits.shape}.' + ) + return logits, criterion(logits, targets) + if isinstance(criterion, torch.nn.CrossEntropyLoss): + if hasattr(model, 'classify'): + model.classify = True + logits = model(images) + return logits, criterion(logits, labels) + return criterion(model, images, labels, optimizer) + def random_noise(self, noise_shape=[10, 3, 32, 32]): random_noise = torch.FloatTensor(*noise_shape).uniform_(-self.epsilon, self.epsilon).to(device) return random_noise @@ -41,13 +59,9 @@ def EM_attack(self, images, labels, model, optimizer, criterion, random_noise=No opt = torch.optim.Adam([perturb_img], lr=1e-3) opt.zero_grad() model.zero_grad() - if isinstance(criterion, torch.nn.CrossEntropyLoss): - if hasattr(model, 'classify'): - model.classify = True - logits = model(perturb_img) - loss = criterion(logits, labels) - else: - logits, loss = criterion(model, perturb_img, labels, optimizer) + logits, loss = self._classification_loss( + criterion, model, perturb_img, labels, optimizer + ) perturb_img.retain_grad() loss.backward() eta = self.step_size * perturb_img.grad.data.sign() * (-1) @@ -67,23 +81,22 @@ def datacook_v1(self, images, labels, model, optimizer, criterion, random_noise= eta = random_noise first_outputs = model(perturb_img) - _, first_predictions = torch.max(first_outputs, dim=1) - if(len(first_predictions)==4): - Pseudo = first_predictions.squeeze(0).long() + if isinstance(criterion, torch.nn.BCEWithLogitsLoss): + Pseudo = (torch.sigmoid(first_outputs) >= 0.5).float() else: - Pseudo = first_predictions.long() + _, first_predictions = torch.max(first_outputs, dim=1) + if(len(first_predictions)==4): + Pseudo = first_predictions.squeeze(0).long() + else: + Pseudo = first_predictions.long() for _ in range(self.num_steps): opt = torch.optim.Adam([perturb_img], lr=1e-3) opt.zero_grad() model.zero_grad() - if isinstance(criterion, torch.nn.CrossEntropyLoss): - if hasattr(model, 'classify'): - model.classify = True - logits = model(perturb_img) - loss = criterion(logits, Pseudo) - else: - logits, loss = criterion(model, perturb_img, Pseudo, optimizer) + logits, loss = self._classification_loss( + criterion, model, perturb_img, Pseudo, optimizer + ) perturb_img.retain_grad() loss.backward() eta = self.step_size * perturb_img.grad.data.sign() * (-1) @@ -103,24 +116,23 @@ def datacook_adv(self, images, labels, model, optimizer, criterion, random_noise perturb_img = Variable(torch.clamp(perturb_img, 0, 1), requires_grad=True) eta = random_noise first_outputs = model(perturb_img) - _, first_predictions = torch.max(first_outputs, dim=1) - if(len(first_predictions)==4): - Pseudo = first_predictions.squeeze(0).long() + if isinstance(criterion, torch.nn.BCEWithLogitsLoss): + Pseudo = (torch.sigmoid(first_outputs) >= 0.5).float() else: - Pseudo = first_predictions.long() + _, first_predictions = torch.max(first_outputs, dim=1) + if(len(first_predictions)==4): + Pseudo = first_predictions.squeeze(0).long() + else: + Pseudo = first_predictions.long() for _ in range(self.num_steps): # opt = torch.optim.SGD([perturb_img], lr=1e-3) opt = torch.optim.Adam([perturb_img], lr=1e-3) opt.zero_grad() model.zero_grad() - if isinstance(criterion, torch.nn.CrossEntropyLoss): - if hasattr(model, 'classify'): - model.classify = True - logits = model(perturb_img) - loss = criterion(logits, Pseudo) - else: - logits, loss = criterion(model, perturb_img, Pseudo, optimizer) + logits, loss = self._classification_loss( + criterion, model, perturb_img, Pseudo, optimizer + ) perturb_img.retain_grad() loss.backward() eta = self.step_size * perturb_img.grad.data.sign() @@ -137,7 +149,8 @@ def Adv_attack(self, images, labels, model, optimizer, criterion, random_noise=N if random_noise is None: random_noise = torch.FloatTensor(*images.shape).uniform_(-self.epsilon, self.epsilon).to(device) - labels = torch.squeeze(labels, 1).long().to(device) + if not isinstance(criterion, torch.nn.BCEWithLogitsLoss): + labels = torch.squeeze(labels, 1).long().to(device) perturb_img = Variable(images.data + random_noise, requires_grad=True) perturb_img = Variable(torch.clamp(perturb_img, 0, 1), requires_grad=True) @@ -146,11 +159,9 @@ def Adv_attack(self, images, labels, model, optimizer, criterion, random_noise=N opt = torch.optim.SGD([perturb_img], lr=1e-3) opt.zero_grad() model.zero_grad() - if isinstance(criterion, torch.nn.CrossEntropyLoss): - logits = model(perturb_img) - loss = criterion(logits, labels) - else: - logits, loss = criterion(model, perturb_img, labels, optimizer) + logits, loss = self._classification_loss( + criterion, model, perturb_img, labels, optimizer + ) perturb_img.retain_grad() loss.backward() @@ -405,7 +416,6 @@ def synthetic_3dperturbations(self, dataset, train_dataset, test_dataset): simple_data_idx = simple_label == label mini_simple_data_for_train = simple_data[simple_data_idx][0:int(sum(orig_data_idx))] -、 mini_simple_data_for_test = simple_data[simple_data_idx][int(sum(orig_data_idx)):int(sum(orig_data_idx)) + int(sum(arr_test_target == label))] orig_data_idx = np.squeeze(orig_data_idx, 1) @@ -424,4 +434,4 @@ def synthetic_3dperturbations(self, dataset, train_dataset, test_dataset): random_noise = random_noise.permute(0, 1, 2, 3) random_test_noise = random_test_noise.permute(0, 1, 2, 3) - return random_noise, random_test_noise \ No newline at end of file + return random_noise, random_test_noise diff --git a/trainer.py b/trainer.py index d56b910..bcc11c4 100644 --- a/trainer.py +++ b/trainer.py @@ -1,7 +1,7 @@ - """ - Adapted from https://github.com/HanxunH/Unlearnable-Examples - Implements a trainer extended functionality. - """ +""" +Adapted from https://github.com/HanxunH/Unlearnable-Examples. +Implements extended trainer functionality. +""" import time import models import torch @@ -35,6 +35,11 @@ def __init__(self, criterion, data_loader, logger, config, train_data_type, glob self.train_data_type = train_data_type print(self.target) + @property + def is_multilabel(self): + task = str(getattr(self.config, 'task_type', '') or '').replace('-', '').replace('_', '').lower() + return task == 'multilabel' or isinstance(self.criterion, torch.nn.BCEWithLogitsLoss) + def _reset_stats(self): self.loss_meters = util.AverageMeter() self.acc_meters = util.AverageMeter() @@ -47,6 +52,8 @@ def train(self, epoch, model, criterion, optimizer, random_noise=None): images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True) if random_noise is not None: + if self.is_multilabel: + raise ValueError('Class-indexed random noise is not defined for multi-label targets.') random_noise = random_noise.detach().to(device) for i in range(len(labels)): class_index = labels[i].item() @@ -69,10 +76,11 @@ def train_model(self, epoch, model, criterion, optimizer, random_noise=None): model.train() for i, (images, labels) in enumerate(self.data_loader[self.target]): images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True) - - if self.train_data_type == 'PathMNIST' or self.train_data_type == 'DermaMNIST' or self.train_data_type == 'OCTMNIST' or self.train_data_type == 'PneumoniaMNIST' or self.train_data_type == 'RetinaMNIST' or self.train_data_type == 'BreastMNIST' or self.train_data_type == 'BloodMNIST' or self.train_data_type == 'TissueMNIST' or self.train_data_type == 'OrganAMNIST' or self.train_data_type == 'OrganCMNIST' or self.train_data_type == 'OrganSMNIST': + if labels.ndim > 1 and labels.shape[1] == 1: labels = torch.squeeze(labels, 1).long().to(device) if random_noise is not None: + if self.is_multilabel: + raise ValueError('Class-indexed random noise is not defined for multi-label targets.') random_noise = random_noise.detach().to(device) for i in range(len(labels)): class_index = labels[i].item() @@ -329,7 +337,15 @@ def hook(layer, inp, out): def train_batch(self, images, labels, model, optimizer): model.zero_grad() optimizer.zero_grad() - if isinstance(self.criterion, torch.nn.CrossEntropyLoss): + if isinstance(self.criterion, torch.nn.BCEWithLogitsLoss): + logits = model(images) + targets = labels.float() + if targets.shape != logits.shape: + raise ValueError( + f'Multi-label target/logit shapes must match; got {targets.shape} and {logits.shape}.' + ) + loss = self.criterion(logits, targets) + elif isinstance(self.criterion, torch.nn.CrossEntropyLoss): logits = model(images) loss = self.criterion(logits, labels) else: @@ -337,7 +353,12 @@ def train_batch(self, images, labels, model, optimizer): loss.backward() grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), self.config.grad_clip) optimizer.step() - if logits.shape[1] >= 5: + if isinstance(self.criterion, torch.nn.BCEWithLogitsLoss): + predicted = torch.sigmoid(logits) >= 0.5 + expected = labels.bool() + acc = predicted.eq(expected).all(dim=1).float().mean().item() + acc5 = 1 + elif logits.shape[1] >= 5: acc, acc5 = util.accuracy(logits, labels, topk=(1, 5)) acc, acc5 = acc.item(), acc5.item() else: diff --git a/util.py b/util.py index f1ed01a..f612b5a 100644 --- a/util.py +++ b/util.py @@ -87,6 +87,8 @@ def accuracy(output, target, topk=(1,)): return res def save_model(filename, epoch, model, optimizer, scheduler, save_best=False, **kwargs): + if not filename.endswith('.pth'): + filename += '.pth' # Torch Save State Dict state = { 'epoch': epoch+1, @@ -97,15 +99,20 @@ def save_model(filename, epoch, model, optimizer, scheduler, save_best=False, ** for key, value in kwargs.items(): state[key] = value torch.save(state, filename) - filename += '_best.pth' + filename = filename[:-4] + '_best.pth' if save_best: torch.save(state, filename) return def load_model(filename, model, optimizer, scheduler, **kwargs): # Load Torch State Dict - filename = filename + '.pth' - checkpoints = torch.load(filename, map_location=device) + if not filename.endswith('.pth'): + suffixed = filename + '.pth' + filename = suffixed if os.path.exists(suffixed) else filename + try: + checkpoints = torch.load(filename, map_location=device, weights_only=False) + except TypeError: + checkpoints = torch.load(filename, map_location=device) model.load_state_dict(checkpoints['model_state_dict']) if optimizer is not None and checkpoints['optimizer_state_dict'] is not None: optimizer.load_state_dict(checkpoints['optimizer_state_dict']) @@ -140,4 +147,3 @@ def update(self, val, n=1): self.count += n self.avg = self.sum / self.count self.max = max(self.max, val) - diff --git a/volume_io.py b/volume_io.py new file mode 100644 index 0000000..ee90e4e --- /dev/null +++ b/volume_io.py @@ -0,0 +1,59 @@ +"""I/O helpers for channel-first cooked medical volumes.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + + +def save_cooked_array_npz(path: str | Path, images, labels) -> Path: + """Atomically save normalized channel-first cooked arrays and labels. + + Unlike the legacy 3D helper, this function preserves a multi-dimensional + label matrix such as ChestMNIST's ``(N,14)`` multi-hot targets. + """ + output = Path(path) + arrays = np.asarray(images, dtype=np.float32) + targets = np.asarray(labels) + if arrays.ndim not in {4, 5}: + raise ValueError( + f"Cooked images must have shape (N,C,H,W) or (N,C,D,H,W); got {arrays.shape}." + ) + if targets.ndim not in {1, 2}: + raise ValueError(f"Cooked labels must have shape (N,) or (N,L); got {targets.shape}.") + if len(arrays) == 0 or len(arrays) != len(targets): + raise ValueError( + f"Cooked images/labels must be non-empty and aligned; " + f"got {len(arrays)} images and {len(targets)} labels." + ) + output.parent.mkdir(parents=True, exist_ok=True) + temporary = output.with_name(output.name + ".tmp.npz") + np.savez_compressed( + temporary, + perturb_images=arrays, + perturb_labels=targets, + ) + temporary.replace(output) + return output + + +def save_cooked_volume_npz(path: str | Path, images, labels) -> Path: + """Atomically save cooked volumes for ``Data3DFolderWithLabel``. + + Images use the ``(N, C, D, H, W)`` contract and labels are flattened to + ``(N,)`` so CrossEntropyLoss receives one class index per volume. + """ + output = Path(path) + volumes = np.asarray(images, dtype=np.float32) + targets = np.asarray(labels, dtype=np.int64).reshape(-1) + if volumes.ndim != 5: + raise ValueError( + f"Cooked 3D images must have shape (N,C,D,H,W); got {volumes.shape}." + ) + if len(volumes) == 0 or len(volumes) != len(targets): + raise ValueError( + f"Cooked 3D images/labels must be non-empty and aligned; " + f"got {len(volumes)} images and {len(targets)} labels." + ) + return save_cooked_array_npz(output, volumes, targets)