add discovery metrics to grace-1L-oam.yml and grace-2L-oam.yml

janosh · Feb 7, 2025 · 4f139f6 · 4f139f6
1 parent 8074b3b
commit 4f139f6
Show file tree

Hide file tree

Showing 2 changed files with 114 additions and 0 deletions.
diff --git a/models/grace/grace-1L-oam.yml b/models/grace/grace-1L-oam.yml
@@ -66,3 +66,60 @@ metrics:
     pred_file: models/grace/grace-1L-oam/2025-02-02-wbm-IS2RE.csv.gz
     pred_file_url:
     pred_col: e_form_per_atom_grace
+    full_test_set:
+      F1: 0.619  # fraction
+      DAF: 4.647 # dimensionless
+      Precision: 0.797 # fraction
+      Recall: 0.505 # fraction
+      Accuracy: 0.893 # fraction
+      TPR: 0.505 # fraction
+      FPR: 0.027 # fraction
+      TNR: 0.973 # fraction
+      FNR: 0.495 # fraction
+      TP: 22278.0 # count
+      FP: 5663.0 # count
+      TN: 207208.0 # count
+      FN: 21814.0 # count
+      MAE: 0.121 # eV/atom
+      RMSE: 0.218 # eV/atom
+      R2: -0.463 # dimensionless
+      missing_preds: 2 # count
+      missing_percent: 0.00% # fraction
+    most_stable_10k:
+      F1: 0.974  # fraction
+      DAF: 6.211 # dimensionless
+      Precision: 0.95 # fraction
+      Recall: 1.0 # fraction
+      Accuracy: 0.95 # fraction
+      TPR: 1.0 # fraction
+      FPR: 1.0 # fraction
+      TNR: 0.0 # fraction
+      FNR: 0.0 # fraction
+      TP: 9495.0 # count
+      FP: 505.0 # count
+      TN: 0.0 # count
+      FN: 0.0 # count
+      MAE: 0.032 # eV/atom
+      RMSE: 0.083 # eV/atom
+      R2: 0.856 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction
+    unique_prototypes:
+      F1: 0.627  # fraction
+      DAF: 5.355 # dimensionless
+      Precision: 0.819 # fraction
+      Recall: 0.508 # fraction
+      Accuracy: 0.906 # fraction
+      TPR: 0.508 # fraction
+      FPR: 0.021 # fraction
+      TNR: 0.979 # fraction
+      FNR: 0.492 # fraction
+      TP: 16963.0 # count
+      FP: 3758.0 # count
+      TN: 178356.0 # count
+      FN: 16411.0 # count
+      MAE: 0.12 # eV/atom
+      RMSE: 0.215 # eV/atom
+      R2: -0.369 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction
diff --git a/models/grace/grace-2L-oam.yml b/models/grace/grace-2L-oam.yml
@@ -66,3 +66,60 @@ metrics:
     pred_file: models/grace/grace-2L-oam/2025-01-28-wbm-IS2RE.csv.gz
     pred_file_url:
     pred_col: e_form_per_atom_grace
+    full_test_set:
+      F1: 0.634  # fraction
+      DAF: 4.77 # dimensionless
+      Precision: 0.819 # fraction
+      Recall: 0.518 # fraction
+      Accuracy: 0.898 # fraction
+      TPR: 0.518 # fraction
+      FPR: 0.024 # fraction
+      TNR: 0.976 # fraction
+      FNR: 0.482 # fraction
+      TP: 22824.0 # count
+      FP: 5061.0 # count
+      TN: 207810.0 # count
+      FN: 21268.0 # count
+      MAE: 0.123 # eV/atom
+      RMSE: 0.223 # eV/atom
+      R2: -0.535 # dimensionless
+      missing_preds: 2 # count
+      missing_percent: 0.00% # fraction
+    most_stable_10k:
+      F1: 0.978  # fraction
+      DAF: 6.262 # dimensionless
+      Precision: 0.957 # fraction
+      Recall: 1.0 # fraction
+      Accuracy: 0.957 # fraction
+      TPR: 1.0 # fraction
+      FPR: 1.0 # fraction
+      TNR: 0.0 # fraction
+      FNR: 0.0 # fraction
+      TP: 9573.0 # count
+      FP: 427.0 # count
+      TN: 0.0 # count
+      FN: 0.0 # count
+      MAE: 0.03 # eV/atom
+      RMSE: 0.085 # eV/atom
+      R2: 0.849 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction
+    unique_prototypes:
+      F1: 0.644  # fraction
+      DAF: 5.506 # dimensionless
+      Precision: 0.842 # fraction
+      Recall: 0.521 # fraction
+      Accuracy: 0.911 # fraction
+      TPR: 0.521 # fraction
+      FPR: 0.018 # fraction
+      TNR: 0.982 # fraction
+      FNR: 0.479 # fraction
+      TP: 17396.0 # count
+      FP: 3270.0 # count
+      TN: 178844.0 # count
+      FN: 15978.0 # count
+      MAE: 0.122 # eV/atom
+      RMSE: 0.221 # eV/atom
+      R2: -0.441 # dimensionless
+      missing_preds: 0 # count
+      missing_percent: 0.00% # fraction