Add specific configs for aligned adept and aligned soartech for live …

…eval
ITM-Kitware · Aug 27, 2024 · 5975497 · 5975497
1 parent 90e7fc7
commit 5975497
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 4 deletions.
diff --git a/README.md b/README.md
@@ -134,10 +134,16 @@ run_align_system +experiment=dry_run_evaluation/random_eval_live
 run_align_system +experiment=dry_run_evaluation/outlines_baseline_eval_live
 ```
 
-### Aligned ADM 1 (Comparative Regression + ICL + Template ADM)
+### Aligned ADM 1 (Comparative Regression + ICL + Template ADM) (ADEPT eval scenarios)
 
 ```
-run_align_system +experiment=dry_run_evaluation/comparative_regression_icl_template_eval_live
+run_align_system +experiment=dry_run_evaluation/comparative_regression_icl_template_eval_live_adept
+```
+
+### Aligned ADM 1 (Comparative Regression + ICL + Template ADM) (SoarTech eval scenarios)
+
+```
+run_align_system +experiment=dry_run_evaluation/comparative_regression_icl_template_eval_live_soartech
 ```
 
 ### Aligned ADM 2 (Hybrid Regression ADM)

diff --git a/...ve_regression_icl_template_eval_live.yaml → ...ression_icl_template_eval_live_adept.yaml b/...ve_regression_icl_template_eval_live.yaml → ...ression_icl_template_eval_live_adept.yaml
@@ -5,7 +5,7 @@ defaults:
 
 interface:
   api_endpoint: "https://darpaitm.caci.com"
-  session_type: eval
+  session_type: adept
   training_session: false
   username: "ALIGN-ADM-ComparativeRegression+ICL+Template"
 
@@ -24,4 +24,4 @@ save_last_unstructured_state_per_scenario: true
 
 hydra:
   run:
-    dir: 'comparative_regression_icl_template_eval_live/${now:%Y-%m-%d__%H-%M-%S}'
+    dir: 'comparative_regression_icl_template_eval_live_adept/${now:%Y-%m-%d__%H-%M-%S}'
diff --git a/...experiment/dry_run_evaluation/comparative_regression_icl_template_eval_live_soartech.yaml b/...experiment/dry_run_evaluation/comparative_regression_icl_template_eval_live_soartech.yaml
@@ -0,0 +1,30 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: soartech
+  training_session: false
+  username: "ALIGN-ADM-ComparativeRegression+ICL+Template"
+
+adm:
+  instance:
+    precision: half
+
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 5
+    predict_outcomes: false
+    generator_batch_size: 5
+
+    incontext:
+      number: 1
+
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'comparative_regression_icl_template_eval_live_soartech/${now:%Y-%m-%d__%H-%M-%S}'