Add all candidate eval configs

align_system/configs/experiment/phase1_evaluation/baseline_Mistral-7B-Instruct-0.3_eval_live.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_transformers_structured_baseline
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: eval
+  training_session: null
+  username: "ALIGN-ADM-OutlinesBaseline-Mistral-7B-Instruct-v0.3"
+
+adm:
+  instance:
+    precision: half
+    model_name: mistralai/Mistral-7B-Instruct-v0.3
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/baseline-Mistral-7B-Instruct-v0.3/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/baseline_Phi-3_eval_live.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_transformers_structured_baseline
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: eval
+  training_session: null
+  username: "ALIGN-ADM-OutlinesBaseline-Phi-3-medium-4k-instruct"
+
+adm:
+  instance:
+    precision: half
+    model_name: microsoft/Phi-3-medium-4k-instruct
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/baseline-Phi-3-medium-4k-instruct/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/baseline_eval_live.yaml

@@ -0,0 +1,22 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_transformers_structured_baseline
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: soartech
+  training_session: null
+  username: "ALIGN-ADM-OutlinesBaseline"
+
+adm:
+  instance:
+    precision: half
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/baseline_soartech/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/baseline_llama3.1_8B_eval_live.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_transformers_structured_baseline
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: eval
+  training_session: null
+  username: "ALIGN-ADM-OutlinesBaseline-Llama-3.1-8B-Instruct"
+
+adm:
+  instance:
+    precision: half
+    model_name: meta-llama/Llama-3.1-8B-Instruct
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/baseline-Llama-3.1-8B-Instruct/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/baseline_llama3.2_3B_eval_live.yaml

@@ -0,0 +1,23 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_transformers_structured_baseline
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: eval
+  training_session: null
+  username: "ALIGN-ADM-OutlinesBaseline-Llama-3.2-3B-Instruct"
+
+adm:
+  instance:
+    precision: half
+    model_name: meta-llama/Llama-3.2-3B-Instruct
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/baseline-Llama-3.2-3B-Instruct/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_0-10scale_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-0-10Scale"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.GreedySampler
+    model_name: mistralai/Mistral-7B-Instruct-v0.2
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 1
+    predict_outcomes: false
+    generator_batch_size: 5
+    kdma_descriptions_map: "align_system/prompt_engineering/kdma_descriptions_0-10-scale.yml"
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 5
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+force_determinism: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-0-10Scale/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_3icl_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-3ICL"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.2
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 10
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 3
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-3ICL/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_7icl_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-7ICL"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.2
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 10
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 7
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-7ICL/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_Mistral-7B-Instruct-0.3_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-10Sample"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.3
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 10
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 5
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-10Sample/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.2
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 10
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 5
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_20sample_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-20Sample"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.2
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 20
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 5
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-20Sample/${now:%Y-%m-%d__%H-%M-%S}'

align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_Mistral-7B-Instruct-0.3_adept_eval.yaml

@@ -0,0 +1,37 @@
+# @package _global_
+defaults:
+  - override /adm: outlines_regression_aligned_comparative/incontext_phase1
+  - override /interface: ta3
+
+interface:
+  api_endpoint: "https://darpaitm.caci.com"
+  session_type: adept
+  training_session: null
+  username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-5Sample"
+
+adm:
+  instance:
+    precision: half
+    sampler:
+      _target_: outlines.samplers.MultinomialSampler
+      temperature: 0.7
+    model_name: mistralai/Mistral-7B-Instruct-v0.3
+  inference_kwargs:
+    kdma_score_examples: true
+    num_samples: 5
+    predict_outcomes: false
+    generator_batch_size: 5
+    incontext:
+      sort_actions: true
+      normalization: null
+      number: 5
+      leave_one_out_strategy: null
+      most_similar_first: false
+
+sort_available_actions: true
+align_to_target: true
+save_last_unstructured_state_per_scenario: true
+
+hydra:
+  run:
+    dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-5Sample/${now:%Y-%m-%d__%H-%M-%S}'