diff --git a/align_system/configs/experiment/phase1_evaluation/baseline_Mistral-7B-Instruct-0.3_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/baseline_Mistral-7B-Instruct-0.3_eval_live.yaml new file mode 100644 index 0000000..0665f34 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/baseline_Mistral-7B-Instruct-0.3_eval_live.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /adm: outlines_transformers_structured_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-Mistral-7B-Instruct-v0.3" + +adm: + instance: + precision: half + model_name: mistralai/Mistral-7B-Instruct-v0.3 + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/baseline-Mistral-7B-Instruct-v0.3/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/baseline_Phi-3_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/baseline_Phi-3_eval_live.yaml new file mode 100644 index 0000000..10fefe9 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/baseline_Phi-3_eval_live.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /adm: outlines_transformers_structured_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-Phi-3-medium-4k-instruct" + +adm: + instance: + precision: half + model_name: microsoft/Phi-3-medium-4k-instruct + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/baseline-Phi-3-medium-4k-instruct/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/baseline_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/baseline_eval_live.yaml new file mode 100644 index 0000000..cd30978 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/baseline_eval_live.yaml @@ -0,0 +1,22 @@ +# @package _global_ +defaults: + - override /adm: outlines_transformers_structured_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: soartech + training_session: null + username: "ALIGN-ADM-OutlinesBaseline" + +adm: + instance: + precision: half + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/baseline_soartech/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/baseline_llama3.1_8B_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/baseline_llama3.1_8B_eval_live.yaml new file mode 100644 index 0000000..34336a3 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/baseline_llama3.1_8B_eval_live.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /adm: outlines_transformers_structured_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-Llama-3.1-8B-Instruct" + +adm: + instance: + precision: half + model_name: meta-llama/Llama-3.1-8B-Instruct + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/baseline-Llama-3.1-8B-Instruct/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/baseline_llama3.2_3B_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/baseline_llama3.2_3B_eval_live.yaml new file mode 100644 index 0000000..b1fbf16 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/baseline_llama3.2_3B_eval_live.yaml @@ -0,0 +1,23 @@ +# @package _global_ +defaults: + - override /adm: outlines_transformers_structured_baseline + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: eval + training_session: null + username: "ALIGN-ADM-OutlinesBaseline-Llama-3.2-3B-Instruct" + +adm: + instance: + precision: half + model_name: meta-llama/Llama-3.2-3B-Instruct + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/baseline-Llama-3.2-3B-Instruct/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_0-10scale_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_0-10scale_eval.yaml new file mode 100644 index 0000000..3b6fd87 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_0-10scale_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-0-10Scale" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + kdma_descriptions_map: "align_system/prompt_engineering/kdma_descriptions_0-10-scale.yml" + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-0-10Scale/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_3icl_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_3icl_adept_eval.yaml new file mode 100644 index 0000000..c2cc3d9 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_3icl_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-3ICL" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 10 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 3 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-3ICL/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_7icl_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_7icl_adept_eval.yaml new file mode 100644 index 0000000..5a1dfc2 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_7icl_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-7ICL" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 10 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 7 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample-7ICL/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_Mistral-7B-Instruct-0.3_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_Mistral-7B-Instruct-0.3_adept_eval.yaml new file mode 100644 index 0000000..a333741 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_Mistral-7B-Instruct-0.3_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-10Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.3 + inference_kwargs: + kdma_score_examples: true + num_samples: 10 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-10Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_adept_eval.yaml new file mode 100644 index 0000000..f0005a4 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_10sample_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 10 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-10Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_20sample_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_20sample_adept_eval.yaml new file mode 100644 index 0000000..3006438 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_20sample_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-20Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 20 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-20Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_Mistral-7B-Instruct-0.3_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_Mistral-7B-Instruct-0.3_adept_eval.yaml new file mode 100644 index 0000000..5fb4d04 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_Mistral-7B-Instruct-0.3_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-5Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.3 + inference_kwargs: + kdma_score_examples: true + num_samples: 5 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT-5Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_adept_eval.yaml new file mode 100644 index 0000000..96321b5 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_5sample_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-5Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 5 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-5Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_Mistral-7B-Instruct-0.3_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_Mistral-7B-Instruct-0.3_adept_eval.yaml new file mode 100644 index 0000000..13ebe38 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_Mistral-7B-Instruct-0.3_adept_eval.yaml @@ -0,0 +1,36 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.3 + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.3-ADEPT/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_Phi-3_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_Phi-3_adept_eval.yaml new file mode 100644 index 0000000..7ae954d --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_Phi-3_adept_eval.yaml @@ -0,0 +1,36 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Phi-3-medium-4k-instruct-ADEPT" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: microsoft/Phi-3-medium-4k-instruct + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Phi-3-medium-4k-instruct-ADEPT/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_adept_eval_action_type_icl.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_adept_eval_action_type_icl.yaml new file mode 100644 index 0000000..aba1e6d --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_adept_eval_action_type_icl.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-ATICL" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + method: matching_actions + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-ATICL/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_charmatchICL_soartech_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_charmatchICL_soartech_eval.yaml new file mode 100644 index 0000000..07447cd --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_charmatchICL_soartech_eval.yaml @@ -0,0 +1,39 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: soartech + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-SoarTech-CharMatchICL" +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + distribution_matching: cumulative_kde + kde_norm: priornorm + priornorm_factor: 0.5 + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + most_similar_first: false + sort_actions: true + normalization: null + number: 4 + leave_one_out_strategy: null + method: matching_characters + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-SoarTech-CharMatchICL/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_history_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_history_adept_eval.yaml new file mode 100644 index 0000000..f7ce4f1 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_history_adept_eval.yaml @@ -0,0 +1,38 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-History" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + distribution_matching: cumulative_average + kde_norm: rawscores + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-History/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.1_8B_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.1_8B_adept_eval.yaml new file mode 100644 index 0000000..7a0c91f --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.1_8B_adept_eval.yaml @@ -0,0 +1,36 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Llama-3.1-8B-Instruct-ADEPT" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: meta-llama/Llama-3.1-8B-Instruct + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Llama-3.1-8B-Instruct-ADEPT/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2B_matching_chars_soartech_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2B_matching_chars_soartech_eval.yaml new file mode 100644 index 0000000..8a16cb7 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2B_matching_chars_soartech_eval.yaml @@ -0,0 +1,39 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: soartech + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Llama-3.2-3B-Instruct-SoarTech-MatchingChars" +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: meta-llama/Llama-3.2-3B-Instruct + inference_kwargs: + distribution_matching: cumulative_kde + kde_norm: priornorm + priornorm_factor: 0.5 + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + most_similar_first: false + sort_actions: true + normalization: null + number: 4 + leave_one_out_strategy: null + method: matching_characters + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Llama-3.2-3B-Instruct-SoarTech-MatchingChars/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2_3B_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2_3B_adept_eval.yaml new file mode 100644 index 0000000..e02a5ac --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_llama3.2_3B_adept_eval.yaml @@ -0,0 +1,36 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Llama-3.2-3B-Instruct-ADEPT" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: meta-llama/Llama-3.2-3B-Instruct + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Llama-3.2-3B-Instruct-ADEPT/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_adept_eval.yaml new file mode 100644 index 0000000..e15edfb --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_adept_eval.yaml @@ -0,0 +1,30 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noICL" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: true + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noICL/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_noTemplate_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_noTemplate_adept_eval.yaml new file mode 100644 index 0000000..62ebb80 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_noICL_noTemplate_adept_eval.yaml @@ -0,0 +1,30 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noICL-noTemplate" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: false + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noICL-noTemplate/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_5sample_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_5sample_adept_eval.yaml new file mode 100644 index 0000000..196dc57 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_5sample_adept_eval.yaml @@ -0,0 +1,37 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noTemplate-5Sample" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.MultinomialSampler + temperature: 0.7 + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: false + num_samples: 5 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +sort_available_actions: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noTemplate-5Sample/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_adept_eval.yaml b/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_adept_eval.yaml new file mode 100644 index 0000000..418e275 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/comp_reg_noTemplate_adept_eval.yaml @@ -0,0 +1,36 @@ +# @package _global_ +defaults: + - override /adm: outlines_regression_aligned_comparative/incontext_phase1 + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: adept + training_session: null + username: "ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noTemplate" + +adm: + instance: + precision: half + sampler: + _target_: outlines.samplers.GreedySampler + model_name: mistralai/Mistral-7B-Instruct-v0.2 + inference_kwargs: + kdma_score_examples: false + num_samples: 1 + predict_outcomes: false + generator_batch_size: 5 + incontext: + sort_actions: true + normalization: null + number: 5 + leave_one_out_strategy: null + most_similar_first: false + +force_determinism: true +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/ALIGN-ADM-ComparativeRegression-Mistral-7B-Instruct-v0.2-ADEPT-noTemplate/${now:%Y-%m-%d__%H-%M-%S}' diff --git a/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_ingroup_bias_train.yaml b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_ingroup_bias_train.yaml new file mode 100644 index 0000000..847bd17 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_ingroup_bias_train.yaml @@ -0,0 +1,19 @@ +# @package _global_ +defaults: + - override /adm: hybrid_kaleido + - override /interface: ta3 + +adm: + inference_kwargs: + kdma_descriptions_map: 'align_system/algorithms/lib/templates/kdma_descriptions_short_dry_run_eval.yml' + +interface: + api_endpoint: "http://127.0.0.1:8089" + session_type: adept + scenario_ids: + - DryRunEval.IO1 + - DryRunEval.IO2 + - DryRunEval.IO3 + training_session: full + +align_to_target: true diff --git a/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_moral_judgement_train.yaml b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_moral_judgement_train.yaml new file mode 100644 index 0000000..4dd2b32 --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_adept_moral_judgement_train.yaml @@ -0,0 +1,18 @@ +# @package _global_ +defaults: + - override /adm: hybrid_kaleido + - override /interface: ta3 + +adm: + inference_kwargs: + kdma_descriptions_map: 'align_system/algorithms/lib/templates/kdma_descriptions_short_dry_run_eval.yml' + +interface: + api_endpoint: "http://127.0.0.1:8089" + session_type: adept + scenario_ids: + - DryRunEval.MJ1 + - DryRunEval.MJ3 + training_session: full + +align_to_target: true diff --git a/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_soartech_qol_train.yaml b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_soartech_qol_train.yaml new file mode 100644 index 0000000..49b555e --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/hybrid_kaleido_soartech_qol_train.yaml @@ -0,0 +1,17 @@ +# @package _global_ +defaults: + - override /adm: hybrid_kaleido + - override /interface: ta3 + +adm: + inference_kwargs: + kdma_descriptions_map: 'align_system/algorithms/lib/templates/kdma_descriptions_short_dry_run_eval.yml' + +interface: + api_endpoint: "http://127.0.0.1:8089" + session_type: soartech + scenario_ids: + - + training_session: full + +align_to_target: true diff --git a/align_system/configs/experiment/phase1_evaluation/random_eval_live.yaml b/align_system/configs/experiment/phase1_evaluation/random_eval_live.yaml new file mode 100644 index 0000000..0313fed --- /dev/null +++ b/align_system/configs/experiment/phase1_evaluation/random_eval_live.yaml @@ -0,0 +1,17 @@ +# @package _global_ +defaults: + - override /adm: random + - override /interface: ta3 + +interface: + api_endpoint: "https://darpaitm.caci.com" + session_type: soartech + training_session: null + username: "ALIGN-ADM-Random" + +align_to_target: true +save_last_unstructured_state_per_scenario: true + +hydra: + run: + dir: 'phase1_eval_live/random_soartech/${now:%Y-%m-%d__%H-%M-%S}'