From e6d4dd6e7f79563d955b9b4bda310633a41f2116 Mon Sep 17 00:00:00 2001 From: Ilya Burakov Date: Sat, 23 Sep 2023 15:52:01 +0000 Subject: [PATCH] Adjust ci synthesis evaluation presets to speed it up Do not evaluate known-to-be-poor configurations so the synthesis evaluation fits under GitHub Actions job runtime limit (6 hours). --- .../src/scripts/evaluation_configs/ci.json | 16 ++++++++++++++++ ml/synthesis/src/scripts/train_evaluate_in_ci.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 ml/synthesis/src/scripts/evaluation_configs/ci.json diff --git a/ml/synthesis/src/scripts/evaluation_configs/ci.json b/ml/synthesis/src/scripts/evaluation_configs/ci.json new file mode 100644 index 000000000..76b10b907 --- /dev/null +++ b/ml/synthesis/src/scripts/evaluation_configs/ci.json @@ -0,0 +1,16 @@ +{ + "output_dir": "evaluation", + "nitta_run_command": "stack exec nitta --", + "nitta_run_timeout_s": 60, + "measurement_tries": 3, + "examples": "all", + "constant_args": "-e", + "with_ml_backend": true, + "evaluated_args": { + "preset": { + "default-sota": "--method=StateOfTheArt", + "default-td": "--method=TopDownByScore --depth-base=1.4", + "ml-td": "--score=ml_production --method=TopDownByScore --depth-base=1.2" + } + } +} diff --git a/ml/synthesis/src/scripts/train_evaluate_in_ci.py b/ml/synthesis/src/scripts/train_evaluate_in_ci.py index 01a22de88..0b8320649 100644 --- a/ml/synthesis/src/scripts/train_evaluate_in_ci.py +++ b/ml/synthesis/src/scripts/train_evaluate_in_ci.py @@ -57,7 +57,7 @@ def _parse_args() -> Namespace: rating_df = None if not args.skip_evaluation: - config = read_evaluation_config_from_json(EVALUATION_CONFIGS_DIR / "full.json") + config = read_evaluation_config_from_json(EVALUATION_CONFIGS_DIR / "ci.json") config.output_dir = EVALUATIONS_DIR / "ci" if config.output_dir.exists(): shutil.rmtree(config.output_dir)