Adjust ci synthesis evaluation presets to speed it up

Do not evaluate known-to-be-poor configurations so the synthesis evaluation fits under GitHub Actions job runtime limit (6 hours).
ryukzak · Sep 23, 2023 · e6d4dd6 · e6d4dd6
1 parent 50ab618
commit e6d4dd6
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 1 deletion.
diff --git a/ml/synthesis/src/scripts/evaluation_configs/ci.json b/ml/synthesis/src/scripts/evaluation_configs/ci.json
@@ -0,0 +1,16 @@
+{
+  "output_dir": "evaluation",
+  "nitta_run_command": "stack exec nitta --",
+  "nitta_run_timeout_s": 60,
+  "measurement_tries": 3,
+  "examples": "all",
+  "constant_args": "-e",
+  "with_ml_backend": true,
+  "evaluated_args": {
+    "preset": {
+      "default-sota": "--method=StateOfTheArt",
+      "default-td": "--method=TopDownByScore --depth-base=1.4",
+      "ml-td": "--score=ml_production --method=TopDownByScore --depth-base=1.2"
+    }
+  }
+}
diff --git a/ml/synthesis/src/scripts/train_evaluate_in_ci.py b/ml/synthesis/src/scripts/train_evaluate_in_ci.py
@@ -57,7 +57,7 @@ def _parse_args() -> Namespace:
 
     rating_df = None
     if not args.skip_evaluation:
-        config = read_evaluation_config_from_json(EVALUATION_CONFIGS_DIR / "full.json")
+        config = read_evaluation_config_from_json(EVALUATION_CONFIGS_DIR / "ci.json")
         config.output_dir = EVALUATIONS_DIR / "ci"
         if config.output_dir.exists():
             shutil.rmtree(config.output_dir)