Merge pull request #224 from michalc/feat/support-nightlies

feat: support some OpenTTD and OpenGFX nightlies

README.md

@@ -180,16 +180,19 @@ The core function of OpenTTDLab is the `run_experiments` function, used to run a
 
    The maximum number of workers to use to run OpenTTD in parallel. If`None`, then `os.cpu_count()` defined how many workers run.
 
-- `openttd_version=None`
+- `openttd_version=None`<br>
+  `opengfx_version=None`
 
-   The version of OpenTTD to use. If `None`, the latest version available at `openttd_cdn_url` is used.
+   The version of OpenTTD or OpenGFX to download and use. For both of these:
 
-   > **Caution**
-   > OpenTTDLab currently does not work with OpenTTD 14.0 or later. The latest version of OpenTTD known to work is 13.4.
+   - If `None`, the latest release version available at `openttd_cdn_url` is downloaded and used.
+
+   - If starting with 8 digits followed by a dash, it is assumed this is a date and so a nightly version, for example `'20230323-master-g83eb73a9b2'`.
 
-- `opengfx_version=None`
+   - Otherwise a release version is assumed, for example `'13.4'`.
 
-   The version of OpenGFX to use. If `None`, the latest version available at `openttd_cdn_url` is used.
+   > **Caution**
+   > OpenTTDLab currently does not work with OpenTTD 14.0 or later. The latest version of OpenTTD known to work is 13.4.
 
 - `openttd_cdn_url='https://cdn.openttd.org/`
 

openttdlab.py

@@ -15,6 +15,7 @@
 import os
 import os.path
 import platform
+import re
 import shutil
 import stat
 import struct
@@ -178,8 +179,14 @@ def extract_zip(archive_location, output_dir):
             openttd_version = str(get_yaml(client, openttd_cdn_url + 'openttd-releases/latest.yaml')['latest'][0]['version'])
         if opengfx_version is None:
             opengfx_version = str(get_yaml(client, openttd_cdn_url + 'opengfx-releases/latest.yaml')['latest'][0]['version'])
-        openttd_manifest = get_yaml(client, openttd_cdn_url + 'openttd-releases/' + openttd_version + '/manifest.yaml')
-        opengfx_manifest = get_yaml(client, openttd_cdn_url + 'opengfx-releases/' + opengfx_version + '/manifest.yaml')
+        openttd_path = \
+            'openttd-nightlies/' + openttd_version[:4] + '/' + openttd_version + '/' if re.match(r'\d{8}-', openttd_version) else \
+            'openttd-releases/' + openttd_version + '/'
+        opengfx_path = \
+            'opengfx-nightlies/' + opengfx_version + '/' if re.match(r'\d{8}-', opengfx_version) else \
+            'opengfx-releases/' + opengfx_version + '/'
+        openttd_manifest = get_yaml(client, openttd_cdn_url + openttd_path + 'manifest.yaml')
+        opengfx_manifest = get_yaml(client, openttd_cdn_url + opengfx_path + 'manifest.yaml')
 
         # Find file details in manifest
         openttd_filename = f"{openttd_manifest['base']}{operating_system}-{architecture}.{openttd_extension}"
@@ -191,8 +198,8 @@ def extract_zip(archive_location, output_dir):
         cache_dir = get_cache_dir()
         openttd_archive_location = os.path.join(cache_dir, openttd_filename)
         opengfx_archive_location = os.path.join(cache_dir, opengfx_filename)
-        stream_to_file_if_necessary(client, openttd_cdn_url + 'openttd-releases/' + openttd_version + '/' + openttd_filename, openttd_archive_location)
-        stream_to_file_if_necessary(client, openttd_cdn_url + 'opengfx-releases/' + opengfx_version + '/' + opengfx_filename, opengfx_archive_location)
+        stream_to_file_if_necessary(client, openttd_cdn_url + openttd_path + openttd_filename, openttd_archive_location)
+        stream_to_file_if_necessary(client, openttd_cdn_url + opengfx_path + opengfx_filename, opengfx_archive_location)
         check_sha_256(openttd_archive_location, openttd_file_details['sha256sum'])
         check_sha_256(opengfx_archive_location, opengfx_file_details['sha256sum'])
 
@@ -207,8 +214,8 @@ def run_done(progress, task, _):
         experiments_list = list(experiments)
         with tempfile.TemporaryDirectory(prefix=f'OpenTTDLab-{run_id}-') as run_dir:
             # Extract the binaries into the run dir
-            openttd_binary_dir = os.path.join(run_dir, f'{openttd_filename}-{openttd_file_details["sha256sum"]}')
-            opengfx_binary_dir = os.path.join(run_dir, f'{opengfx_filename}-{opengfx_file_details["sha256sum"]}')
+            openttd_binary_dir = os.path.join(run_dir, f'{openttd_filename}')
+            opengfx_binary_dir = os.path.join(run_dir, f'{opengfx_filename}')
             Path(openttd_binary_dir).mkdir(parents=True, exist_ok=True)
             Path(opengfx_binary_dir).mkdir(parents=True, exist_ok=True)
             extractors[openttd_extension](openttd_archive_location, openttd_binary_dir)

test_openttdlab.py

@@ -111,6 +111,37 @@ def test_run_experiments_local_ai_early_version_of_openttd():
     }
 
 
+def test_run_experiments_local_ai_early_nightly_of_openttd():
+    results = run_experiments(
+        openttd_version='20230323-master-g83eb73a9b2',
+        opengfx_version='20230522-master-g4220c498b2',
+        experiments=(
+            {
+                'seed': seed,
+                'ais': (
+                    local_file('./fixtures/54524149-trAIns-2.1.tar', 'trAIns'),
+                ),
+                'days': 365 * 5 + 1,
+            }
+            for seed in range(2, 4)
+        ),
+        result_processor=_basic_data,
+    )
+
+    assert len(results) == 118
+    assert results[117] == {
+        'openttd_version': '20230323-master-g83eb73a9b2',
+        'opengfx_version': '20230522-master-g4220c498b2',
+        'seed': 3,
+        'name': 'trAIns AI',
+        'date': date(1954, 12, 1),
+        'current_loan': 300000,
+        'money': 1182246,
+        'terrain_type': 1,
+        'error': False,
+    }
+
+
 def test_run_experiments_local_folder_from_tar():
 
     with tempfile.TemporaryDirectory() as d: