From aeb79380880510fba8c335b3ca3bf14603c5eb38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 19 Nov 2023 14:35:28 +0100
Subject: [PATCH 01/11] Split invoking of extract into separate method

---
 wahoomc/osm_maps_functions.py | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index d9aeefb2..a5c69011 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -172,27 +172,13 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
                         or self.last_changed_is_identical_to_last_run(key) is False:
                     log.info(
                         '+ Filtering unwanted map objects out of map of %s', key)
+                        
+                    tags_to_keep = translate_tags_to_keep(sys_platform=platform.system())
+                    self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac)
 
-                    # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html
-                    cmd = ['osmium', 'tags-filter', '--remove-tags']
-                    cmd.append(val['map_file'])
-                    cmd.extend(translate_tags_to_keep(
-                        sys_platform=platform.system()))
-                    cmd.extend(['-o', out_file_pbf_filtered_mac])
-                    cmd.append('--overwrite')
-
-                    run_subprocess_and_log_output(
-                        cmd, f'! Error in Osmium with country: {key}')
-
-                    cmd = ['osmium', 'tags-filter', '--remove-tags']
-                    cmd.append(val['map_file'])
-                    cmd.extend(translate_tags_to_keep(
-                        name_tags=True, sys_platform=platform.system()))
-                    cmd.extend(['-o', out_file_pbf_filtered_names_mac])
-                    cmd.append('--overwrite')
+                    tags_to_keep = translate_tags_to_keep(name_tags=True, sys_platform=platform.system())
+                    self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_names_mac)
 
-                    run_subprocess_and_log_output(
-                        cmd, f'! Error in Osmium with country: {key}')
 
                 val['filtered_file'] = out_file_pbf_filtered_mac
                 val['filtered_file_names'] = out_file_pbf_filtered_names_mac
@@ -202,6 +188,16 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
 
         log.info('+ Filter tags from country osm.pbf files: OK, %s', timings.stop_and_return())
 
+    def invoke_filter_tags_osmium_linux(self, country, map_file, tags_to_keep, out_filename):
+        # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html
+        cmd = ['osmium', 'tags-filter', '--remove-tags']
+        cmd.append(map_file)
+        cmd.extend(tags_to_keep)
+        cmd.extend(['-o', out_filename])
+        cmd.append('--overwrite')
+
+        run_subprocess_and_log_output(cmd, f'! Error in Osmium with country: {country}')
+
     def generate_land(self):
         """
         Generate land for all tiles

From e35776108440514a2e60c5cb9587ea4ac7844358 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 19 Nov 2023 15:09:35 +0100
Subject: [PATCH 02/11] Prototype of using async for running external programs,
 just for tag filtering on linux for now

---
 wahoomc/main.py               |  3 +-
 wahoomc/osm_maps_functions.py | 63 ++++++++++++++++++++++++++++-------
 2 files changed, 53 insertions(+), 13 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index 89f938af..fab4ac6a 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -4,6 +4,7 @@
 #!/usr/bin/python
 
 # import official python packages
+import asyncio
 import logging
 
 # import custom python packages
@@ -75,7 +76,7 @@ def run(run_level):
         o_osm_maps = OsmMaps(o_osm_data)
 
         # Filter tags from country osm.pbf files'
-        o_osm_maps.filter_tags_from_country_osm_pbf_files()
+        asyncio.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
 
         # Generate land
         o_osm_maps.generate_land()
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index a5c69011..b8e195d5 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -5,6 +5,7 @@
 
 # import official python packages
 from datetime import datetime
+import asyncio
 import glob
 import multiprocessing
 import os
@@ -59,6 +60,40 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""):
         log.debug('subprocess debug output:')
         log.debug(process.stdout)
 
+async def run_async_subprocess_and_log_output(cmd, args, error_message, cwd=""):
+    """
+    run given cmd-subprocess and issue error message if wished
+    """
+    process = await asyncio.create_subprocess_exec(
+#        create_subprocess_shell,
+        cmd,
+        *args,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE)
+
+    stdout, stderr = await process.communicate()
+
+#    if not cwd:
+#        process = subprocess.run(
+#            cmd, capture_output=True, text=True, encoding="utf-8", check=False)
+#
+#    else:
+#        process = subprocess.run(  # pylint: disable=consider-using-with
+#            cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False)
+
+
+    if error_message and process.returncode != 0:  # 0 means success
+        log.error('subprocess error output:')
+        if process.stderr:
+            log.error(process.stderr)
+
+        log.error(error_message)
+        sys.exit()
+
+    elif process.stdout:
+        log.debug('subprocess debug output:')
+        log.debug(process.stdout)
+
 
 def get_timestamp_last_changed(file_path):
     """
@@ -84,7 +119,7 @@ def __init__(self, o_osm_data):
         create_empty_directories(
             USER_OUTPUT_DIR, self.o_osm_data.tiles, self.o_osm_data.border_countries)
 
-    def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-statements
+    async def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-statements
         """
         Filter tags from country osm.pbf files
         """
@@ -174,11 +209,14 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
                         '+ Filtering unwanted map objects out of map of %s', key)
                         
                     tags_to_keep = translate_tags_to_keep(sys_platform=platform.system())
-                    self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac)
+                    tags_to_keep_names = translate_tags_to_keep(name_tags=True, sys_platform=platform.system())
 
-                    tags_to_keep = translate_tags_to_keep(name_tags=True, sys_platform=platform.system())
-                    self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_names_mac)
+                    log.debug('start run filtered')
+                    task1 = asyncio.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac))
 
+                    log.debug('start run filtered names')
+                    task2 = asyncio.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep_names, out_file_pbf_filtered_names_mac))
+                    await asyncio.gather(task1, task2)
 
                 val['filtered_file'] = out_file_pbf_filtered_mac
                 val['filtered_file_names'] = out_file_pbf_filtered_names_mac
@@ -188,15 +226,16 @@ def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-many-st
 
         log.info('+ Filter tags from country osm.pbf files: OK, %s', timings.stop_and_return())
 
-    def invoke_filter_tags_osmium_linux(self, country, map_file, tags_to_keep, out_filename):
+    async def invoke_filter_tags_osmium_linux(self, country, map_file, tags_to_keep, out_filename):
         # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html
-        cmd = ['osmium', 'tags-filter', '--remove-tags']
-        cmd.append(map_file)
-        cmd.extend(tags_to_keep)
-        cmd.extend(['-o', out_filename])
-        cmd.append('--overwrite')
-
-        run_subprocess_and_log_output(cmd, f'! Error in Osmium with country: {country}')
+        cmd = 'osmium'
+        args = ['tags-filter', '--remove-tags']
+        args.append(map_file)
+        args.extend(tags_to_keep)
+        args.extend(['-o', out_filename])
+        args.append('--overwrite')
+
+        await run_async_subprocess_and_log_output(cmd, args, f'! Error in Osmium with country: {country}')
 
     def generate_land(self):
         """

From 6b8dbdd595dcbccb61d07a6ef5fac0caef597bc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 26 Nov 2023 17:01:18 +0100
Subject: [PATCH 03/11] further hacking with asyncio, it is now working with
 semaphore

---
 wahoomc/main.py               |  11 +-
 wahoomc/osm_maps_functions.py | 298 ++++++++++++++++------------------
 2 files changed, 148 insertions(+), 161 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index fab4ac6a..b7a4c698 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -76,10 +76,15 @@ def run(run_level):
         o_osm_maps = OsmMaps(o_osm_data)
 
         # Filter tags from country osm.pbf files'
+#        with asyncio.Runner(true) as runner:
+#            runner.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
         asyncio.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
+        
 
         # Generate land
-        o_osm_maps.generate_land()
+        asyncio.run(o_osm_maps.generate_land())
+#        o_osm_maps.generate_land()
+
 
         # Generate sea
         o_osm_maps.generate_sea()
@@ -89,7 +94,9 @@ def run(run_level):
             o_osm_maps.generate_elevation(o_input_data.use_srtm1)
 
         # Split filtered country files to tiles
-        o_osm_maps.split_filtered_country_files_to_tiles()
+        asyncio.run(o_osm_maps.split_filtered_country_files_to_tiles())
+
+        exit(0)
 
         # Merge splitted tiles with land and sea
         o_osm_maps.merge_splitted_tiles_with_land_and_sea(
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index b8e195d5..8fb5df61 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -60,18 +60,19 @@ def run_subprocess_and_log_output(cmd, error_message, cwd=""):
         log.debug('subprocess debug output:')
         log.debug(process.stdout)
 
-async def run_async_subprocess_and_log_output(cmd, args, error_message, cwd=""):
+async def run_async_subprocess_and_log_output(semaphore, cmd, args, error_message, cwd=""):
     """
     run given cmd-subprocess and issue error message if wished
     """
-    process = await asyncio.create_subprocess_exec(
+    async with semaphore:
+        process = await asyncio.create_subprocess_exec(
 #        create_subprocess_shell,
-        cmd,
-        *args,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE)
+            cmd,
+            *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE)
 
-    stdout, stderr = await process.communicate()
+        stdout, stderr = await process.communicate()
 
 #    if not cwd:
 #        process = subprocess.run(
@@ -82,17 +83,17 @@ async def run_async_subprocess_and_log_output(cmd, args, error_message, cwd=""):
 #            cmd, capture_output=True, cwd=cwd, text=True, encoding="utf-8", check=False)
 
 
-    if error_message and process.returncode != 0:  # 0 means success
-        log.error('subprocess error output:')
-        if process.stderr:
-            log.error(process.stderr)
+        if error_message and process.returncode != 0:  # 0 means success
+            log.error('subprocess error output:')
+            if process.stderr:
+                log.error(process.stderr)
 
-        log.error(error_message)
-        sys.exit()
+            log.error(error_message)
+            sys.exit()
 
-    elif process.stdout:
-        log.debug('subprocess debug output:')
-        log.debug(process.stdout)
+        elif process.stdout:
+            log.debug('subprocess debug output:')
+            log.debug(process.stdout)
 
 
 def get_timestamp_last_changed(file_path):
@@ -126,107 +127,52 @@ async def filter_tags_from_country_osm_pbf_files(self):  # pylint: disable=too-m
 
         log.info('-' * 80)
         log.info('# Filter tags from country osm.pbf files')
+        tasks = set()
         timings = Timings()
+        semaphore = asyncio.Semaphore(31)
         for key, val in self.o_osm_data.border_countries.items():
             # evaluate contry directory, create if not exists
             country_dir = os.path.join(USER_OUTPUT_DIR, key)
 
             # set names for filtered files for WIN, later on add ".pbf" for macOS/Linux
-            out_file_o5m_filtered_win = os.path.join(country_dir,
-                                                     'filtered.o5m')
-            out_file_o5m_filtered_names_win = os.path.join(country_dir,
-                                                           'filtered_names.o5m')
-
-            # Windows
-            if platform.system() == "Windows":
-                out_file_o5m = os.path.join(country_dir, 'outFile.o5m')
-                # only create o5m file if not there already or force processing (no user input possible)
-                # --> speeds up processing if one only wants to test tags / POIs
-                if not os.path.isfile(out_file_o5m) or self.o_osm_data.force_processing is True \
-                        or self.last_changed_is_identical_to_last_run(key) is False:
-                    log.info('+ Converting map of %s to o5m format', key)
-                    cmd = [self.osmconvert_path]
-                    cmd.extend(['-v', '--hash-memory=2500', '--complete-ways',
-                                '--complete-multipolygons', '--complete-boundaries',
-                                '--drop-author', '--drop-version'])
-                    cmd.append(val['map_file'])
-                    cmd.append('-o='+out_file_o5m)
-
-                    run_subprocess_and_log_output(
-                        cmd, f'! Error in OSMConvert with country: {key}')
-                else:
-                    log.info('+ Map of %s already in o5m format', key)
-
-                # filter out tags:
-                # - if no filtered files exist
-                # - force processing is set (this is also when new map files were dowwnloaded)
-                # - the defined TAGS_TO_KEEP_UNIVERSAL constants have changed are changed (user input or new release)
-                if not os.path.isfile(out_file_o5m_filtered_win) or not os.path.isfile(out_file_o5m_filtered_names_win) \
-                        or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False \
-                        or self.last_changed_is_identical_to_last_run(key) is False:
-                    log.info(
-                        '+ Filtering unwanted map objects out of map of %s', key)
-                    cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)]
-                    cmd.append(out_file_o5m)
-                    cmd.append(
-                        '--keep="' + translate_tags_to_keep(sys_platform=platform.system()) + '"')
-                    cmd.append('--keep-tags="all type= layer= ' +
-                               translate_tags_to_keep(sys_platform=platform.system()) + '"')
-                    cmd.append('-o=' + out_file_o5m_filtered_win)
-
-                    run_subprocess_and_log_output(
-                        cmd, f'! Error in OSMFilter with country: {key}')
-
-                    cmd = [get_tooling_win_path('osmfilter', in_user_dir=True)]
-                    cmd.append(out_file_o5m)
-                    cmd.append(
-                        '--keep="' + translate_tags_to_keep(
-                            name_tags=True, sys_platform=platform.system()) + '"')
-                    cmd.append('--keep-tags="all type= name= layer= ' +
-                               translate_tags_to_keep(
-                                   name_tags=True, sys_platform=platform.system()) + '"')
-                    cmd.append('-o=' + out_file_o5m_filtered_names_win)
-
-                    run_subprocess_and_log_output(
-                        cmd, f'! Error in OSMFilter with country: {key}')
-
-                val['filtered_file'] = out_file_o5m_filtered_win
-                val['filtered_file_names'] = out_file_o5m_filtered_names_win
-
-            # Non-Windows
-            else:
-                out_file_pbf_filtered_mac = f'{out_file_o5m_filtered_win}.pbf'
-                out_file_pbf_filtered_names_mac = f'{out_file_o5m_filtered_names_win}.pbf'
-
-                # filter out tags:
-                # - if no filtered files exist
-                # - force processing is set (this is also when new map files were dowwnloaded)
-                # - the defined TAGS_TO_KEEP_UNIVERSAL constants have changed are changed (user input or new release)
-                if not os.path.isfile(out_file_pbf_filtered_mac) or not os.path.isfile(out_file_pbf_filtered_names_mac) \
-                        or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False \
-                        or self.last_changed_is_identical_to_last_run(key) is False:
-                    log.info(
-                        '+ Filtering unwanted map objects out of map of %s', key)
+            out_file_o5m_filtered_win = os.path.join(country_dir, 'filtered.o5m')
+            out_file_o5m_filtered_names_win = os.path.join(country_dir, 'filtered_names.o5m')
+
+            out_file_pbf_filtered_mac = f'{out_file_o5m_filtered_win}.pbf'
+            out_file_pbf_filtered_names_mac = f'{out_file_o5m_filtered_names_win}.pbf'
+
+            # filter out tags:
+            # - if no filtered files exist
+            # - force processing is set (this is also when new map files were dowwnloaded)
+            # - the defined TAGS_TO_KEEP_UNIVERSAL constants have changed are changed (user input or new release)
+            if not os.path.isfile(out_file_pbf_filtered_mac) or not os.path.isfile(out_file_pbf_filtered_names_mac) \
+                     or self.o_osm_data.force_processing is True or self.tags_are_identical_to_last_run(key) is False \
+                     or self.last_changed_is_identical_to_last_run(key) is False:
+                log.info('+ Filtering unwanted map objects out of map of %s', key)
                         
-                    tags_to_keep = translate_tags_to_keep(sys_platform=platform.system())
-                    tags_to_keep_names = translate_tags_to_keep(name_tags=True, sys_platform=platform.system())
+                tags_to_keep = translate_tags_to_keep(sys_platform=platform.system())
+                tags_to_keep_names = translate_tags_to_keep(name_tags=True, sys_platform=platform.system())
 
-                    log.debug('start run filtered')
-                    task1 = asyncio.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac))
+#                    async with asyncio.TaskGroup() as tg:                    
+                log.debug('start run filtered')
+                tasks.add(asyncio.create_task(self.invoke_filter_tags_osmium_linux(semaphore, key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac)))
+#                        tg.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep, out_file_pbf_filtered_mac))
 
-                    log.debug('start run filtered names')
-                    task2 = asyncio.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep_names, out_file_pbf_filtered_names_mac))
-                    await asyncio.gather(task1, task2)
+                log.debug('start run filtered names')
+                tasks.add(asyncio.create_task(self.invoke_filter_tags_osmium_linux(semaphore, key, val['map_file'], tags_to_keep_names, out_file_pbf_filtered_names_mac)))
+#                        tg.create_task(self.invoke_filter_tags_osmium_linux(key, val['map_file'], tags_to_keep_names, out_file_pbf_filtered_names_mac))
 
-                val['filtered_file'] = out_file_pbf_filtered_mac
-                val['filtered_file_names'] = out_file_pbf_filtered_names_mac
+            val['filtered_file'] = out_file_pbf_filtered_mac
+            val['filtered_file_names'] = out_file_pbf_filtered_names_mac
 
             # write config file for country
             self.write_country_config_file(key)
+            
+        await asyncio.gather(*tasks)
 
         log.info('+ Filter tags from country osm.pbf files: OK, %s', timings.stop_and_return())
 
-    async def invoke_filter_tags_osmium_linux(self, country, map_file, tags_to_keep, out_filename):
+    async def invoke_filter_tags_osmium_linux(self, semaphore, country, map_file, tags_to_keep, out_filename):
         # https://docs.osmcode.org/osmium/latest/osmium-tags-filter.html
         cmd = 'osmium'
         args = ['tags-filter', '--remove-tags']
@@ -235,9 +181,10 @@ async def invoke_filter_tags_osmium_linux(self, country, map_file, tags_to_keep,
         args.extend(['-o', out_filename])
         args.append('--overwrite')
 
-        await run_async_subprocess_and_log_output(cmd, args, f'! Error in Osmium with country: {country}')
+#        log.info('osmium filter tags, %s', out_filename)
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmium with country: {country}')
 
-    def generate_land(self):
+    async def generate_land(self):
         """
         Generate land for all tiles
         """
@@ -246,53 +193,87 @@ def generate_land(self):
         log.info('# Generate land for each coordinate')
         timings = Timings()
         tile_count = 1
+        semaphore = asyncio.Semaphore(60)
+        land_sea_tasks = []
+        land1_tasks = []
         for tile in self.o_osm_data.tiles:
-            land_file = os.path.join(USER_OUTPUT_DIR,
-                                     f'{tile["x"]}', f'{tile["y"]}', 'land.shp')
-            out_file_land1 = os.path.join(USER_OUTPUT_DIR,
-                                          f'{tile["x"]}', f'{tile["y"]}', 'land')
+            land_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'land.shp')
+            out_file_land1 = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'land')
             timings_tile = Timings()
 
             # create land.dbf, land.prj, land.shp, land.shx
             if not os.path.isfile(land_file) or self.o_osm_data.force_processing is True:
-                self.log_tile_info(tile["x"], tile["y"], tile_count)
-                cmd = ['ogr2ogr', '-overwrite', '-skipfailures']
+#                self.log_tile_info(tile["x"], tile["y"], tile_count)
+#                cmd = ['ogr2ogr', '-overwrite', '-skipfailures']
                 # Try to prevent getting outside of the +/-180 and +/- 90 degrees borders. Normally the +/- 0.1 are there to prevent white lines at border borders.
+                correction = 0.1
                 if tile["x"] == 255 or tile["y"] == 255 or tile["x"] == 0 or tile["y"] == 0:
-                    cmd.extend(['-spat', f'{tile["left"]:.6f}',
-                                f'{tile["bottom"]:.6f}',
-                                f'{tile["right"]:.6f}',
-                                f'{tile["top"]:.6f}'])
-                else:
-                    cmd.extend(['-spat', f'{tile["left"]-0.1:.6f}',
-                                f'{tile["bottom"]-0.1:.6f}',
-                                f'{tile["right"]+0.1:.6f}',
-                                f'{tile["top"]+0.1:.6f}'])
-                cmd.append(land_file)
-                cmd.append(LAND_POLYGONS_PATH)
-
-                run_subprocess_and_log_output(
-                    cmd, f'! Error generating land for tile: {tile["x"]},{tile["y"]}')
+                    correction = 0.0
+
+                spatLeft = f'{tile["left"]-correction:.6f}'
+                spatBottom = f'{tile["bottom"]-correction:.6f}'
+                spatRight = f'{tile["right"]+correction:.6f}'
+                spatTop = f'{tile["top"]+correction:.6f}'
+                                
+                task1 = asyncio.create_task(self.invoke_create_land_and_sea_ogr2ogr_linux(semaphore, tile["x"], tile["y"], spatLeft, spatBottom, spatRight, spatTop, land_file, LAND_POLYGONS_PATH))
+                land_sea_tasks.append(task1)
+#                await asyncio.gather(task1)
+                                
+#                cmd.append(land_file)
+#                cmd.append(LAND_POLYGONS_PATH)
+
+#                run_subprocess_and_log_output(
+#                    cmd, f'! Error generating land for tile: {tile["x"]},{tile["y"]}')
 
             # create land1.osm
             if not os.path.isfile(out_file_land1+'1.osm') or self.o_osm_data.force_processing is True:
-                # Windows
-                if platform.system() == "Windows":
-                    cmd = ['python', os.path.join(RESOURCES_DIR,
-                                                  'shape2osm.py'), '-l', out_file_land1, land_file]
-
-                # Non-Windows
-                else:
-                    cmd = ['python', os.path.join(RESOURCES_DIR,
-                                                  'shape2osm.py'), '-l', out_file_land1, land_file]
-
-                run_subprocess_and_log_output(
-                    cmd, f'! Error creating land.osm for tile: {tile["x"]},{tile["y"]}')
+#                # Windows
+#                if platform.system() == "Windows":
+#                    cmd = ['python', os.path.join(RESOURCES_DIR,
+#                                                  'shape2osm.py'), '-l', out_file_land1, land_file]
+#
+#                # Non-Windows
+#                else:
+#                    cmd = ['python', os.path.join(RESOURCES_DIR,
+#                                                  'shape2osm.py'), '-l', out_file_land1, land_file]
+
+#               run_subprocess_and_log_output(
+#                    cmd, f'! Error creating land.osm for tile: {tile["x"]},{tile["y"]}')
+                task2 = asyncio.create_task(self.invoke_create_land1_python_linux(semaphore, tile["x"], tile["y"], land_file, out_file_land1))
+                land1_tasks.append(task2)
+                    
+                    
             self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return())
             tile_count += 1
 
+        log.info('start land sea')
+    #    semaphore = asyncio.Semaphore(4)
+#        async with semaphore:       
+        await asyncio.gather(*land_sea_tasks)
+        log.info('start land1')
+#        async with semaphore:       
+        await asyncio.gather(*land1_tasks)
+
         log.info('+ Generate land for each coordinate: OK, %s', timings.stop_and_return())
 
+    async def invoke_create_land_and_sea_ogr2ogr_linux(self, semaphore, tileX, tileY, spatLeft, spatBottom, spatRight, spatTop, land_file, land_polygon_path):
+        cmd = 'ogr2ogr'
+        args = ['-overwrite', '-skipfailures']
+        args.extend(['-spat', f'{spatLeft}', f'{spatBottom}', f'{spatRight}', f'{spatTop}'])
+        args.append(land_file)
+        args.append(land_polygon_path)
+
+#        log.info('+ Generate land for each coordinate: OK, %s, %s', tileX, tileY)
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error generating land for tile: {tileX},{tileY}')
+
+    async def invoke_create_land1_python_linux(self, semaphore, tileX, tileY, land_file, out_land_file):
+        cmd = 'python'
+        args = [os.path.join(RESOURCES_DIR, 'shape2osm.py'), '-l', out_land_file, land_file]
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error creating land1.osm for tile: {tileX},{tileY}')
+
+
     def generate_sea(self):
         """
         Generate sea for all tiles
@@ -307,7 +288,7 @@ def generate_sea(self):
                                         f'{tile["x"]}', f'{tile["y"]}', 'sea.osm')
             timings_tile = Timings()
             if not os.path.isfile(out_file_sea) or self.o_osm_data.force_processing is True:
-                self.log_tile_info(tile["x"], tile["y"], tile_count)
+#                self.log_tile_info(tile["x"], tile["y"], tile_count)
                 with open(os.path.join(RESOURCES_DIR, 'sea.osm'), encoding="utf-8") as sea_file:
                     sea_data = sea_file.read()
 
@@ -349,6 +330,7 @@ def generate_elevation(self, use_srtm1):
 
         hgt_path = os.path.join(USER_DL_DIR, 'hgt')
 
+        tasks = set()
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:
@@ -397,13 +379,15 @@ def generate_elevation(self, use_srtm1):
 
         log.info('+ Generate contour lines for each coordinate: OK, %s', timings.stop_and_return())
 
-    def split_filtered_country_files_to_tiles(self):
+    async def split_filtered_country_files_to_tiles(self):
         """
         Split filtered country files to tiles
         """
 
         log.info('-' * 80)
         log.info('# Split filtered country files to tiles')
+        semaphore = asyncio.Semaphore(20)
+        tasks = set()
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:
@@ -411,7 +395,7 @@ def split_filtered_country_files_to_tiles(self):
             for country, val in self.o_osm_data.border_countries.items():
                 if country not in tile['countries']:
                     continue
-                self.log_tile_info(tile["x"], tile["y"], tile_count, country)
+#                self.log_tile_info(tile["x"], tile["y"], tile_count, country)
                 timings_tile = Timings()
                 out_file = os.path.join(USER_OUTPUT_DIR,
                                         f'{tile["x"]}', f'{tile["y"]}', f'split-{country}.osm.pbf')
@@ -447,34 +431,29 @@ def split_filtered_country_files_to_tiles(self):
 
                 # Non-Windows
                 else:
-                    cmd = ['osmium', 'extract']
-                    cmd.extend(
-                        ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}'])
-                    cmd.append(val['filtered_file'])
-                    cmd.extend(['-s', 'smart'])
-                    cmd.extend(['-o', out_file])
-                    cmd.extend(['--overwrite'])
-
-                    run_subprocess_and_log_output(
-                        cmd, '! Error in Osmium with country: {country}. macOS/out_file')
-
-                    cmd = ['osmium', 'extract']
-                    cmd.extend(
-                        ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}'])
-                    cmd.append(val['filtered_file_names'])
-                    cmd.extend(['-s', 'smart'])
-                    cmd.extend(['-o', out_file_names])
-                    cmd.extend(['--overwrite'])
-
-                    run_subprocess_and_log_output(
-                        cmd, '! Error in Osmium with country: {country}. macOS/out_file_names')
+                    tasks.add(asyncio.create_task(self.invoke_split_country_to_tile_linux(semaphore, country, tile, val['filtered_file'], out_file)))
+                    tasks.add(asyncio.create_task(self.invoke_split_country_to_tile_linux(semaphore, country, tile, val['filtered_file_names'], out_file_names)))
 
                 self.log_tile_debug(tile["x"], tile["y"], tile_count, f'{country} {timings_tile.stop_and_return()}')
 
             tile_count += 1
 
+        await asyncio.gather(*tasks)
+
         log.info('+ Split filtered country files to tiles: OK, %s', timings.stop_and_return())
 
+    async def invoke_split_country_to_tile_linux(self, semaphore, country, tile, input_file, out_file):
+        cmd = 'osmium'
+        args = ['extract']
+
+        args.extend(['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}'])
+        args.append(input_file)
+        args.extend(['-s', 'smart'])
+        args.extend(['-o', out_file])
+        args.extend(['--overwrite'])
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmium with country: {country}. {out_file}')
+
     def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, contour): # pylint: disable=too-many-locals
         """
         Merge splitted tiles with land elevation and sea
@@ -483,6 +462,7 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto
 
         log.info('-' * 80)
         log.info('# Merge splitted tiles with land, elevation, and sea')
+        tasks = set()
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:  # pylint: disable=too-many-nested-blocks

From f0213c04be3976b3a70adc285a68c3729117b9ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 13:45:58 +0100
Subject: [PATCH 04/11] Use asyncio to merge tile files

---
 wahoomc/main.py               |  6 +--
 wahoomc/osm_maps_functions.py | 89 ++++++++++++++++++-----------------
 2 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index b7a4c698..5143455f 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -96,11 +96,11 @@ def run(run_level):
         # Split filtered country files to tiles
         asyncio.run(o_osm_maps.split_filtered_country_files_to_tiles())
 
-        exit(0)
 
         # Merge splitted tiles with land and sea
-        o_osm_maps.merge_splitted_tiles_with_land_and_sea(
-            o_input_data.process_border_countries, o_input_data.contour)
+        asyncio.run(o_osm_maps.merge_splitted_tiles_with_land_and_sea(o_input_data.process_border_countries, o_input_data.contour))
+
+        exit(0)
 
         # Creating .map files
         o_osm_maps.create_map_files(o_input_data.save_cruiser,
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index 8fb5df61..21079593 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -454,7 +454,7 @@ async def invoke_split_country_to_tile_linux(self, semaphore, country, tile, inp
 
         await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmium with country: {country}. {out_file}')
 
-    def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, contour): # pylint: disable=too-many-locals
+    async def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, contour): # pylint: disable=too-many-locals
         """
         Merge splitted tiles with land elevation and sea
         - elevation data only if requested
@@ -462,11 +462,12 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto
 
         log.info('-' * 80)
         log.info('# Merge splitted tiles with land, elevation, and sea')
+        semaphore = asyncio.Semaphore(30)
         tasks = set()
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:  # pylint: disable=too-many-nested-blocks
-            self.log_tile_info(tile["x"], tile["y"], tile_count)
+ #           self.log_tile_info(tile["x"], tile["y"], tile_count)
             timings_tile = Timings()
 
             out_tile_dir = os.path.join(USER_OUTPUT_DIR,
@@ -482,54 +483,54 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto
             # sort land* osm files
             self.sort_osm_files(tile)
 
-            # Windows
-            if platform.system() == "Windows":
-                cmd = [OSMOSIS_WIN_FILE_PATH]
-            # Non-Windows
-            else:
-                cmd = ['osmosis']
+            tasks.add(asyncio.create_task(self.invoke_merge_tile_linux(semaphore, process_border_countries, contour, tile, land_files, elevation_files, out_tile_dir, out_file_merged)))
 
-            loop = 0
-            # loop through all countries of tile, if border-countries should be processed.
-            # if border-countries should not be processed, only process the "entered" country
-            for country in tile['countries']:
-                if process_border_countries or country in self.o_osm_data.border_countries:
-                    cmd.append('--rbf')
-                    cmd.append(os.path.join(
-                        out_tile_dir, f'split-{country}.osm.pbf'))
-                    cmd.append('workers=' + self.workers)
-                    if loop > 0:
-                        cmd.append('--merge')
-
-                    cmd.append('--rbf')
-                    cmd.append(os.path.join(
-                        out_tile_dir, f'split-{country}-names.osm.pbf'))
-                    cmd.append('workers=' + self.workers)
-                    cmd.append('--merge')
-
-                    loop += 1
-
-            for land in land_files:
-                cmd.extend(
-                    ['--rx', 'file='+land, '--s', '--m'])
+#            self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return())
+            tile_count += 1
 
-            if contour:
-                for elevation in elevation_files:
-                    cmd.extend(
-                        ['--rx', 'file='+elevation, '--s', '--m'])
+        await asyncio.gather(*tasks)
 
-            cmd.extend(
-                ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m'])
-            cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR,
-                                                                  'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true'])
+        log.info('+ Merge splitted tiles with land, elevation, and sea: OK, %s', timings.stop_and_return())
 
-            run_subprocess_and_log_output(
-                cmd, f'! Error in Osmosis with tile: {tile["x"]},{tile["y"]}')
+    async def invoke_merge_tile_linux(self, semaphore, process_border_countries, contour, tile, land_files, elevation_files, out_tile_dir, out_file_merged):
+        if platform.system() == "Windows":
+            cmd = OSMOSIS_WIN_FILE_PATH
+        # Non-Windows
+        else:
+            cmd = 'osmosis'
+
+        loop = 0
+        args = []
+        # loop through all countries of tile, if border-countries should be processed.
+        # if border-countries should not be processed, only process the "entered" country
+        for country in tile['countries']:
+            if process_border_countries or country in self.o_osm_data.border_countries:
+                args.append('--rbf')
+                args.append(os.path.join(out_tile_dir, f'split-{country}.osm.pbf'))
+                args.append('workers=' + self.workers)
+                if loop > 0:
+                    args.append('--merge')
+
+                args.append('--rbf')
+                args.append(os.path.join(out_tile_dir, f'split-{country}-names.osm.pbf'))
+                args.append('workers=' + self.workers)
+                args.append('--merge')
+
+                loop += 1
 
-            self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return())
-            tile_count += 1
+        for land in land_files:
+            args.extend(
+                ['--rx', 'file='+land, '--s', '--m'])
 
-        log.info('+ Merge splitted tiles with land, elevation, and sea: OK, %s', timings.stop_and_return())
+        if contour:
+            for elevation in elevation_files:
+                args.extend(
+                    ['--rx', 'file='+elevation, '--s', '--m'])
+
+        args.extend(['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m'])
+        args.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR,
+                                                                  'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true'])
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmium with tile: {tile["x"]},{tile["y"]}')
 
     def sort_osm_files(self, tile):
         """

From 4b87d153d57319562642d85e9399ddae6555be67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 14:18:00 +0100
Subject: [PATCH 05/11] Use asyncio for sorting tile files, but still unsure if
 we really need the sorting step

---
 wahoomc/osm_maps_functions.py | 36 ++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index 21079593..a3ae271f 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -467,6 +467,14 @@ async def merge_splitted_tiles_with_land_and_sea(self, process_border_countries,
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:  # pylint: disable=too-many-nested-blocks
+            # sort land* osm files
+            tasks.add(asyncio.create_task(self.sort_osm_files(semaphore, tile)))
+
+        await asyncio.gather(*tasks)
+        log.info('+ Sorted: OK, %s', timings.stop_and_return())
+
+        tasks = set()
+        for tile in self.o_osm_data.tiles:  # pylint: disable=too-many-nested-blocks
  #           self.log_tile_info(tile["x"], tile["y"], tile_count)
             timings_tile = Timings()
 
@@ -480,8 +488,6 @@ async def merge_splitted_tiles_with_land_and_sea(self, process_border_countries,
                 os.path.join(out_tile_dir, 'elevation*.osm'))
 
             # merge splitted tiles with land and sea every time because the result is different per constants (user input)
-            # sort land* osm files
-            self.sort_osm_files(tile)
 
             tasks.add(asyncio.create_task(self.invoke_merge_tile_linux(semaphore, process_border_countries, contour, tile, land_files, elevation_files, out_tile_dir, out_file_merged)))
 
@@ -532,7 +538,7 @@ async def invoke_merge_tile_linux(self, semaphore, process_border_countries, con
                                                                   'tunnel-transform.xml'), '--wb', out_file_merged, 'omitmetadata=true'])
         await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmium with tile: {tile["x"]},{tile["y"]}')
 
-    def sort_osm_files(self, tile):
+    async def sort_osm_files(self, semaphore, tile):
         """
         sort land*.osm files to be in this order: nodes, then ways, then relations.
         this is mandatory for osmium-merge since:
@@ -546,20 +552,24 @@ def sort_osm_files(self, tile):
         land_files = glob.glob(os.path.join(USER_OUTPUT_DIR,
                                             f'{tile["x"]}', f'{tile["y"]}', 'land*.osm'))
 
+        tasks = set()
         for land in land_files:
-            if platform.system() == "Windows":
-                cmd = [OSMOSIS_WIN_FILE_PATH]
-            else:
-                cmd = ['osmosis']
+            tasks.add(asyncio.create_task(self.invoke_sort_land_files_linux(semaphore, tile, land)))
 
-            cmd.extend(['--read-xml', 'file='+land])
-            cmd.append('--sort')
-            cmd.extend(['--write-xml', 'file='+land])
+        await asyncio.gather(*tasks)
+        log.debug('+ Sorting land* osm files: OK')
 
-        run_subprocess_and_log_output(
-            cmd, f'Error in Osmosis with sorting land* osm files of tile: {tile["x"]},{tile["y"]}')
+    async def invoke_sort_land_files_linux(self, semaphore, tile, land):
+        if platform.system() == "Windows":
+            cmd = OSMOSIS_WIN_FILE_PATH
+        else:
+            cmd = 'osmosis'
 
-        log.debug('+ Sorting land* osm files: OK')
+        args = ['--read-xml', 'file='+land]
+        args.append('--sort')
+        args.extend(['--write-xml', 'file='+land])
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmosis with sorting land* osm files of tile: {tile["x"]},{tile["y"]}')
 
     def create_map_files(self, save_cruiser, tag_wahoo_xml):
         """

From 42248da131503152199cbb2c670a8f2ead416a8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 15:05:46 +0100
Subject: [PATCH 06/11] Use asyncio for creating map files and compressing them

---
 wahoomc/main.py               |   5 +-
 wahoomc/osm_maps_functions.py | 112 ++++++++++++++++++++--------------
 2 files changed, 68 insertions(+), 49 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index 5143455f..3e7fa9a6 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -100,11 +100,8 @@ def run(run_level):
         # Merge splitted tiles with land and sea
         asyncio.run(o_osm_maps.merge_splitted_tiles_with_land_and_sea(o_input_data.process_border_countries, o_input_data.contour))
 
-        exit(0)
-
         # Creating .map files
-        o_osm_maps.create_map_files(o_input_data.save_cruiser,
-                                    o_input_data.tag_wahoo_xml)
+        asyncio.run(o_osm_maps.create_map_files(o_input_data.save_cruiser, o_input_data.tag_wahoo_xml))
 
         # Zip .map.lzma files
         o_osm_maps.make_and_zip_files('.map.lzma', o_input_data.zip_folder)
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index a3ae271f..73fb6e55 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -571,7 +571,7 @@ async def invoke_sort_land_files_linux(self, semaphore, tile, land):
 
         await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in Osmosis with sorting land* osm files of tile: {tile["x"]},{tile["y"]}')
 
-    def create_map_files(self, save_cruiser, tag_wahoo_xml):
+    async def create_map_files(self, save_cruiser, tag_wahoo_xml):
         """
         Creating .map files
         """
@@ -580,64 +580,47 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml):
         log.info('# Creating .map files for tiles')
 
         # Number of threads to use in the mapwriter plug-in
-        threads = multiprocessing.cpu_count() - 1
-        if int(threads) < 1:
-            threads = 1
+#        threads = multiprocessing.cpu_count() - 1
+#        if int(threads) < 1:
+#            threads = 1
 
+        semaphore = asyncio.Semaphore(15)
+        tasks = set()
         timings = Timings()
         tile_count = 1
         for tile in self.o_osm_data.tiles:
-            self.log_tile_info(tile["x"], tile["y"], tile_count)
+#            self.log_tile_info(tile["x"], tile["y"], tile_count)
             timings_tile = Timings()
 
-            out_file_map = os.path.join(USER_OUTPUT_DIR,
-                                        f'{tile["x"]}', f'{tile["y"]}.map')
+            out_file_map = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map')
 
             # apply tag-wahoo xml every time because the result is different per .xml file (user input)
-            merged_file = os.path.join(USER_OUTPUT_DIR,
-                                       f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf')
+            merged_file = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', 'merged.osm.pbf')
 
-            # Windows
-            if platform.system() == "Windows":
-                cmd = [OSMOSIS_WIN_FILE_PATH, '--rbf', merged_file,
-                       'workers=' + self.workers, '--mw', 'file='+out_file_map]
-            # Non-Windows
-            else:
-                cmd = ['osmosis', '--rb', merged_file,
-                       '--mw', 'file='+out_file_map]
-
-            cmd.append(
-                f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}')
-            cmd.append('zoom-interval-conf=10,0,17')
-            cmd.append(f'threads={threads}')
-            # add path to tag-wahoo xml file
-            try:
-                cmd.append(
-                    f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}')
-            except TagWahooXmlNotFoundError:
-                log.error(
-                    'The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml)
-                sys.exit()
+            tasks.add(asyncio.create_task(self.invoke_create_map_file_linux(semaphore, tile, tag_wahoo_xml, merged_file, out_file_map)))
+            tile_count += 1
 
-            run_subprocess_and_log_output(
-                cmd, f'Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?')
+        await asyncio.gather(*tasks)
 
-            # Windows
-            if platform.system() == "Windows":
-                cmd = [get_tooling_win_path('lzma'), 'e', out_file_map,
-                       out_file_map+'.lzma', f'-mt{threads}', '-d27', '-fb273', '-eos']
-            # Non-Windows
-            else:
-                # force overwrite of output file and (de)compress links
-                cmd = ['lzma', out_file_map, '-f']
+        log.info('+ Created .map files for tiles: OK, %s', timings.stop_and_return())
 
-                # --keep: do not delete source file
-                if save_cruiser:
-                    cmd.append('--keep')
+        semaphore = asyncio.Semaphore(30)
+        tasks = set()
+        tile_count = 1
+        for tile in self.o_osm_data.tiles:
+            timings_tile = Timings()
 
-            run_subprocess_and_log_output(
-                cmd, f'! Error creating map files for tile: {tile["x"]},{tile["y"]}')
+            out_file_map = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map')
 
+            tasks.add(asyncio.create_task(self.invoke_compress_map_file_linux(semaphore, tile, save_cruiser, out_file_map)))
+
+        await asyncio.gather(*tasks)
+
+        log.info('+ Compressed .map files for tiles: OK, %s', timings.stop_and_return())
+
+        tile_count = 1
+        for tile in self.o_osm_data.tiles:
+            timings_tile = Timings()
             # Create "tile present" file
             with open(out_file_map + '.lzma.17', mode='wb') as tile_present_file:
                 tile_present_file.close()
@@ -647,6 +630,45 @@ def create_map_files(self, save_cruiser, tag_wahoo_xml):
 
         log.info('+ Creating .map files for tiles: OK, %s', timings.stop_and_return())
 
+    async def invoke_create_map_file_linux(self, semaphore, tile, tag_wahoo_xml, merged_file, out_file_map):
+        # Windows
+        if platform.system() == "Windows":
+            cmd = OSMOSIS_WIN_FILE_PATH
+            args = ['--rbf', merged_file, 'workers=1', '--mw', 'file='+out_file_map]
+        # Non-Windows
+        else:
+            cmd = 'osmosis'
+            args = ['--rb', merged_file, '--mw', 'file='+out_file_map]
+
+        args.append(f'bbox={tile["bottom"]:.6f},{tile["left"]:.6f},{tile["top"]:.6f},{tile["right"]:.6f}')
+        args.append('zoom-interval-conf=10,0,17')
+        args.append(f'threads=1')
+        # add path to tag-wahoo xml file
+        try:
+            args.append(f'tag-conf-file={get_tag_wahoo_xml_path(tag_wahoo_xml)}')
+        except TagWahooXmlNotFoundError:
+            log.error('The tag-wahoo xml file was not found: ˚%s˚. Does the file exist and is your input correct?', tag_wahoo_xml)
+            sys.exit()
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in creating map file via Osmosis with tile: {tile["x"]},{tile["y"]}. mapwriter plugin installed?')
+
+    async def invoke_compress_map_file_linux(self, semaphore, tile, save_cruiser, out_file_map):
+        # Windows
+        if platform.system() == "Windows":
+            cmd = get_tooling_win_path('lzma')
+            args = ['e', out_file_map, out_file_map+'.lzma', f'-mt1', '-d27', '-fb273', '-eos']
+        # Non-Windows
+        else:
+            # force overwrite of output file and (de)compress links
+            cmd = 'lzma'
+            args = [out_file_map, '-f']
+
+            # --keep: do not delete source file
+            if save_cruiser:
+                cmd.append('--keep')
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error creating map files for tile: {tile["x"]},{tile["y"]}')
+
     def make_and_zip_files(self, extension, zip_folder):
         """
         make or make and zip .map or .map.lzma files

From ea921e90e5b31032a5bff405e337add90a6c8d3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 15:18:59 +0100
Subject: [PATCH 07/11] Some minor cleanup of asyncio, and tuning of semaphores

---
 wahoomc/main.py               | 4 ----
 wahoomc/osm_maps_functions.py | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index 3e7fa9a6..52eea423 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -76,14 +76,11 @@ def run(run_level):
         o_osm_maps = OsmMaps(o_osm_data)
 
         # Filter tags from country osm.pbf files'
-#        with asyncio.Runner(true) as runner:
-#            runner.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
         asyncio.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
         
 
         # Generate land
         asyncio.run(o_osm_maps.generate_land())
-#        o_osm_maps.generate_land()
 
 
         # Generate sea
@@ -96,7 +93,6 @@ def run(run_level):
         # Split filtered country files to tiles
         asyncio.run(o_osm_maps.split_filtered_country_files_to_tiles())
 
-
         # Merge splitted tiles with land and sea
         asyncio.run(o_osm_maps.merge_splitted_tiles_with_land_and_sea(o_input_data.process_border_countries, o_input_data.contour))
 
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index 73fb6e55..e8764622 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -386,7 +386,7 @@ async def split_filtered_country_files_to_tiles(self):
 
         log.info('-' * 80)
         log.info('# Split filtered country files to tiles')
-        semaphore = asyncio.Semaphore(20)
+        semaphore = asyncio.Semaphore(6)
         tasks = set()
         timings = Timings()
         tile_count = 1

From a95b96ec283ffe08f3c803de6d10e0bbf3c4c26a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 15:26:04 +0100
Subject: [PATCH 08/11] The create map file step is quite memory consuming,
 tune down semaphores

---
 wahoomc/osm_maps_functions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index e8764622..68042b38 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -584,7 +584,7 @@ async def create_map_files(self, save_cruiser, tag_wahoo_xml):
 #        if int(threads) < 1:
 #            threads = 1
 
-        semaphore = asyncio.Semaphore(15)
+        semaphore = asyncio.Semaphore(12)
         tasks = set()
         timings = Timings()
         tile_count = 1

From 5adb15671263a70fb3219dca1c50950a8ef51200 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sun, 3 Dec 2023 15:37:30 +0100
Subject: [PATCH 09/11] Add missing setting of tile filename

---
 wahoomc/osm_maps_functions.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index 68042b38..a3b07e76 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -621,6 +621,9 @@ async def create_map_files(self, save_cruiser, tag_wahoo_xml):
         tile_count = 1
         for tile in self.o_osm_data.tiles:
             timings_tile = Timings()
+
+            out_file_map = os.path.join(USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}.map')
+
             # Create "tile present" file
             with open(out_file_map + '.lzma.17', mode='wb') as tile_present_file:
                 tile_present_file.close()

From 8e236579010558f8f4fe80cb15a8a18c519b7424 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sat, 9 Dec 2023 15:56:13 +0100
Subject: [PATCH 10/11] Use asyncio also for invoking contour generation, use
 pyhgtmap for elevation generation

Temporarily remove srg.. support, since it involves fetching files using login cookie
and that seems to run into trouble. Got trouble even with semaphore(1), so do not think
it is related to running many instances in parallel, need to check how the pyhgtmap
is handling the download.
The pyhgtmap should possibly be changed to rather use M2M API with token, instead of
simulating logging in as a web client.
---
 wahoomc/main.py               |  4 +---
 wahoomc/osm_maps_functions.py | 38 ++++++++++++++++++-----------------
 wahoomc/setup_functions.py    |  4 ++--
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/wahoomc/main.py b/wahoomc/main.py
index 52eea423..4d9a811b 100644
--- a/wahoomc/main.py
+++ b/wahoomc/main.py
@@ -78,17 +78,15 @@ def run(run_level):
         # Filter tags from country osm.pbf files'
         asyncio.run(o_osm_maps.filter_tags_from_country_osm_pbf_files())
         
-
         # Generate land
         asyncio.run(o_osm_maps.generate_land())
 
-
         # Generate sea
         o_osm_maps.generate_sea()
 
         # Generate elevation
         if o_input_data.contour:
-            o_osm_maps.generate_elevation(o_input_data.use_srtm1)
+            asyncio.run(o_osm_maps.generate_elevation(o_input_data.use_srtm1))
 
         # Split filtered country files to tiles
         asyncio.run(o_osm_maps.split_filtered_country_files_to_tiles())
diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index a3b07e76..511f0a37 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -319,7 +319,7 @@ def generate_sea(self):
 
         log.info('+ Generate sea for each coordinate: OK, %s', timings.stop_and_return())
 
-    def generate_elevation(self, use_srtm1):
+    async def generate_elevation(self, use_srtm1):
         """
         Generate contour lines for all tiles
         """
@@ -330,6 +330,7 @@ def generate_elevation(self, use_srtm1):
 
         hgt_path = os.path.join(USER_DL_DIR, 'hgt')
 
+        semaphore = asyncio.Semaphore(28)
         tasks = set()
         timings = Timings()
         tile_count = 1
@@ -350,35 +351,36 @@ def generate_elevation(self, use_srtm1):
                 # 2) set source
                 elevation_source = '--source=srtm1,view1,view3,srtm3'
             else:
-                # 1) search vor view1 elevation files
+                # 1) search for view1 elevation files
                 out_file_elevation_existing = glob.glob(os.path.join(
                     USER_OUTPUT_DIR, str(tile["x"]), str(tile["y"]), 'elevation*view1*.osm'))
                 # 2) set source
-                elevation_source = '--source=view1,view3,srtm3'
+                elevation_source = '--source=view1,view3'
 
             # check for already existing elevation .osm file (the ones matched via glob)
             if not (len(out_file_elevation_existing) == 1 and os.path.isfile(out_file_elevation_existing[0])) \
                     or self.o_osm_data.force_processing is True:
-                self.log_tile_info(tile["x"], tile["y"], tile_count)
-                timings_tile = Timings()
-                cmd = ['phyghtmap']
-                cmd.append('-a ' + f'{tile["left"]}' + ':' + f'{tile["bottom"]}' +
-                           ':' + f'{tile["right"]}' + ':' + f'{tile["top"]}')
-                cmd.extend(['-o', f'{out_file_elevation}', '-s 10', '-c 100,50', elevation_source,
-                            '--jobs=8', '--viewfinder-mask=1', '--start-node-id=20000000000',
-                            '--max-nodes-per-tile=0', '--start-way-id=2000000000', '--write-timestamp',
-                            '--no-zero-contour', '--hgtdir=' + hgt_path])
-                cmd.append('--earthexplorer-user=' + username)
-                cmd.append('--earthexplorer-password=' + password)
-
-                run_subprocess_and_log_output(
-                    cmd, f'! Error in phyghtmap with tile: {tile["x"]},{tile["y"]}. Win_macOS/elevation')
-                self.log_tile_debug(tile["x"], tile["y"], tile_count, timings_tile.stop_and_return())
+#                self.log_tile_info(tile["x"], tile["y"], tile_count)
+                tasks.add(asyncio.create_task(self.invoke_generate_elevation_for_tile(semaphore, tile, elevation_source, hgt_path, username, password, out_file_elevation)))
 
             tile_count += 1
 
+        await asyncio.gather(*tasks)
+
         log.info('+ Generate contour lines for each coordinate: OK, %s', timings.stop_and_return())
 
+    async def invoke_generate_elevation_for_tile(self, semaphore, tile, elevation_source, hgt_path, username, password, out_file_elevation):
+        cmd = 'pyhgtmap'
+        args = ['-a ' + f'{tile["left"]}' + ':' + f'{tile["bottom"]}' + ':' + f'{tile["right"]}' + ':' + f'{tile["top"]}']
+        args.extend(['-o', f'{out_file_elevation}', '-s 10', '-c 100,50', elevation_source,
+                            '--jobs=1', '--viewfinder-mask=1', '--start-node-id=20000000000',
+                            '--max-nodes-per-tile=0', '--start-way-id=2000000000', '--write-timestamp',
+                            '--no-zero-contour', '--hgtdir=' + hgt_path])
+        args.append('--earthexplorer-user=' + username)
+        args.append('--earthexplorer-password=' + password)
+
+        await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error in phyghtmap with tile: {tile["x"]},{tile["y"]}')
+
     async def split_filtered_country_files_to_tiles(self):
         """
         Split filtered country files to tiles
diff --git a/wahoomc/setup_functions.py b/wahoomc/setup_functions.py
index 604359a4..da547b96 100644
--- a/wahoomc/setup_functions.py
+++ b/wahoomc/setup_functions.py
@@ -134,9 +134,9 @@ def check_installation_of_programs_credentials_for_contour_lines():
                     \nPlease refer to the Quickstart Guide of wahooMapsCreator for instructions:\n- https://github.com/treee111/wahooMapsCreator/blob/develop/docs/QUICKSTART_ANACONDA.md#additional-programs-for-generating-contour-lines \
                     \nor create an issue:\n- https://github.com/treee111/wahooMapsCreator/issues"
 
-    if not is_program_installed("phyghtmap"):
+    if not is_program_installed("pyhgtmap"):
         sys.exit(
-            f"phyghtmap is not installed. {text_to_docu}")
+            f"pyhgtmap is not installed. {text_to_docu}")
 
     username, password = read_earthexplorer_credentials()
 

From d205e4030a97743b0cbfd164202f5293e47de113 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf=20H=C3=B8gemark?= <alf@i100.no>
Date: Sat, 9 Dec 2023 16:11:04 +0100
Subject: [PATCH 11/11] Fix building command arguments for compressing

---
 wahoomc/osm_maps_functions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py
index 511f0a37..d54121cd 100644
--- a/wahoomc/osm_maps_functions.py
+++ b/wahoomc/osm_maps_functions.py
@@ -670,7 +670,7 @@ async def invoke_compress_map_file_linux(self, semaphore, tile, save_cruiser, ou
 
             # --keep: do not delete source file
             if save_cruiser:
-                cmd.append('--keep')
+                args.append('--keep')
 
         await run_async_subprocess_and_log_output(semaphore, cmd, args, f'! Error creating map files for tile: {tile["x"]},{tile["y"]}')