From f9e0056460c677fe988a421d90c08c6bb044b134 Mon Sep 17 00:00:00 2001 From: Ebe66 <88671974+Ebe66@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:21:34 +0200 Subject: [PATCH 1/2] Basic Squadrats integration Adds your Squadrats progress to the generated maps. Until I see some activity on the outstanding PR's I won't be adding any documentation to this as it seems the project is pretty much dead atm. Minimal instructions: You need to have gpsbabel installed (see the Wandrer PR) From the Squadrats website map view download the kml file and put it in the maps folder Start whooMapsCreator via the gui and select the integrate squadrats option from the advanced tab, or use the -ds cli parameter . --- device_themes/vtm_theme_poi/icons/hatch-b.svg | 14 ++ device_themes/vtm_theme_poi/icons/hatch-r.svg | 14 ++ .../vtm_theme_poi/icons/stripesdown-b.svg | 9 + .../vtm_theme_poi/icons/stripesup-r.svg | 9 + device_themes/vtm_theme_poi/vtm-elemnt.xml | 19 ++ wahoomc/input.py | 7 + wahoomc/main.py | 7 +- wahoomc/osm_maps_functions.py | 167 +++++++++++++++++- .../tag_wahoo_adjusted/tag-wahoo-poi.xml | 5 + wahoomc/resources/tags-to-keep.json | 1 + wahoomc/resources/tunnel-transform.xml | 12 ++ 11 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 device_themes/vtm_theme_poi/icons/hatch-b.svg create mode 100644 device_themes/vtm_theme_poi/icons/hatch-r.svg create mode 100644 device_themes/vtm_theme_poi/icons/stripesdown-b.svg create mode 100644 device_themes/vtm_theme_poi/icons/stripesup-r.svg diff --git a/device_themes/vtm_theme_poi/icons/hatch-b.svg b/device_themes/vtm_theme_poi/icons/hatch-b.svg new file mode 100644 index 00000000..c198711f --- /dev/null +++ b/device_themes/vtm_theme_poi/icons/hatch-b.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/device_themes/vtm_theme_poi/icons/hatch-r.svg b/device_themes/vtm_theme_poi/icons/hatch-r.svg new file mode 100644 index 00000000..f266ba85 --- /dev/null +++ b/device_themes/vtm_theme_poi/icons/hatch-r.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/device_themes/vtm_theme_poi/icons/stripesdown-b.svg b/device_themes/vtm_theme_poi/icons/stripesdown-b.svg new file mode 100644 index 00000000..72b61965 --- /dev/null +++ b/device_themes/vtm_theme_poi/icons/stripesdown-b.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/device_themes/vtm_theme_poi/icons/stripesup-r.svg b/device_themes/vtm_theme_poi/icons/stripesup-r.svg new file mode 100644 index 00000000..4315b864 --- /dev/null +++ b/device_themes/vtm_theme_poi/icons/stripesup-r.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/device_themes/vtm_theme_poi/vtm-elemnt.xml b/device_themes/vtm_theme_poi/vtm-elemnt.xml index 2a6661c9..7a05cfca 100644 --- a/device_themes/vtm_theme_poi/vtm-elemnt.xml +++ b/device_themes/vtm_theme_poi/vtm-elemnt.xml @@ -172,6 +172,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/wahoomc/input.py b/wahoomc/input.py index 25bc55bc..345884b6 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -88,6 +88,8 @@ def process_call_of_the_tool(): help="output debug logger messages") options_args.add_argument('-hdd', '--hdd_mode', action='store_true', help="use mapwriter hdd mode") + options_args.add_argument('-ds', '--do_squadrats', action='store_true', + help="integrate squadrats data into map files") args = parser_top.parse_args() @@ -120,6 +122,7 @@ def process_call_of_the_tool(): o_input_data.verbose = args.verbose o_input_data.hdd_mode = args.hdd_mode + o_input_data.integrate_squadrats = args.do_squadrats return o_input_data @@ -198,6 +201,7 @@ def __init__(self): self.zip_folder = False self.save_cruiser = False self.hdd_mode = False + self.integrate_squadrats = False self.verbose = False @@ -316,6 +320,7 @@ def handle_create_map(self, event): # pylint: disable=unused-argument self.o_input_data.zip_folder = tab2.first.checkb_zip_folder_val.get() self.o_input_data.verbose = tab2.first.checkb_verbose_val.get() self.o_input_data.hdd_mode = tab2.first.checkb_mapwriter_ram_hdd_val.get() + self.o_input_data.integrate_squadrats = tab2.first.checkb_integrate_squadrats_val.get() # get text without \n in the end self.o_input_data.tag_wahoo_xml = tab2.second.input_tag_wahoo_xml.get() @@ -477,3 +482,5 @@ def __init__(self, parent, oInputData): "output debug logger messages", 4) self.checkb_mapwriter_ram_hdd_val = create_checkbox(self, oInputData.verbose, "Use mapwriter HDD mode", 5) + self.checkb_integrate_squadrats_val = create_checkbox(self, oInputData.verbose, + "Integrate Squadrats files", 6) \ No newline at end of file diff --git a/wahoomc/main.py b/wahoomc/main.py index dfa17b11..4b41e373 100644 --- a/wahoomc/main.py +++ b/wahoomc/main.py @@ -74,6 +74,9 @@ def run(run_level): o_osm_maps = OsmMaps(o_osm_data) + if o_input_data.integrate_squadrats: + o_osm_maps.prepare_squadrats_data() + # Filter tags from country osm.pbf files' o_osm_maps.filter_tags_from_country_osm_pbf_files() @@ -88,11 +91,11 @@ def run(run_level): o_osm_maps.generate_elevation(o_input_data.use_srtm1) # Split filtered country files to tiles - o_osm_maps.split_filtered_country_files_to_tiles() + o_osm_maps.split_filtered_country_files_to_tiles(o_input_data.integrate_squadrats) # Merge splitted tiles with land and sea o_osm_maps.merge_splitted_tiles_with_land_and_sea( - o_input_data.process_border_countries, o_input_data.contour) + o_input_data.process_border_countries, o_input_data.contour,o_input_data.integrate_squadrats) # Creating .map files o_osm_maps.create_map_files(o_input_data.save_cruiser, diff --git a/wahoomc/osm_maps_functions.py b/wahoomc/osm_maps_functions.py index 94128805..d8310296 100644 --- a/wahoomc/osm_maps_functions.py +++ b/wahoomc/osm_maps_functions.py @@ -28,6 +28,7 @@ from wahoomc.constants import VERSION from wahoomc.constants import OSMOSIS_WIN_FILE_PATH from wahoomc.constants import USER_DL_DIR +from wahoomc.constants import USER_MAPS_DIR from wahoomc.timings import Timings @@ -84,6 +85,122 @@ def __init__(self, o_osm_data): create_empty_directories( USER_OUTPUT_DIR, self.o_osm_data.tiles, self.o_osm_data.border_countries) + def prepare_squadrats_data(self): # pylint: disable=too-many-statements,too-many-branches + """_ + Find Squadrats KML and convert + """ + + log.info('-' * 80) + log.info('# Convert Squatrats files to osm.pbf files') + timings = Timings() + + if platform.system() == "Windows": + # Check for new Squadrat kml file in the maps directory. Format must be squadrats*.km + squadratskml_files = glob.glob(f'{USER_MAPS_DIR}/squadrats*.kml') + if squadratskml_files: + for file in squadratskml_files: + log.info('+ Converting Squadrats KML file %s to OSM.', file) + # Call gpsbabel to convert to osm example gpsbabel -w -r -t -i kml -f file-in -o osm,tag=wandrer:untraveled,tagnd=wandrer:untraveled -F file-out + cmd = ['gpsbabel', '-w', '-r', '-t', '-i', 'kml', '-f', file, '-o', + 'osm,tag=squadrats:complete,tagnd=squadrats:complete', '-F', file.replace(".kml", ".osm")] + #subprocess.run(cmd, check=True, cwd=USER_MAPS_DIR) + run_subprocess_and_log_output( + cmd, f'! Error converting KML file: {file} to OSM. Is GPSBabel installed and in the path?', cwd=USER_MAPS_DIR) + + squadratsosm_files = glob.glob(f'{USER_MAPS_DIR}/squadrats*.osm') + if squadratsosm_files: + start_way_id = 30000000000 + for file in squadratsosm_files: + log.info('+ Replacing negative ID\'s with Large positive ones in %s and converting to .osm.pbf.', file) + start_way_id_str = f"{start_way_id}" + with open(file, encoding='utf8') as fhandle: + osm_data = fhandle.read() + fhandle.close() + + osm_data = osm_data.replace("\"-", "\"" + start_way_id_str) + + with open(file, 'w', encoding='utf8') as ofhandle: + ofhandle.write(osm_data) + ofhandle.close() + + # Increase way counter for the next file. No duplicate way numbers allowed! + start_way_id = start_way_id + 10000000 + + # Convert to osm.pbf + cmd = [self.osmconvert_path] + cmd.extend(['-v', '--hash-memory=2500', '--complete-ways', '--complete-multipolygons', + '--complete-boundaries', '--drop-author', '--drop-version']) + cmd.append(file) + cmd.append('-o='+file.replace(".osm", ".osm.pbf")) + run_subprocess_and_log_output( + cmd, f'! Error converting OSM file: {file} to OSM.PBF.') + + try: + log.info('+ Removing intermediate files and renaming processed input files') + for file in squadratskml_files: + oldbasename = os.path.basename(file) + os.rename(file, file.replace( + oldbasename, "Processed-"+oldbasename)) + + for file in squadratsosm_files: + os.remove(file) + except: # pylint: disable=bare-except + pass + + # Non-Windows + else: + # Check for new Squadrats kml files in the maps directory. Format must be squadrats*.kmz + squadratskml_files = glob.glob(f'{USER_MAPS_DIR}/squadrats*.kml') + if squadratskml_files: + for file in squadratskml_files: + log.info('+ Converting Squadrats KML file %s to OSM.', file) + # Call gpsbabel to convert to osm example gpsbabel -w -r -t -i kml -f file-in -o osm,tag=wandrer:untraveled,tagnd=wandrer:untraveled -F file-out + cmd = ['gpsbabel', '-w', '-r', '-t', '-i', 'kml', '-f', file, '-o', + 'osm,tag=squadrats:complete,tagnd=squadrats:complete', '-F', file.replace(".kml", ".osm")] + #subprocess.run(cmd, check=True, cwd=USER_MAPS_DIR) + run_subprocess_and_log_output( + cmd, f'! Error converting KML file: {file} to OSM. Is GPSBabel installed?', cwd=USER_MAPS_DIR) + + squadratsosm_files = glob.glob(f'{USER_MAPS_DIR}/squadrats*.osm') + if squadratsosm_files: + start_way_id = 30000000000 + for file in squadratsosm_files: + log.info('+ Replacing negative ID\'s with Large positive ones in %s and converting to .osm.pbf.', file) + start_way_id_str = f"{start_way_id}" + with open(file, encoding='utf8') as fhandle: + osm_data = fhandle.read() + fhandle.close() + + osm_data = osm_data.replace("\"-", "\"" + start_way_id_str) + + with open(file, 'w', encoding='utf8') as ofhandle: + ofhandle.write(osm_data) + ofhandle.close() + + # Increase way counter for the next file. No duplicate way numbers allowed! + start_way_id = start_way_id + 10000000 + + # Convert to osm.pbf + cmd = ['osmium', 'cat'] + cmd.append(file) + cmd.append('-o'+file.replace(".osm", ".osm.pbf")) + run_subprocess_and_log_output( + cmd, f'! Error converting OSM file: {file} to OSM.PBF.') + + try: + log.info('+ Removing intermediate files and renaming processed input files') + for file in squadratskml_files: + oldbasename = os.path.basename(file) + os.rename(file, file.replace( + oldbasename, "Processed-"+oldbasename)) + + for file in squadratsosm_files: + os.remove(file) + except: # pylint: disable=bare-except + pass + + log.info('+ End Convert Squadrats file(s) to osm.pbf files: OK, %s', timings.stop_and_return()) + def filter_tags_from_country_osm_pbf_files(self): # pylint: disable=too-many-statements """ Filter tags from country osm.pbf files @@ -362,7 +479,7 @@ def generate_elevation(self, use_srtm1): log.info('+ Generate contour lines for each coordinate: OK, %s', timings.stop_and_return()) - def split_filtered_country_files_to_tiles(self): + def split_filtered_country_files_to_tiles(self,process_squadrats): # pylint: disable=too-many-statements """ Split filtered country files to tiles """ @@ -371,7 +488,7 @@ def split_filtered_country_files_to_tiles(self): log.info('# Split filtered country files to tiles') timings = Timings() tile_count = 1 - for tile in self.o_osm_data.tiles: + for tile in self.o_osm_data.tiles: # pylint: disable=too-many-nested-blocks for country, val in self.o_osm_data.border_countries.items(): if country not in tile['countries']: @@ -410,6 +527,24 @@ def split_filtered_country_files_to_tiles(self): run_subprocess_and_log_output( cmd, '! Error in osmconvert with country: {country}. Win/out_file_names') + if process_squadrats: + in_squadrats_files = [] + in_squadrats_files = glob.glob(os.path.join(USER_MAPS_DIR, 'squadrats*.osm.pbf')) + for squadrats_map in in_squadrats_files: + out_squadrats = os.path.join( + USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', f'split-{os.path.basename(squadrats_map)}') + if not os.path.isfile(out_squadrats): + cmd = [self.osmconvert_path, '--hash-memory=2500'] + cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' + + ',' + f'{tile["right"]}' + ',' + f'{tile["top"]}') + cmd.extend( + ['--complete-ways', '--complete-multipolygons', '--complete-boundaries']) + cmd.append(squadrats_map) + cmd.append('-o='+out_squadrats) + #print(cmd) + run_subprocess_and_log_output( + cmd, '! Error in osmconvert while processing Squadrats file: {squadrats_map}.') + # Non-Windows else: cmd = ['osmium', 'extract'] @@ -434,13 +569,28 @@ def split_filtered_country_files_to_tiles(self): run_subprocess_and_log_output( cmd, '! Error in Osmium with country: {country}. macOS/out_file_names') + if process_squadrats: + in_squadrats_files = [] + in_squadrats_files = glob.glob(os.path.join(USER_MAPS_DIR, 'squadrats*.osm.pbf')) + for squadrats_map in in_squadrats_files: + out_squadrats = os.path.join( + USER_OUTPUT_DIR, f'{tile["x"]}', f'{tile["y"]}', f'split-{os.path.basename(squadrats_map)}') + if not os.path.isfile(out_squadrats): + cmd = ['osmium', 'extract'] + cmd.extend( + ['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}']) + cmd.append(squadrats_map) + cmd.append('-o'+out_squadrats) + run_subprocess_and_log_output( + cmd, '! Error in osmconvert while processing Squadrats file: {squadrats_map}.') + self.log_tile_debug(tile["x"], tile["y"], tile_count, f'{country} {timings_tile.stop_and_return()}') tile_count += 1 log.info('+ Split filtered country files to tiles: OK, %s', timings.stop_and_return()) - def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, contour): # pylint: disable=too-many-locals + def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, contour, process_squadrats): # pylint: disable=too-many-locals """ Merge splitted tiles with land elevation and sea - elevation data only if requested @@ -463,6 +613,9 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto elevation_files = glob.glob( os.path.join(out_tile_dir, 'elevation*.osm')) + squadrats_files = glob.glob( + os.path.join(out_tile_dir, 'split-squadrats*.osm.pbf')) + # merge splitted tiles with land and sea every time because the result is different per constants (user input) # sort land* osm files self.sort_osm_files(tile) @@ -503,6 +656,14 @@ def merge_splitted_tiles_with_land_and_sea(self, process_border_countries, conto cmd.extend( ['--rx', 'file='+elevation, '--s', '--m']) + if process_squadrats: + for squadrats in squadrats_files: + cmd.append('--rbf') + cmd.append(os.path.join( + out_tile_dir, f'{tile["x"]}', f'{tile["y"]}', f'{squadrats}')) + cmd.append('workers=' + self.workers) + cmd.append('--merge') + cmd.extend( ['--rx', 'file='+os.path.join(out_tile_dir, 'sea.osm'), '--s', '--m']) cmd.extend(['--tag-transform', 'file=' + os.path.join(RESOURCES_DIR, diff --git a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml index 6f43434f..a859ded9 100644 --- a/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml +++ b/wahoomc/resources/tag_wahoo_adjusted/tag-wahoo-poi.xml @@ -85,6 +85,11 @@ + + + + + diff --git a/wahoomc/resources/tags-to-keep.json b/wahoomc/resources/tags-to-keep.json index b306b27c..179571a1 100644 --- a/wahoomc/resources/tags-to-keep.json +++ b/wahoomc/resources/tags-to-keep.json @@ -75,6 +75,7 @@ "station", "stop" ], + "squadrats": "", "surface": "", "tracktype": "", "tunnel": "", diff --git a/wahoomc/resources/tunnel-transform.xml b/wahoomc/resources/tunnel-transform.xml index 4faf5670..a3a976be 100644 --- a/wahoomc/resources/tunnel-transform.xml +++ b/wahoomc/resources/tunnel-transform.xml @@ -18,4 +18,16 @@ + + + Squadrats Name tag + + + + + + + + + \ No newline at end of file From 9a0b7fcbd2320d26f1b02b869ff775ea63bc788b Mon Sep 17 00:00:00 2001 From: Ebe66 <88671974+Ebe66@users.noreply.github.com> Date: Sun, 22 Jun 2025 19:26:18 +0200 Subject: [PATCH 2/2] Update input.py --- wahoomc/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wahoomc/input.py b/wahoomc/input.py index 345884b6..055d4f27 100644 --- a/wahoomc/input.py +++ b/wahoomc/input.py @@ -483,4 +483,4 @@ def __init__(self, parent, oInputData): self.checkb_mapwriter_ram_hdd_val = create_checkbox(self, oInputData.verbose, "Use mapwriter HDD mode", 5) self.checkb_integrate_squadrats_val = create_checkbox(self, oInputData.verbose, - "Integrate Squadrats files", 6) \ No newline at end of file + "Integrate Squadrats files", 6)