diff --git a/.gitignore b/.gitignore index c35fba4..7d15226 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +CONTEXT_SESSION.md +__pycache__/ +*.png *.pbf *.geojson *.bin @@ -12,3 +15,10 @@ competitive_analysis_vector_tiles.md rsync_copy.sh MAP/ VECTMAP/ +migracion.md +venv/ +CHANGELOG_FGB_MIGRATION.md +fgb_output/ +rsync_copy.sh +FGBMAP/ +NAVMAP/ diff --git a/README.md b/README.md index 1e2687e..11d3578 100644 --- a/README.md +++ b/README.md @@ -1,441 +1,224 @@ -# OSM Vector Tile Generator +# OSM Tile Generator for IceNav -Highly optimized Python script for generating vector map tiles from OpenStreetMap (OSM) data using a custom binary format. +Converts OpenStreetMap PBF files to NAV tiles for [IceNav](https://github.com/jgauchia/IceNav-v3) ESP32-based GPS navigator. -The generated tiles are extremely compact and optimized for fast rendering in custom map applications, featuring advanced compression techniques, dynamic color palette optimization, and intelligent line width handling. +## Features ---- - -## What Does the Script Do? - -- **GOL format support** using gol CLI for efficient OSM data processing -- **Streaming GeoJSON processing** with incremental JSON parsing via ijson -- **Dynamic resource allocation** based on available system memory -- **Generates compact binary tiles** with efficient coordinate encoding -- **Global color palette optimization** with compact indices -- **Hybrid line width handling**: OSM tags + CartoDB-style zoom-based defaults -- **Geometry smoothing** at high zoom levels (≥16) for smooth curves -- **Geometry clipping** to tile boundaries using Cohen-Sutherland and Sutherland-Hodgman algorithms -- **Layer-based priority system** for correct rendering order -- **Adaptive batch sizing** based on zoom level and available memory -- **Real-time progress tracking** with performance metrics - ---- - -## Drawing Command Set - -The script implements these drawing commands for tile generation: - -### Geometry Commands -| Command | Code | Purpose | Data Format | -|---------|------|---------|-------------| -| `POLYLINE` | 2 (0x02) | Multi-point line with width | varint(width) + varint(num_points) + coordinates | -| `STROKE_POLYGON` | 3 (0x03) | Polygon outline with width | varint(width) + varint(num_points) + coordinates | - -### State Management Commands -| Command | Code | Purpose | Data Format | -|---------|------|---------|-------------| -| `SET_COLOR` | 128 (0x80) | Set RGB332 color directly | uint8(rgb332) | -| `SET_COLOR_INDEX` | 129 (0x81) | Set color from palette | varint(palette_index) | - -**Note**: All commands are defined in the `DRAW_COMMANDS` dictionary in the script. Only these 4 commands are actively used in tile generation. - ---- - -## Line Width System - -The script uses a sophisticated hybrid approach for determining line widths: +- **Direct PBF processing** - No intermediate formats (GOL, Docker, etc.) +- **Tile-based structure** - Standard z/x/y tile layout (like PNG/OSM tiles) +- **No clipping artifacts** - Features stored complete (no visible seams at tile edges) +- **Feature filtering** - Configurable via `features.json` +- **ESP32 optimized** - Small tiles, efficient for SD card access +- **Progress bar** - Visual progress per zoom level during generation -### 1. Physical Width from OSM Tags -When available, the script uses physical width tags from OSM data: -- `width`: Explicit width measurement -- `maxwidth`: Maximum width -- `est_width`: Estimated width -- `diameter`: For circular features -- `gauge`: For railway tracks +## Requirements -Physical widths are converted from various units (meters, feet, inches) to pixels based on zoom level and latitude, then clamped to reasonable min/max constraints. +- Python 3.8+ +- Virtual environment (recommended) -### 2. CartoDB-Style Zoom Defaults -When no physical tags exist, the script applies zoom-based styling optimized for small screens: +## Installation -**Major Roads** (visible but not overwhelming at low zoom): -- Motorway: 1px@z6 → 16px@z18 -- Trunk: 1px@z6 → 14px@z18 -- Primary: 1px@z8 → 14px@z18 - -**Connecting Roads** (thin until zoomed in): -- Secondary: 1px@z11 → 12px@z18 -- Tertiary: 1px@z12 → 10px@z18 - -**Minor Roads** (hairline until very close): -- Residential: 0.5px@z13 → 8px@z18 -- Service: 1px@z15 → 6px@z18 - -**Waterways** (drastically reduced): -- River: 1px@z8 → 10px@z18 -- Stream: 1px@z14 → 3px@z18 - -**Transport**: -- Railway: 1px@z10 → 5px@z18 -- Runway: 1px@z10 → 24px@z18 +```bash +# Clone repository +git clone https://github.com/jgauchia/Tile-Generator.git +cd Tile-Generator -### 3. Width Constraints -All calculated widths are clamped with strict min/max values: -- Low zoom (≤10): Maximum 3px for any feature -- Medium zoom (≤12): Maximum 4px for any feature -- High zoom: Feature-specific maximums (e.g., motorway max 18px) +# Create virtual environment +python3 -m venv venv +source venv/bin/activate -This ensures clean rendering on small screens without features blocking each other. +# Install dependencies +pip install geopandas pyogrio shapely pygame osmium +``` --- -## Geometry Processing Pipeline - -### 1. Smoothing (Zoom ≥ 16) -At high zoom levels, geometries are smoothed by interpolating additional points: -- Roundabouts get extra smoothing -- Maximum segment distance decreases with zoom -- Preserves closed rings for polygons +## Generate NAV Tiles -### 2. Clipping -Geometries are clipped to tile boundaries using industry-standard algorithms: -- **Lines**: Cohen-Sutherland line clipping -- **Polygons**: Sutherland-Hodgman polygon clipping -- Tolerance increased to 1e-5 for better continuity across tile edges - -### 3. Coordinate Conversion -Geographic coordinates (lat/lon) are converted to tile pixel coordinates: -- Float pixels (0-256) scaled to uint16 (0-65536) for precision -- Delta encoding for compact storage -- Zigzag encoding for signed deltas - ---- +```bash +source venv/bin/activate +python pbf_to_nav.py features.json [--zoom 6-17] +``` -## Performance Optimizations +**Arguments:** -### Memory Management -- **Adaptive batch sizes**: Automatically adjusted based on zoom level and available memory -- **Dynamic worker allocation**: Based on CPU cores and available RAM -- **Garbage collection**: Triggered after processing batches -- **Memory monitoring**: System memory detected and utilized efficiently +| Argument | Description | Default | +|----------|-------------|---------| +| `input.pbf` | OpenStreetMap PBF file | Required | +| `output_dir` | Output directory | Required | +| `features.json` | Feature configuration | Required | +| `--zoom` | Zoom range (e.g., `6-17`, `10-14`, `12`) | `6-17` | -### Processing Speed -- **Streaming JSON parsing**: Uses ijson to avoid loading entire dataset -- **Batch processing**: Features processed in configurable batch sizes -- **Thread pool execution**: Parallel tile writing with optimal worker count -- **Progress tracking**: Real-time feedback with features/second metrics +**Example:** -### Storage Efficiency -- **Variable-length encoding**: Varint and zigzag encoding for compact coordinates -- **Global color palette**: Colors indexed once, referenced by index -- **Delta encoding**: Coordinate differences stored instead of absolutes -- **RGB332 format**: 8-bit color (3R, 3G, 2B) reduces palette storage +```bash +python pbf_to_nav.py andorra.osm.pbf ./nav_output features.json --zoom 6-17 +``` --- -## Usage +## View NAV Tiles -### Basic Usage ```bash -python tile_generator.py input.gol output_dir features.json --zoom 6-17 +python nav_viewer.py --lat --lon [--zoom ] ``` -### Arguments -| Argument | Description | Default | -|----------|-------------|---------| -| `gol_file` | Path to .gol file | Required | -| `output_dir` | Output directory for tiles | Required | -| `config_file` | Features JSON configuration | Required | -| `--zoom` | Zoom level(s) (e.g. `12` or `6-17`) | `6-17` | -| `--max-file-size` | Max tile size in KB | 128 | -| `--batch-size` | Base batch size (auto-adjusted) | 10000 | - -### Examples +**Example:** -**Process specific zoom level:** ```bash -python tile_generator.py london.gol tiles/ features.json --zoom 12 +python nav_viewer.py ./nav_output --lat 42.5063 --lon 1.5218 --zoom 14 ``` -**Process zoom range with custom batch size:** -```bash -python tile_generator.py region.gol tiles/ features.json --zoom 10-15 --batch-size 5000 -``` +**Viewer Controls:** -**Large region with smaller tile size:** -```bash -python tile_generator.py planet.gol tiles/ features.json --zoom 6-17 --max-file-size 64 -``` +| Key | Action | +|-----|--------| +| Arrow keys | Pan map | +| Mouse drag | Pan map | +| `[` / `]` or Mouse wheel | Zoom in/out | +| `B` | Toggle background (white/black) | +| `F` | Toggle polygon fill | +| `G` | Toggle tile grid | +| `Q` / `ESC` | Quit | --- -## Configuration +## NAV Binary Format Specification -### Features JSON Format -The script uses a JSON configuration file to define feature styling: +NAV is a proprietary binary format designed as a lightweight alternative to FlatGeobuf for embedded devices with limited resources. Unlike FlatGeobuf (which uses FlatBuffers serialization and R-Tree spatial indexing), NAV uses a minimal sequential structure optimized for ESP32's SD card access patterns. -```json -{ - "highway=motorway": { - "color": "#E892A2", - "priority": 12, - "zoom": 6 - }, - "highway=primary": { - "color": "#FCD6A4", - "priority": 10, - "zoom": 8 - }, - "building": { - "color": "#D9D0C9", - "priority": 14, - "zoom": 13 - }, - "waterway=river": { - "color": "#A0C8F0", - "priority": 3, - "zoom": 8 - } -} -``` +**Key differences from FlatGeobuf:** +- No FlatBuffers dependency - pure binary format +- No R-Tree index - tiles are small enough for sequential reading +- int32 scaled coordinates instead of float64 (~50% smaller) +- Minimal header overhead -**Configuration Features:** -- **Tag matching**: Supports both `key=value` and `key` only patterns -- **Color**: Hex color code (converted to RGB332 internally) -- **Priority**: Rendering order (higher = drawn later/on top) -- **Zoom**: Minimum zoom level for feature visibility -- **Automatic palette**: Colors automatically indexed into optimal palette - -### Priority System -The script uses a sophisticated priority calculation: -1. **Base priority** from configuration -2. **Layer priority** from OSM `layer` tag (×1000 multiplier) -3. **Feature type priorities**: - - Water features: 100-300 - - Land use: 200 - - Underground (tunnels): 500 - - Railways: 600 - - Roads: 700-1200 (by importance) - - Bridges: 1300 - - Buildings: 1400 - - Amenities: 1500 +**File Header (22 bytes):** ---- +| Offset | Size | Field | Description | +|--------|------|-------|-------------| +| 0 | 4 | Magic | `NAV1` (0x4E, 0x41, 0x56, 0x31) | +| 4 | 2 | Feature count | Number of features (little-endian) | +| 6 | 4 | Min Lon | Bounding box min longitude (int32 scaled) | +| 10 | 4 | Min Lat | Bounding box min latitude (int32 scaled) | +| 14 | 4 | Max Lon | Bounding box max longitude (int32 scaled) | +| 18 | 4 | Max Lat | Bounding box max latitude (int32 scaled) | -## Binary Tile Format +**Feature Record:** -### File Structure -``` -[varint: num_commands] -[command_1] -[command_2] -... -[command_n] -``` +| Size | Field | Description | +|------|-------|-------------| +| 1 | Geometry type | 1=Point, 2=LineString, 3=Polygon | +| 2 | Color | RGB565 color (little-endian) | +| 1 | Zoom/Priority | High nibble = min_zoom, low nibble = priority/7 | +| 1 | Width | Line width in pixels (1-15, from OSM `width`/`lanes` tags) | +| 2 | Coord count | Number of coordinates (little-endian) | +| 8×N | Coordinates | lon(int32) + lat(int32) pairs | +| 1 | Ring count | (Polygons only) Number of rings | +| 2×R | Ring ends | (Polygons only) End index of each ring | -### Command Structure +**Width Calculation:** -**SET_COLOR (0x80)** -``` -[0x80][rgb332_byte] -``` +Width is derived from OSM tags and converted to pixels at the tile's zoom level: +- `width=*` tag: meters converted to pixels +- `lanes=*` tag: lanes × 3.5m converted to pixels +- Default: 1 pixel if no width tag present -**SET_COLOR_INDEX (0x81)** -``` -[0x81][varint: palette_index] -``` +Formula: `pixels = width_meters / (156543 × cos(lat) / 2^zoom)` -**POLYLINE (0x02)** -``` -[0x02][varint: width][varint: num_points] -[zigzag: x1][zigzag: y1] -[zigzag: dx2][zigzag: dy2] -... -``` +**Coordinate Scaling:** -**STROKE_POLYGON (0x03)** -``` -[0x03][varint: width][varint: num_points] -[zigzag: x1][zigzag: y1] -[zigzag: dx2][zigzag: dy2] -... -``` +Coordinates are stored as int32 scaled by 10,000,000 (1e7): +- `int32_value = (int32_t)(float_coord * 10000000)` +- `float_coord = (double)int32_value / 10000000.0` -### Palette File Format -``` -palette.bin: -[uint32: num_colors] -[r1][g1][b1] // RGB888 for color 0 -[r2][g2][b2] // RGB888 for color 1 -... -``` +This provides ~1cm precision while using half the space of float64. --- ## Output Structure ``` -output_dir/ -├── palette.bin # Global color palette (RGB888) -├── 6/ # Zoom level 6 -│ └── 32/ -│ └── 21.bin # Tile (x=32, y=21, z=6) -├── 12/ # Zoom level 12 -│ ├── 2048/ -│ │ ├── 1536.bin -│ │ └── 1537.bin -│ └── 2049/ -└── 17/ # Zoom level 17 +output/ +├── 6/ +│ ├── 32/ +│ │ ├── 23.nav +│ │ └── 24.nav +│ └── 33/ +│ └── ... +├── 13/ +│ └── ... +└── 17/ └── ... ``` ---- - -## System Requirements - -### Required -- **Python 3.7+** -- **gol CLI** ([download](https://www.geodesk.com/download/)) -- **Python packages**: `ijson` - -### Recommended -- **Multi-core CPU**: Script utilizes all available cores -- **4-8GB RAM**: For processing large regions -- **SSD storage**: For better I/O performance - -### Installation -```bash -# Install Python dependencies -pip install ijson - -# Install gol CLI -# Download from: https://www.geodesk.com/download/ -``` - ---- - -## Performance Statistics +Standard z/x/y tile structure: +- First level: zoom level +- Second level: tile X coordinate +- Third level: tile Y coordinate (`.nav` file) -The script provides comprehensive processing reports: - -``` -System: 8 CPU cores, 16384MB RAM -> Using 6 workers -Resource settings: 6 workers, base batch size: 10000 -Building color palette... -Palette: 39 colors -Palette written - -Processing zoom 12 (batch size: 4000, workers: 6)... -Zoom 12: 54482 features processed, 139 tiles written -Zoom 12 completed in 2m 45.32s - -Total tiles written: 139 -Total size: 1.2MB -Average tile size: 8847 bytes -Palette file: tiles/palette.bin -Total processing time: 2m 45.32s -``` +**Note:** Features are NOT clipped to tile boundaries. Each feature is stored complete in every tile it intersects. --- -## Advanced Usage - -### Processing Large Regions -For planet-scale or large extracts, process in smaller zoom ranges: -```bash -# Low zoom levels (broader features) -python tile_generator.py planet.gol tiles/ features.json --zoom 6-10 - -# Medium zoom levels -python tile_generator.py planet.gol tiles/ features.json --zoom 11-14 +## SD Card Structure -# High zoom levels (detailed features) -python tile_generator.py planet.gol tiles/ features.json --zoom 15-17 -``` +Copy the output directory to your SD card: -### Memory-Constrained Systems -Reduce batch size for systems with limited RAM: -```bash -python tile_generator.py input.gol tiles/ features.json --batch-size 2000 ``` - -### Custom Tile Size Limits -Adjust maximum tile file size: -```bash -# Smaller tiles (64KB) for bandwidth-constrained environments -python tile_generator.py input.gol tiles/ features.json --max-file-size 64 - -# Larger tiles (512KB) for high-detail regions -python tile_generator.py input.gol tiles/ features.json --max-file-size 512 +/sdcard/NAVMAP/ +├── 6/ +│ └── 32/ +│ └── 23.nav +├── 13/ +│ └── 4123/ +│ └── 2456.nav +└── 17/ + └── ... ``` --- -## Troubleshooting +## Feature Configuration -### Common Issues +The `features.json` file defines which OSM features to include: -**gol CLI not found:** -```bash -# Download and install gol from https://www.geodesk.com/download/ -# Add to PATH or use absolute path +```json +{ + "highway=motorway": { + "zoom": 6, + "color": "#ff9999", + "priority": 60 + }, + "highway=primary": { + "zoom": 6, + "color": "#ffcc99", + "priority": 62 + }, + "building": { + "zoom": 15, + "color": "#dddddd", + "priority": 80 + } +} ``` -**Memory issues:** -- Script automatically adjusts based on available memory -- Reduce `--batch-size` for very constrained systems -- Process zoom ranges separately - -**Slow processing:** -- Verify GOL file is not corrupted -- Check system resources (CPU, RAM, disk I/O) -- Use SSD for better performance -- Script automatically optimizes worker count - -**Missing features in tiles:** -- Check `zoom` setting in features.json -- Verify feature tags match configuration -- Review gol query output for filtering issues +| Field | Description | +|-------|-------------| +| `zoom` | Minimum zoom level for feature visibility | +| `color` | Hex color (converted to RGB565 internally) | +| `priority` | Render order (lower = background, higher = foreground) | --- -## Technical Details - -### Coordinate Precision -- **Input**: Geographic coordinates (lat/lon, decimal degrees) -- **Internal**: Float pixels (0-256 per tile) -- **Storage**: Uint16 (0-65536) for sub-pixel precision -- **Delta encoding**: Reduces storage by ~60-70% +## Download PBF Files -### Color System -- **Input**: Hex colors (#RRGGBB) -- **Conversion**: RGB888 → RGB332 (8-bit) -- **Storage**: 3 bytes per palette color (RGB888) -- **Usage**: 1 byte per palette index in commands - -### Width Calculation -1. Check OSM tags for physical width -2. Convert to pixels using zoom and latitude -3. Apply feature-specific constraints (min/max) -4. Fall back to CartoDB-style defaults if no tags -5. Interpolate between zoom breakpoints - -### Clipping Algorithms -- **Cohen-Sutherland**: Fast line segment clipping with outcodes -- **Sutherland-Hodgman**: Polygon clipping against rectangular viewport -- **Tolerance**: 1e-5 for continuity across tile edges - ---- +Get OSM extracts from [Geofabrik](https://download.geofabrik.de/) ## License -This project is open source and available under the MIT License. - ---- +MIT License -## Acknowledgments +## Related Projects -- **OpenStreetMap**: For providing open map data -- **Geodesk**: For GOL format and gol CLI tool -- **ijson**: For incremental JSON parsing \ No newline at end of file +- [IceNav](https://github.com/jgauchia/IceNav-v3) - ESP32-based GPS navigator diff --git a/docs/bin_tile_format.md b/docs/bin_tile_format.md deleted file mode 100644 index 94de5d4..0000000 --- a/docs/bin_tile_format.md +++ /dev/null @@ -1,353 +0,0 @@ -# Tile Binary Format Specification - -This document describes the binary format produced by the tile generation script (`tile_generator.py`) for vector map tiles. The format is intended for efficient rendering and compact storage of map data at various zoom levels. Applications and libraries can use this specification to parse and render the resulting `.bin` files. - ---- - -## File Overview - -Each file represents a single tile for a given zoom level (`z`), x coordinate (`x`), and y coordinate (`y`). -The filename and directory structure is: -``` -{output_dir}/{z}/{x}/{y}.bin -``` - -The file contains a sequence of drawing commands encoded in a compact binary format. -All coordinates are encoded as `uint16` values (range 0–65535) relative to the tile. - -The format uses **state commands** (SET_COLOR, SET_COLOR_INDEX) to eliminate color redundancy and achieve maximum compression through a dynamic palette system. - ---- - -## File Structure - -The file is a single binary blob with the structure: - -| Field | Type | Description | -|----------------|-----------|-------------------------------------------| -| num_commands | varint | Number of drawing commands in the tile | -| commands[] | variable | Sequence of drawing commands | - -- All variable-length integers use [protobuf varint encoding](https://developers.google.com/protocol-buffers/docs/encoding#varints). -- All signed integers are encoded with [zigzag encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers). - ---- - -## Drawing Command Structure - -Commands are separated into **state commands** and **geometry commands**: - -### State Commands (Set Current Color) -| Field | Type | Description | -|--------------|-----------|---------------------------------------------| -| type | varint | State command type (0x80 or 0x81) | -| color_data | variable | Color information (see below) | - -### Geometry Commands (Use Current Color) -| Field | Type | Description | -|--------------|-----------|---------------------------------------------| -| type | varint | Drawing command type (see command types) | -| parameters | variable | Command-specific data (no color field) | - ---- - -## Command Types - -### Basic Geometry Commands -| Name | Value | Description | -|---------------------|-------|------------------------------------------------------| -| LINE | 0x01 | Single line (from x1,y1 to x2,y2) | -| POLYLINE | 0x02 | Polyline (sequence of points) | -| STROKE_POLYGON | 0x03 | Closed polygon outline (sequence of points) | - -### State Commands -| Name | Value | Description | -|---------------------|-------|------------------------------------------------------| -| SET_COLOR | 0x80 | Set current color using RGB332 direct value (fallback) | -| SET_COLOR_INDEX | 0x81 | Set current color using dynamic palette index | - ---- - -## State Command Parameters - -### SET_COLOR (type 0x80) -Sets the current color for subsequent geometry commands using direct RGB332 value. Used as fallback when color is not in the dynamic palette. - -| Field | Type | Description | -|-------------|---------|--------------------------------| -| color | uint8 | RGB332 color value (0-255) | - -### SET_COLOR_INDEX (type 0x81) -Sets the current color using a dynamic palette index. This is the primary method for color assignment. - -| Field | Type | Description | -|-------------|---------|------------------------------------------| -| color_index | varint | Index into dynamic color palette (0-N) | - ---- - -## Geometry Command Parameters - -**Important**: Geometry commands do **NOT** include color fields. The current color is set by the most recent SET_COLOR or SET_COLOR_INDEX command. - -All geometry commands now include a **width parameter** before the coordinate data. - -### LINE (type 0x01) -| Field | Type | Description | -|-------------|---------|-------------------------| -| width | varint | Line width in pixels | -| x1, y1 | int32 | Starting coordinates | -| x2, y2 | int32 | Ending coordinates | - -**Encoding**: varint(width), zigzag(x1), zigzag(y1), zigzag(x2 - x1), zigzag(y2 - y1) - -### POLYLINE (type 0x02) -| Field | Type | Description | -|-------------|---------|----------------------------------------| -| width | varint | Line width in pixels | -| num_points | varint | Number of points | -| points[] | int32 | Sequence of (x, y) | - -**Encoding**: -- varint(width) -- varint(num_points) -- zigzag(x0), zigzag(y0) – first point absolute -- zigzag(x1 - x0), zigzag(y1 - y0) – delta encoding for subsequent points - -### STROKE_POLYGON (type 0x03) -| Field | Type | Description | -|-------------|---------|----------------------------------------| -| width | varint | Line width in pixels | -| num_points | varint | Number of points | -| points[] | int32 | Sequence of (x, y) | - -**Encoding**: -- varint(width) -- varint(num_points) -- zigzag(x0), zigzag(y0) – first point absolute -- zigzag(x1 - x0), zigzag(y1 - y0) – delta encoding for subsequent points - ---- - -## Line Width Calculation - -The tile generator includes intelligent line width calculation based on OSM tags and zoom level: - -### Width from 'width' Tag (Highest Priority) -If a feature has a `width` tag (in meters), it's converted to pixels using Mercator projection: -- Considers the feature's latitude for accurate meter-to-pixel conversion -- Applies zoom-based scaling factors -- Clamps to minimum (0.5px) and maximum (20px) values - -### Default Widths by Feature Type -When no `width` tag is present, default widths are calculated based on feature type and zoom level: - -**Highways:** -- motorway, trunk: `max(2, int(4 * zoom/18.0))` -- primary, motorway_link, trunk_link: `max(1, int(3 * zoom/18.0))` -- secondary, primary_link: `max(1, int(2.5 * zoom/18.0))` -- tertiary, secondary_link: `max(1, int(2 * zoom/18.0))` -- residential, unclassified, road: `max(1, int(1.5 * zoom/18.0))` -- service, track: `1` -- Other (footway, path, etc.): `1` - -**Railways:** -- All types: `max(1, int(2 * zoom/18.0))` - -**Waterways:** -- river, canal: `max(1, int(3 * zoom/18.0))` -- stream, drain, ditch: `max(1, int(1.5 * zoom/18.0))` -- Other: `1` - -**Aeroways:** -- runway, taxiway: `max(1, int(4 * zoom/18.0))` - -**Power lines:** -- power=line: `max(1, int(1.5 * zoom/18.0))` - -**Barriers:** -- fence, wall, retaining_wall: `max(1, int(1 * zoom/18.0))` - -**Man-made:** -- pipeline, embankment: `max(1, int(2 * zoom/18.0))` - -**Natural:** -- cliff, ridge, arete: `max(1, int(2 * zoom/18.0))` - ---- - -## Color Encoding - -### RGB332 Format -- `color` is stored as a single byte (`uint8`) in [RGB332 format](https://en.wikipedia.org/wiki/List_of_monochrome_and_RGB_palettes#RGB332). -- **Bit layout**: `RRRGGGBB` (3 bits red, 3 bits green, 2 bits blue) - -### Dynamic Palette -- **Automatic generation**: Palette is built from unique colors in the configuration file -- **Index mapping**: Each hex color (e.g., `#ff0000`) receives a unique index (0-N) -- **Alphabetical ordering**: Colors are sorted alphabetically for consistency -- **Efficient encoding**: Indices use varint encoding (smaller for low indices) -- **Primary method**: Most colors use SET_COLOR_INDEX for optimal compression - -### Palette File -The generator creates a `palette.bin` file in the output directory: - -| Field | Type | Description | -|----------------|-----------|-------------------------------------------| -| num_colors | uint32 | Number of colors in palette | -| colors[] | RGB888 | Array of 24-bit RGB colors (R, G, B) | - -Each color is stored as 3 bytes (R, G, B) in the range 0-255, expanded from RGB332 format. - ---- - -## Coordinate System - -- All coordinates are relative to the tile, with the top-left of the tile as (0,0) and bottom-right as (65535,65535). -- This allows for sub-pixel precision and scalable rendering at different resolutions. -- **Delta encoding**: Most commands use coordinate differences for better compression. -- **Zigzag encoding**: Signed coordinate differences are encoded efficiently. - ---- - -## Layer Priority System - -Features are rendered in priority order determined by the `get_layer_priority()` function: - -| Priority | Feature Type | -|----------|------------------------------------------------------| -| 100 | Water features (natural=water/coastline/bay, waterway=riverbank/dock/boatyard) | -| 200 | Land use and natural areas (landuse, natural=wood/forest/scrub/heath/grassland/beach/sand/wetland, leisure=park/nature_reserve/garden) | -| 300 | Waterways (waterway=river/stream/canal) | -| 400 | Natural terrain features (natural=peak/ridge/volcano/cliff) | -| 500 | Tunnels (tunnel=yes) | -| 600 | Railways | -| 700 | Pedestrian ways (highway=path/footway/cycleway/steps/pedestrian/track) | -| 800 | Tertiary roads (highway=tertiary/tertiary_link) | -| 900 | Secondary roads (highway=secondary/secondary_link) | -| 1000 | Primary roads (highway=primary/primary_link) | -| 1100 | Trunk roads (highway=trunk/trunk_link) | -| 1200 | Motorways (highway=motorway/motorway_link) | -| 1300 | Bridges and aeroways (bridge=yes, aeroway) | -| 1400 | Buildings | -| 1500 | Amenities | - -The OSM `layer` tag can add ±1000 * layer_value to the base priority. - ---- - -## Geometry Simplification - -The generator includes Douglas-Peucker simplification for zoom levels ≥ 16: - -### Simplification Tolerance by Zoom Level -- Zoom 16: 0.000015 -- Zoom 17: 0.000012 -- Zoom 18: 0.00001 -- Zoom 19: 0.000008 -- Zoom ≥20: 0.000005 - -### Simplification Rules -- Only applied to features with ≥50 vertices -- LineStrings with <10 points are not simplified -- Polygons maintain their closed ring structure -- Can be disabled with `--no-simplify` flag - ---- - -## Optimization Benefits - -### Dynamic Palette System -- **Primary optimization**: Replaces RGB332 values with compact indices -- **Automatic generation**: Palette built from configuration file -- **Maximum compression**: Frequently used colors get low indices (smaller varint) -- **Benefit**: 25-40% file size reduction compared to embedded colors - -### State Command Architecture -- Eliminates redundant color fields in geometry commands -- One color command can apply to multiple geometry commands -- **Benefit**: Additional 15-25% file size reduction in dense tiles - -### Width-Based Rendering -- Intelligent width calculation based on OSM tags -- Zoom-level adaptive scaling -- Meter-to-pixel conversion with Mercator projection -- **Benefit**: Realistic and consistent feature rendering across zoom levels - -### File Size Reduction -- **Dynamic palette**: 25-40% reduction compared to embedded colors -- **State commands**: Additional 15-25% reduction in dense tiles -- **Delta encoding**: Efficient coordinate compression -- **Geometry simplification**: 30-50% reduction for high-zoom complex features -- **Total improvement**: Up to 60% smaller than unoptimized format - ---- - -## Performance Considerations - -### Rendering Performance -- **Fewer state changes**: Grouped commands reduce GPU state changes -- **Efficient primitives**: Basic line and polygon rendering -- **Width-aware rendering**: Single pass with variable line widths -- **Cache efficiency**: Sequential commands improve cache hits - -### Memory Usage -- **Palette storage**: Minimal overhead (typically <200 bytes for palette.bin) -- **Parser state**: Single color variable -- **Coordinate buffers**: Small buffers for delta decoding - ---- - -## Compatibility and Forward Compatibility - -### Version Compatibility -- **Basic parsers**: Must implement commands 0x01-0x03, 0x80-0x81 -- **Width parameter**: All geometry commands now include width (as of current version) -- **Fallback rendering**: Unknown commands can be skipped -- **State preservation**: Color state persists across unknown commands - -### Implementation Guidelines -- Always implement basic commands (0x01-0x03, 0x80-0x81) -- Load and use the palette.bin file for color interpretation -- Support width parameters for all geometry commands -- Unknown command types should be skipped gracefully -- Maintain color state across all commands - ---- - -## Usage - -### Basic Implementation Requirements -1. Read the file and parse the initial varint (`num_commands`) -2. Initialize `current_color = 0xFF` -3. Load dynamic palette from `palette.bin` -4. Implement state command handlers (SET_COLOR, SET_COLOR_INDEX) -5. Implement basic geometry commands (LINE, POLYLINE, STROKE_POLYGON) -6. Support width parameter in all geometry commands - -### Command-Line Usage -```bash -python tile_generator.py input.gol output_dir config.json --zoom 6-17 [options] - -Options: - --max-file-size KB Maximum tile file size (default: 128KB) - --batch-size N Base batch size, auto-adjusted per zoom (default: 10000) - --no-simplify Disable Douglas-Peucker simplification -``` - -### Error Handling -- Invalid command types should be skipped gracefully -- Out-of-bounds palette indices should use default color (0xFF) -- Missing palette should fall back to direct RGB332 interpretation -- Corrupted files should fail safely without crashing -- Coordinate overflow should clamp to tile boundaries (0-65535) - ---- - -## References - -- [Protocol Buffers Varint Encoding](https://developers.google.com/protocol-buffers/docs/encoding#varints) -- [Zigzag Encoding](https://developers.google.com/protocol-buffers/docs/encoding#signed-integers) -- [RGB332 Color Format](https://en.wikipedia.org/wiki/List_of_monochrome_and_RGB_palettes#RGB332) - ---- \ No newline at end of file diff --git a/docs/features_json_format.md b/docs/features_json_format.md deleted file mode 100644 index db572c3..0000000 --- a/docs/features_json_format.md +++ /dev/null @@ -1,161 +0,0 @@ -# Features JSON Format Specification - -This document describes the structure and usage of the `features.json` configuration file for vector tile generation. -The file defines feature styling, priority, and minimum zoom level for OpenStreetMap (OSM) tags to be rendered in the generated tiles. - ---- - -## File Overview - -- The file is a JSON object containing **feature definitions** for OpenStreetMap data styling and filtering. -- **Feature definitions** specify styling, priority, and minimum zoom level for OSM tags. - ---- - -## Feature Definitions - -Each feature definition is a key-value pair where: -- The key is a **feature selector** for OSM data, matching tags in the form `key=value` or just `key`. -- The value is an object specifying options for rendering that feature. - ---- - -## Feature Selector - -- The key is either: - - `"key=value"`: Matches OSM features where the given tag equals the specified value. - - `"key"`: Matches any OSM feature with the given key present (any value). - -**Examples:** -```json -"natural=coastline": {...} -"building": {...} -``` - ---- - -## Feature Parameters - -Each feature definition object can include: - -| Parameter | Type | Description | -|--------------|----------|---------------------------------------------------------------------------------------------------| -| zoom | integer | Minimum zoom level at which the feature is rendered. | -| color | string | Fill/stroke color in HTML hexadecimal format (`#rrggbb`). | -| description | string | Human-readable description of the feature. | -| priority | integer | Rendering priority (lower numbers are rendered first/underneath; higher numbers are on top). | - ---- - -### Parameter Details - -- **zoom** - - Specifies the lowest zoom at which the feature will be rendered. - - Features are omitted from tiles with a zoom less than this value. - -- **color** - - Specifies the color to render the feature. - - Uses standard 6-character hex notation (e.g., `#ffbe00` for yellow). - - This is mapped to a compact color encoding (RGB332) in the binary tiles. - -- **description** - - Provides a short, human-readable label for the feature type. - -- **priority** - - Controls draw order. - - Lower values are rendered below higher values in the tile. - ---- - -## Example - -```json -{ - "natural=coastline": { - "zoom": 6, - "color": "#0077FF", - "description": "Coastlines", - "priority": 1 - }, - "natural=water": { - "zoom": 12, - "color": "#3399FF", - "description": "Water bodies", - "priority": 1 - }, - "building": { - "zoom": 15, - "color": "#BBBBBB", - "description": "Buildings", - "priority": 9 - }, - "highway=motorway": { - "zoom": 6, - "color": "#FFFFFF", - "description": "Motorways", - "priority": 10 - }, - "highway=primary": { - "zoom": 6, - "color": "#FFA500", - "description": "Primary roads", - "priority": 12 - }, - "amenity=hospital": { - "zoom": 15, - "color": "#FF0000", - "description": "Hospitals", - "priority": 8 - } -} -``` - -- **Feature definitions**: Examples showing different feature types and their properties -- **Complete configuration**: The actual `features.json` contains 497 feature definitions covering all major OSM categories - ---- - -## How Matching Works - -- During tile generation, each OSM feature is checked against the keys in the JSON: - - If the feature has a matching tag (`key=value`), the corresponding entry is used for rendering. - - If only the key matches (e.g., `"building"`), the entry applies for any value of that key. - -- If a feature matches multiple entries, the most specific (key=value) takes precedence over generic key matches. - -- Features are filtered by zoom level: only features with `zoom` parameter less than or equal to the current tile zoom level are included. - -- Features are rendered in priority order: lower priority numbers are drawn first (underneath), higher priority numbers are drawn on top. - ---- - -## Polygon Rendering - -Polygons can be rendered in two modes (controlled by the tile viewer application): - -- **Filled mode**: Polygons are filled with their specified color and have outlines: - - **Interior segments**: Outlines are 40% darker than the fill color - - **Border segments**: Outlines use the original polygon color -- **Outline mode**: Polygons show only outlines: - - **Interior segments**: Outlines use the original polygon color - - **Border segments**: Outlines use the background color (making them invisible) - -This creates a visual distinction between polygon interiors and tile borders. - -## Usage Notes - -- You can extend the file with new feature selectors and styling as needed. -- All parameters are optional, but `zoom` and `color` are recommended for correct rendering. -- This file drives both filtering (which features are extracted from the OSM source) and styling (how they appear in the output tiles). -- Priority values range from 1-35 in the current configuration, with natural features having the lowest priorities and place labels having the highest. -- Zoom levels range from 6-17, with major features (coastlines, motorways) appearing at low zoom levels and detailed features (individual trees, street lamps) appearing at high zoom levels. -- Color values use standard HTML hex notation and are automatically converted to RGB332 format for efficient storage in binary tiles. - ---- - -## References - -- [OpenStreetMap Tagging](https://wiki.openstreetmap.org/wiki/Tags) -- [Hex Color Codes](https://www.w3schools.com/colors/colors_picker.asp) - ---- \ No newline at end of file diff --git a/docs/tile_viewer.md b/docs/tile_viewer.md deleted file mode 100644 index f8649fc..0000000 --- a/docs/tile_viewer.md +++ /dev/null @@ -1,345 +0,0 @@ -# Tile Viewer - -`tile_viewer.py` is an advanced Python application for visualizing map tiles, supporting both custom vector tiles in `.bin` format and raster tiles in `.png` format. -It provides an interactive map viewer with mouse and keyboard controls, allowing exploration of tiles rendered in either format. - -## Features - -- **High Performance**: LRU cache system with configurable memory limits -- **Multi-threading**: Persistent thread pool for efficient tile loading -- **Error Recovery**: Graceful handling of missing or corrupted tiles -- **Lazy Loading**: Only loads visible tiles to optimize memory usage -- **Advanced Rendering**: Supports advanced compression commands (GRID_PATTERN, CIRCLE, PREDICTED_LINE, DASHED_LINE, DOTTED_LINE) -- **Dynamic Palette**: Loads color palettes from binary files (`palette.bin`) or configuration files (`features.json`) -- **Modern UI**: Beautiful icons and improved button text rendering -- **Layer Support**: Renders tiles in correct layer order (TERRAIN, WATER, BUILDINGS, OUTLINES, ROADS, LABELS) -- **Polygon Filling**: Toggle between filled and outlined polygon rendering - ---- - -## What Does This Script Do? - -- Loads and renders tiles from a directory containing tiles in `.bin` (vector) **or** `.png` (raster) format -- Supports multiple zoom levels and smooth panning -- Displays tile boundaries, coordinates, and labels -- Offers interactive controls via mouse and keyboard -- Can show GPS cursor coordinates and fill polygons in vector tiles -- Implements intelligent caching and memory management -- Provides detailed logging and error reporting -- Loads dynamic color palettes for accurate tile rendering - -**Note:** If both `.bin` and `.png` files are present for a tile, `.bin` is preferred and rendered as vector graphics. -This script is useful for visualizing both the output of the vector tile generator and other map tiles in PNG format. - ---- - -## Directory Structure - -The tile viewer expects the following directory structure: - -``` -TILES_DIRECTORY/ -├── 6/ # Zoom level 6 -│ ├── 0/ # Tile X coordinate -│ │ ├── 0.bin # Vector tile (preferred) -│ │ └── 0.png # Raster tile (fallback) -│ ├── 1/ -│ │ ├── 0.bin -│ │ └── 0.png -│ └── ... -├── 7/ # Zoom level 7 -│ ├── 0/ -│ │ ├── 0.bin -│ │ └── 0.png -│ └── ... -├── 8/ # Zoom level 8 -├── ... -└── palette.bin # Binary color palette (optional, preferred) - -features.json # Feature configuration (optional, fallback) - in script directory -``` - -### Tile File Formats - -- **`.bin` files**: Custom vector tile format with compressed drawing commands -- **`.png` files**: Standard raster tile format (fallback) -- **`palette.bin`**: Binary color palette file generated by `tile_generator.py` (in tile directory) -- **`features.json`**: JSON configuration file with feature definitions and colors (in script directory) - ---- - -## Dependencies - -### Python Packages - -You need the following Python packages: - -- `pygame` - Graphics and window management -- `struct` - Binary data handling (built-in) -- `threading` - Multi-threading support (built-in) -- `concurrent.futures` - Thread pool management (built-in) -- `collections` - Data structures (built-in) -- `functools` - Function utilities (built-in) - -### System Dependencies - -For optimal performance, you may also need: - -- `python3-dev` - Python development headers -- `libsdl2-dev` - SDL2 development libraries (for pygame) - -### Installation - -#### Using pip (Python packages): - -```sh -pip install pygame -``` - -#### Using apt (System packages): - -```sh -# Ubuntu/Debian -sudo apt update -sudo apt install python3-pygame python3-dev libsdl2-dev - -# Or install pygame system-wide -sudo apt install python3-pygame -``` - -#### Complete installation script: - -```sh -# Update package list -sudo apt update - -# Install system dependencies -sudo apt install python3 python3-pip python3-dev libsdl2-dev - -# Install Python packages -pip3 install pygame - -# Verify installation -python3 -c "import pygame; print('Pygame version:', pygame.version.ver)" -``` - ---- - -## How It Works - -### Controls - -- **Mouse Drag:** Pan the viewport across the map -- **Mouse Wheel / `[ ]` keys:** Zoom in and out between available zoom levels -- **Arrow Keys:** Move viewport left, right, up, or down -- **Buttons (on the right toolbar):** - - **Background:** Toggle background color (black/white) - - **Tile Labels:** Show/hide tile coordinates and file names - - **GPS Cursor:** Show/hide cursor latitude/longitude (in decimal and GMS) - - **Fill Polygons:** Toggle filled polygons rendering (for vector `.bin` tiles) -- **`l` key:** Toggle tile labels -- **`f` key:** Toggle polygon filling -- **Toolbar buttons:** Click to activate features with improved multi-line text rendering - -### Status Bar - -- Shows the current zoom level and progress bars for indexing/loading tiles -- Displays cache statistics and performance metrics -- Real-time logging of operations and errors - -### Startup - -Start the script with the tile directory as argument: - -```sh -python3 tile_viewer.py TILES_DIRECTORY -``` - ---- - -## Configuration - -The viewer uses hardcoded default configuration values. All viewer-specific settings are built into the application: - -### Default Configuration - -```python -DEFAULT_CONFIG = { - 'tile_size': 256, - 'viewport_size': 768, - 'toolbar_width': 160, - 'statusbar_height': 40, - 'max_cache_size': 1000, - 'thread_pool_size': 4, - 'background_colors': [(0, 0, 0), (255, 255, 255)], - 'log_level': 'INFO', - 'config_file': 'features.json', - 'fps_limit': 30, - 'fill_polygons': False -} -``` - -### Configuration Options - -- `tile_size`: Size of individual tiles in pixels (default: 256) -- `viewport_size`: Size of the viewport window (default: 768) -- `toolbar_width`: Width of the right toolbar in pixels (default: 160) -- `statusbar_height`: Height of the status bar in pixels (default: 40) -- `max_cache_size`: Maximum number of tiles to cache (default: 1000) -- `thread_pool_size`: Number of worker threads for tile loading (default: 4) -- `background_colors`: Array of background colors for toggle (default: black and white) -- `log_level`: Logging level (default: "INFO") -- `fps_limit`: Maximum frames per second (default: 30) -- `fill_polygons`: Whether to fill polygons by default (default: False) - -**Note:** All configuration values are hardcoded for optimal performance and consistency. - ---- - -## Palette Loading - -The viewer supports two methods for loading color palettes: - -### 1. Binary Palette (Preferred) - -The viewer automatically looks for `palette.bin` in the tile directory: - -```sh -# Generated by tile_generator.py in the tile directory -TILES_DIRECTORY/ -├── 6/ -├── 7/ -├── ... -└── palette.bin -``` - -**Format:** 4-byte header (number of colors) + RGB888 color data (3 bytes per color) - -### 2. JSON Configuration (Fallback) - -If `palette.bin` is not found, the viewer falls back to `features.json` for color extraction: - -The viewer extracts unique colors from feature definitions in `features.json` to build the palette dynamically. - ---- - -## Performance Features - -- **LRU Cache**: Intelligent memory management with configurable limits -- **Thread Pool**: Persistent worker threads for efficient tile loading -- **Lazy Loading**: Only loads visible tiles to optimize memory usage -- **Coordinate Caching**: Cached coordinate conversions for better performance -- **Error Recovery**: Graceful handling of missing or corrupted tiles -- **Memory Pool**: Reusable objects to reduce garbage collection -- **Streaming**: Processes tiles in batches to reduce memory footprint - ---- - -## Supported Drawing Commands - -The viewer supports a wide range of drawing commands for vector tiles: - -### Basic Commands -- `LINE` (1): Draw a line between two points -- `POLYLINE` (2): Draw a polyline with multiple points -- `STROKE_POLYGON` (3): Draw a polygon outline -- `HORIZONTAL_LINE` (5): Draw a horizontal line -- `VERTICAL_LINE` (6): Draw a vertical line - -### Advanced Commands -- `RECTANGLE` (0x82): Draw a rectangle -- `STRAIGHT_LINE` (0x83): Draw a straight line -- `HIGHWAY_SEGMENT` (0x84): Draw highway segments -- `GRID_PATTERN` (0x85): Draw grid patterns -- `BLOCK_PATTERN` (0x86): Draw city block patterns -- `CIRCLE` (0x87): Draw circles -- `PREDICTED_LINE` (0x8A): Draw predicted lines -- `COMPRESSED_POLYLINE` (0x8B): Draw compressed polylines -- `OPTIMIZED_POLYGON` (0x8C): Draw optimized polygons -- `HOLLOW_POLYGON` (0x8D): Draw hollow polygons -- `OPTIMIZED_TRIANGLE` (0x8E): Draw optimized triangles -- `OPTIMIZED_RECTANGLE` (0x8F): Draw optimized rectangles -- `OPTIMIZED_CIRCLE` (0x90): Draw optimized circles -- `SIMPLE_RECTANGLE` (0x96): Draw simple rectangles -- `SIMPLE_CIRCLE` (0x97): Draw simple circles -- `SIMPLE_TRIANGLE` (0x98): Draw simple triangles -- `DASHED_LINE` (0x99): Draw dashed lines -- `DOTTED_LINE` (0x9A): Draw dotted lines - -### State Commands -- `SET_COLOR` (0x80): Set direct RGB332 color -- `SET_COLOR_INDEX` (0x81): Set color from palette index -- `SET_LAYER` (0x88): Set rendering layer -- `RELATIVE_MOVE` (0x89): Move cursor relatively - ---- - -## Rendering Layers - -Tiles are rendered in the following layer order (from bottom to top): - -1. **TERRAIN** (0): Background terrain, water bodies -2. **WATER** (1): Rivers, lakes, oceans -3. **BUILDINGS** (2): Buildings, structures -4. **OUTLINES** (3): Polygon outlines, borders -5. **ROADS** (4): Roads, highways, paths -6. **LABELS** (5): Text labels, symbols - ---- - -## Troubleshooting - -### Common Issues - -1. **"No zoom level directories found"** - - Ensure your tile directory contains subdirectories named with numbers (6, 7, 8, etc.) - - Check that the directory path is correct - -2. **"Could not load palette from palette.bin"** - - The viewer will fall back to `features.json` - - Ensure `palette.bin` was generated by `tile_generator.py` - -3. **"Tile file does not exist"** - - Check that tile files exist in the expected `{zoom}/{x}/{y}.bin` or `.png` format - - Verify file permissions - -4. **Performance Issues** - - Reduce `max_cache_size` in configuration - - Lower `thread_pool_size` for systems with limited cores - - Check available system memory - -### Debug Mode - -Enable debug logging by modifying the configuration: - -```python -# In tile_viewer.py, change: -logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') -``` - ---- - -## Notes - -- If the tile directory contains both `.bin` and `.png` files, `.bin` is preferred for vector rendering -- The viewer supports multiple zoom levels, switching seamlessly with mouse wheel or bracket keys -- The GPS tooltip shows coordinates at the mouse position, both in decimal degrees and degrees/minutes/seconds (GMS) -- You can visualize both vector tiles (`.bin`) and raster tiles (`.png`) in the same directory -- The application includes comprehensive logging for debugging and monitoring -- Modern UI with beautiful icons and improved text rendering -- Polygon filling can be toggled on/off for different visualization needs -- The viewer automatically handles tile borders and interior segments differently for better visual quality - ---- - -## Integration with Tile Generator - -This viewer is designed to work seamlessly with `tile_generator.py`: - -1. **Generate tiles**: Run `tile_generator.py` to create `.bin` tiles and `palette.bin` -2. **View tiles**: Run `tile_viewer.py` to visualize the generated tiles -3. **Automatic palette**: The viewer automatically loads the correct color palette from `palette.bin` -4. **Layer support**: All rendering layers are properly supported -5. **Command compatibility**: All drawing commands are fully supported - -The viewer provides an excellent way to preview and debug tiles generated by the tile generator. \ No newline at end of file diff --git a/features.json b/features.json index cf2f773..93b25c4 100644 --- a/features.json +++ b/features.json @@ -1,482 +1,419 @@ { - "tile_size": 256, - "viewport_size": 768, - "toolbar_width": 160, - "statusbar_height": 40, - "max_cache_size": 1000, - "thread_pool_size": 4, - "background_colors": [ - [0, 0, 0], - [255, 255, 255] - ], - "log_level": "INFO", - "config_file": "features.json", - "fps_limit": 30, - "fill_polygons": true, + "_comment": "OSM-style colors in RGB565 format. Priority: layer_base + (priority % 10).", + "_layers": "water:10, landuse:20, terrain:30, railways:40, roads:50, infrastructure:60, buildings:70, amenities:80, places:90", "natural=coastline": { "zoom": 6, - "color": "#88c9fa", - "priority": 100 + "color": "#aad3df", + "priority": 10 }, "natural=water": { "zoom": 12, - "color": "#88c9fa", - "priority": 100 + "color": "#aad3df", + "priority": 11 }, "natural=bay": { "zoom": 12, - "color": "#88c9fa", - "priority": 100 + "color": "#aad3df", + "priority": 11 }, "waterway=riverbank": { "zoom": 12, - "color": "#88c9fa", - "priority": 100 + "color": "#aad3df", + "priority": 12 }, "waterway=dock": { "zoom": 14, - "color": "#88c9fa", - "priority": 100 - }, - "waterway=boatyard": { - "zoom": 14, - "color": "#88c9fa", - "priority": 100 + "color": "#aad3df", + "priority": 12 }, "waterway=river": { "zoom": 8, - "color": "#88c9fa", - "priority": 70 + "color": "#aad3df", + "priority": 15 }, "waterway=stream": { - "zoom": 8, - "color": "#88c9fa", - "priority": 70 + "zoom": 12, + "color": "#aad3df", + "priority": 16 }, "waterway=canal": { "zoom": 12, - "color": "#88c9fa", - "priority": 70 - }, - "natural=spring": { - "zoom": 17, - "color": "#88c9fa", - "priority": 26 + "color": "#aad3df", + "priority": 16 }, "natural=beach": { "zoom": 12, - "color": "#f7e9c9", - "priority": 90 + "color": "#fff1ba", + "priority": 20 }, "natural=sand": { "zoom": 12, - "color": "#f7e9c9", - "priority": 90 + "color": "#f5e9c6", + "priority": 20 }, "natural=wetland": { "zoom": 12, - "color": "#c2e0e4", - "priority": 90 + "color": "#add19e", + "priority": 21 }, "natural=wood": { "zoom": 12, - "color": "#9bc184", - "priority": 90 + "color": "#add19e", + "priority": 22 }, "landuse=forest": { "zoom": 12, - "color": "#9bc184", - "priority": 90 - }, - "natural=forest": { - "zoom": 12, - "color": "#9bc184", - "priority": 90 + "color": "#add19e", + "priority": 22 }, "natural=scrub": { - "zoom": 12, - "color": "#b5d0a3", - "priority": 90 + "zoom": 14, + "color": "#c8d7ab", + "priority": 22 }, "natural=heath": { - "zoom": 12, - "color": "#d4d8a3", - "priority": 90 + "zoom": 14, + "color": "#d6d99f", + "priority": 22 }, "natural=grassland": { + "zoom": 14, + "color": "#cdebb0", + "priority": 22 + }, + "landuse=farmland": { "zoom": 12, - "color": "#c8e6a9", - "priority": 90 + "color": "#eef0d5", + "priority": 23 }, "landuse=meadow": { "zoom": 14, - "color": "#c8e6a9", - "priority": 90 + "color": "#cdebb0", + "priority": 23 }, "landuse=grass": { "zoom": 14, - "color": "#c8e6a9", - "priority": 90 + "color": "#cdebb0", + "priority": 23 + }, + + "landuse=residential": { + "zoom": 12, + "color": "#e0dfdf", + "priority": 24 }, - "landuse=orchard": { + "landuse=commercial": { "zoom": 14, - "color": "#c8e6a9", - "priority": 90 + "color": "#f2dad9", + "priority": 24 }, - "landuse=vineyard": { + "landuse=retail": { "zoom": 14, - "color": "#c8e6a9", - "priority": 90 + "color": "#ffd6d1", + "priority": 24 }, - "landuse=farmland": { - "zoom": 12, - "color": "#e8e4b5", - "priority": 90 + "landuse=garages": { + "zoom": 15, + "color": "#dfddce", + "priority": 24 }, - "natural=tree_row": { - "zoom": 16, - "color": "#9bc184", - "priority": 29 + "landuse=industrial": { + "zoom": 12, + "color": "#ebdbe8", + "priority": 24 }, - "natural=tree": { - "zoom": 17, - "color": "#9bc184", - "priority": 28 + "landuse=cemetery": { + "zoom": 14, + "color": "#aacbaf", + "priority": 24 }, "landuse=park": { "zoom": 12, - "color": "#b5e3b5", - "priority": 85 + "color": "#c8facc", + "priority": 26 }, "leisure=park": { "zoom": 12, - "color": "#b5e3b5", - "priority": 85 - }, - "leisure=nature_reserve": { - "zoom": 12, - "color": "#b5e3b5", - "priority": 85 + "color": "#c8facc", + "priority": 26 }, "leisure=garden": { "zoom": 14, - "color": "#b5e3b5", - "priority": 85 - }, - "leisure=pitch": { - "zoom": 12, - "color": "#b5e3b5", - "priority": 85 + "color": "#cdebb0", + "priority": 26 }, - "leisure=golf_course": { - "zoom": 12, - "color": "#b5e3b5", - "priority": 85 + "leisure=playground": { + "zoom": 15, + "color": "#c8facc", + "priority": 26 }, - - "landuse=residential": { + "leisure=nature_reserve": { "zoom": 12, - "color": "#e8e6e3", - "priority": 80 + "color": "#add19e", + "priority": 25 }, - "place=suburb": { + "leisure=golf_course": { "zoom": 14, - "color": "#e8e6e3", - "priority": 80 + "color": "#def6c0", + "priority": 25 }, - "landuse=commercial": { + "leisure=recreation_ground": { "zoom": 14, - "color": "#f0e0d0", - "priority": 80 + "color": "#c8facc", + "priority": 25 }, - "landuse=retail": { + "landuse=recreation_ground": { "zoom": 14, - "color": "#f0e0d0", - "priority": 80 + "color": "#c8facc", + "priority": 25 }, - "landuse=industrial": { - "zoom": 12, - "color": "#d8d8d8", - "priority": 80 + + "amenity=parking": { + "zoom": 15, + "color": "#eeeeee", + "priority": 27 }, - "landuse=construction": { - "zoom": 14, - "color": "#e8d8c8", - "priority": 80 + "amenity=marketplace": { + "zoom": 15, + "color": "#f0e0d0", + "priority": 27 }, - - "landuse=cemetery": { + "landuse=village_green": { "zoom": 14, - "color": "#c0c0c0", - "priority": 85 + "color": "#cdebb0", + "priority": 26 }, - "landuse=allotments": { + "leisure=common": { "zoom": 14, - "color": "#d8e8d8", - "priority": 85 - }, - - "leisure=stadium": { - "zoom": 12, - "color": "#d0d0d0", - "priority": 85 - }, - "leisure=sports_centre": { - "zoom": 12, - "color": "#d0d0d0", - "priority": 85 - }, - "leisure=playground": { - "zoom": 12, - "color": "#e8d8a8", - "priority": 85 + "color": "#c8facc", + "priority": 26 }, "natural=peak": { - "zoom": 15, - "color": "#8b7355", - "priority": 30 - }, - "natural=ridge": { - "zoom": 15, - "color": "#8b7355", + "zoom": 14, + "color": "#8b4513", "priority": 30 }, "natural=volcano": { - "zoom": 15, - "color": "#d84200", + "zoom": 14, + "color": "#d40000", "priority": 30 }, "natural=cliff": { "zoom": 15, - "color": "#8b7355", - "priority": 30 - }, - - "tunnel=yes": { - "zoom": 12, - "color": "#a0a0a0", - "priority": 40 + "color": "#999999", + "priority": 31 }, "railway=rail": { "zoom": 8, - "color": "#808080", - "priority": 65 + "color": "#888888", + "priority": 40 }, "railway=subway": { - "zoom": 12, - "color": "#808080", - "priority": 65 + "zoom": 14, + "color": "#999999", + "priority": 41 }, "railway=tram": { - "zoom": 12, - "color": "#808080", - "priority": 65 + "zoom": 14, + "color": "#888888", + "priority": 41 + }, + + "tunnel=yes": { + "zoom": 14, + "color": "#aaaaaa", + "priority": 50 }, "highway=motorway": { "zoom": 6, - "color": "#ff9999", - "priority": 50 + "color": "#e990a0", + "priority": 60 }, "highway=motorway_link": { - "zoom": 6, - "color": "#ff9999", - "priority": 50 + "zoom": 8, + "color": "#e990a0", + "priority": 60 }, "highway=trunk": { "zoom": 6, - "color": "#ffbbbb", - "priority": 49 + "color": "#f9b29c", + "priority": 61 }, "highway=trunk_link": { - "zoom": 6, - "color": "#ffbbbb", - "priority": 49 + "zoom": 8, + "color": "#f9b29c", + "priority": 61 }, "highway=primary": { "zoom": 6, - "color": "#ffcc99", - "priority": 48 + "color": "#fcd6a4", + "priority": 62 }, "highway=primary_link": { - "zoom": 6, - "color": "#ffcc99", - "priority": 48 + "zoom": 10, + "color": "#fcd6a4", + "priority": 62 }, "highway=secondary": { - "zoom": 12, - "color": "#ffff99", - "priority": 47 + "zoom": 10, + "color": "#f7f496", + "priority": 63 }, "highway=secondary_link": { "zoom": 12, - "color": "#ffff99", - "priority": 47 + "color": "#f7f496", + "priority": 63 }, "highway=tertiary": { "zoom": 12, - "color": "#ffffcc", - "priority": 46 + "color": "#ffffff", + "priority": 64 }, "highway=tertiary_link": { "zoom": 12, - "color": "#ffffcc", - "priority": 46 + "color": "#ffffff", + "priority": 64 }, "highway=residential": { "zoom": 14, - "color": "#f8f8f8", - "priority": 45 + "color": "#ffffff", + "priority": 65 }, "highway=living_street": { "zoom": 14, - "color": "#f8f8f8", - "priority": 44 + "color": "#ededed", + "priority": 65 }, "highway=unclassified": { "zoom": 12, - "color": "#f8f8f8", - "priority": 34 + "color": "#ffffff", + "priority": 65 }, "highway=service": { - "zoom": 14, - "color": "#f0f0f0", - "priority": 43 - }, - "highway=pedestrian": { "zoom": 15, - "color": "#f8f8f8", - "priority": 42 + "color": "#ffffff", + "priority": 66 }, "highway=track": { - "zoom": 15, - "color": "#e8d8b0", - "priority": 41 + "zoom": 14, + "color": "#996600", + "priority": 67 }, "highway=path": { "zoom": 15, - "color": "#e8d8b0", - "priority": 40 + "color": "#fa8072", + "priority": 68 + }, + "highway=pedestrian": { + "zoom": 14, + "color": "#dddde8", + "priority": 67 }, "highway=footway": { - "zoom": 15, - "color": "#f0e8d8", - "priority": 39 + "zoom": 16, + "color": "#fa8072", + "priority": 68 }, "highway=cycleway": { - "zoom": 15, - "color": "#a0c8f0", - "priority": 38 + "zoom": 14, + "color": "#0000ff", + "priority": 68 }, "highway=steps": { "zoom": 16, - "color": "#f0e8d8", - "priority": 37 - }, - "highway=crossing": { - "zoom": 16, - "color": "#ffff00", - "priority": 36 - }, - "highway=bus_stop": { - "zoom": 17, - "color": "#4682b4", - "priority": 35 - }, - "highway=street_lamp": { - "zoom": 17, - "color": "#ffffcc", - "priority": 27 + "color": "#fa8072", + "priority": 68 }, "bridge=yes": { "zoom": 12, - "color": "#c0c0c0", - "priority": 60 + "color": "#b8b8b8", + "priority": 70 }, "man_made=bridge": { "zoom": 12, - "color": "#c0c0c0", - "priority": 60 + "color": "#b8b8b8", + "priority": 70 }, "aeroway=runway": { "zoom": 12, - "color": "#e8e8e8", - "priority": 55 + "color": "#bbbbcc", + "priority": 75 }, "aeroway=taxiway": { "zoom": 14, - "color": "#e0e0e0", - "priority": 55 + "color": "#bbbbcc", + "priority": 75 }, "aeroway=apron": { "zoom": 14, - "color": "#d8d8d8", - "priority": 55 + "color": "#dadae0", + "priority": 74 }, "building": { "zoom": 15, - "color": "#dddddd", - "priority": 60 + "color": "#d9d0c9", + "priority": 80 }, - "man_made=tower": { + "leisure=stadium": { + "zoom": 14, + "color": "#c8facc", + "priority": 81 + }, + "leisure=sports_centre": { "zoom": 15, - "color": "#cccccc", - "priority": 55 + "color": "#c8facc", + "priority": 81 }, - - "amenity=parking": { + "leisure=pitch": { "zoom": 15, - "color": "#e8e8e8", - "priority": 75 + "color": "#88e0be", + "priority": 79 }, + "amenity=hospital": { - "zoom": 15, - "color": "#ffcccc", - "priority": 55 + "zoom": 14, + "color": "#f0c0c0", + "priority": 85 }, "amenity=school": { "zoom": 15, - "color": "#fff8dc", - "priority": 55 + "color": "#f0f0d8", + "priority": 85 }, "amenity=university": { - "zoom": 15, - "color": "#fff8dc", - "priority": 55 - }, - "amenity=place_of_worship": { - "zoom": 15, - "color": "#fff8dc", - "priority": 55 + "zoom": 14, + "color": "#f0f0d8", + "priority": 85 }, "place=state": { - "zoom": 8, - "color": "#4a5c6a", - "priority": 25 + "zoom": 6, + "color": "#000000", + "priority": 90 }, "place=town": { - "zoom": 12, - "color": "#5a6c7a", - "priority": 24 + "zoom": 10, + "color": "#000000", + "priority": 91 }, "place=village": { "zoom": 12, - "color": "#6a7c8a", - "priority": 23 + "color": "#000000", + "priority": 92 }, "place=hamlet": { - "zoom": 15, - "color": "#7a8c9a", - "priority": 22 + "zoom": 14, + "color": "#000000", + "priority": 93 } -} \ No newline at end of file +} diff --git a/generate_tiles.sh b/generate_tiles.sh deleted file mode 100755 index 16aa8c0..0000000 --- a/generate_tiles.sh +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash - -# Script to generate tiles from PBF using Docker -# Converts PBF -> GOL -> Tiles automatically -# Optimized version with minimal dependencies - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Banner -echo -e "${BLUE}" -echo "╔════════════════════════════════════════╗" -echo "║ PBF → GOL → Tiles Generator ║" -echo "╚════════════════════════════════════════╝" -echo -e "${NC}" - -# Initialize variables -FORCE_CLEAN=false -ZOOM_ARGS="" - -# Parse all arguments -while [[ $# -gt 0 ]]; do - case $1 in - --clean-docker) - FORCE_CLEAN=true - shift - ;; - --zoom) - ZOOM_ARGS="$ZOOM_ARGS --zoom $2" - shift 2 - ;; - --max-file-size) - ZOOM_ARGS="$ZOOM_ARGS --max-file-size $2" - shift 2 - ;; - *) - # Positional arguments - if [ -z "$INPUT_PBF" ]; then - INPUT_PBF=$1 - elif [ -z "$OUTPUT_DIR" ]; then - OUTPUT_DIR=$1 - elif [ -z "$CONFIG_FILE" ]; then - CONFIG_FILE=$1 - else - echo "Unknown argument: $1" - exit 1 - fi - shift - ;; - esac -done - -# Check required arguments -if [ -z "$INPUT_PBF" ] || [ -z "$OUTPUT_DIR" ] || [ -z "$CONFIG_FILE" ]; then - echo "Usage: $0 [options]" - echo "" - echo "Options:" - echo " --clean-docker Force rebuild of Docker image" - echo " --zoom N-M Zoom range (e.g., 6-17)" - echo " --max-file-size KB Maximum tile file size in KB" - echo "" - echo "Example:" - echo " $0 map.osm.pbf tiles features.json --zoom 6-17" - echo " $0 map.osm.pbf tiles features.json --zoom 12 --max-file-size 512" - exit 1 -fi - -# Validate input -if [ ! -f "$INPUT_PBF" ]; then - echo -e "${RED}✗ PBF file not found: $INPUT_PBF${NC}" - exit 1 -fi - -if [ ! -f "$CONFIG_FILE" ]; then - echo -e "${RED}✗ Config file not found: $CONFIG_FILE${NC}" - exit 1 -fi - -# Get absolute paths -INPUT_ABS=$(realpath "$INPUT_PBF") -CONFIG_ABS=$(realpath "$CONFIG_FILE") -OUTPUT_ABS=$(realpath -m "$OUTPUT_DIR") - -INPUT_NAME=$(basename "$INPUT_PBF") -CONFIG_NAME=$(basename "$CONFIG_FILE") -OUTPUT_NAME=$(basename "$OUTPUT_DIR") - -# Create output directory -mkdir -p "$OUTPUT_ABS" -echo -e "${GREEN}✓ Output directory ready:${NC} $OUTPUT_ABS" -echo "" - -echo -e "${GREEN}Input:${NC} $INPUT_NAME" -echo -e "${GREEN}Output:${NC} $OUTPUT_NAME" -echo -e "${GREEN}Config:${NC} $CONFIG_NAME" -echo "" -echo -e "${BLUE}Mounting paths:${NC}" -echo " PBF: $INPUT_ABS -> /input/$INPUT_NAME" -echo " Config: $CONFIG_ABS -> /config/$CONFIG_NAME" -echo " Output: $OUTPUT_ABS -> /output" -echo "" - -# Check Docker -if ! command -v docker &> /dev/null; then - echo -e "${RED}✗ Docker is not installed${NC}" - exit 1 -fi - -DOCKER_CMD="docker" -if [ "$EUID" -ne 0 ]; then - DOCKER_CMD="sudo docker" -fi - -echo -e "${YELLOW}Preparing Docker environment...${NC}" - -# Create Dockerfile -cat > /tmp/Dockerfile.tiles.light << 'EOF' -FROM python:3.11-slim - -# Install only essential dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - wget ca-certificates \ - && rm -rf /var/lib/apt/lists/* >/dev/null 2>&1 - -RUN pip install --no-cache-dir --upgrade pip >/dev/null 2>&1 - -# Install minimal Python dependencies -RUN pip install --no-cache-dir \ - ijson==3.2.3 \ - tqdm==4.66.1 \ - psutil==5.9.8 >/dev/null 2>&1 - -WORKDIR /work - -# Display installed packages for verification -RUN pip list && \ - echo "Docker image ready for tile generation" -EOF - -# Handle --clean-docker option -if [ "$FORCE_CLEAN" = true ]; then - echo -e "${YELLOW}--clean-docker flag: rebuilding Docker image${NC}" - IMAGE_EXISTS=$($DOCKER_CMD images tile-generator:light -q 2>/dev/null || true) - if [ -n "$IMAGE_EXISTS" ]; then - echo " Removing existing image..." - $DOCKER_CMD rmi tile-generator:light > /dev/null 2>&1 || true - echo " ✓ Existing image removed" - fi - echo "" -fi - -# Check if Docker image exists -IMAGE_EXISTS=$($DOCKER_CMD images tile-generator:light -q 2>/dev/null || true) -if [ -z "$IMAGE_EXISTS" ]; then - echo -e "${YELLOW}Building Docker image${NC}" - echo "" - - #$DOCKER_CMD build --progress=plain -t tile-generator:light -f /tmp/Dockerfile.tiles.light . >/dev/null 2>&1 - #$DOCKER_CMD build --progress=tty -t tile-generator:light -f /tmp/Dockerfile.tiles.light . - #DOCKER_BUILDKIT=1 \ - #$DOCKER_CMD build --progress=plain -t tile-generator:light -f /tmp/Dockerfile.tiles.light . - DOCKER_BUILDKIT=1 \ - $DOCKER_CMD build -t tile-generator:light -f /tmp/Dockerfile.tiles.light . - - - - echo "" - echo -e "${GREEN}✓ Docker image created${NC}" - - # Show image size - IMAGE_SIZE=$($DOCKER_CMD images tile-generator:light --format "{{.Size}}" 2>/dev/null || echo "unknown") - echo " Image size: $IMAGE_SIZE" -else - echo -e "${GREEN}✓ Docker image already exists${NC}" - IMAGE_SIZE=$($DOCKER_CMD images tile-generator:light --format "{{.Size}}" 2>/dev/null || echo "unknown") - echo " Image size: $IMAGE_SIZE" -fi - -rm -f /tmp/Dockerfile.tiles.light - -echo "" - -# Check if gol is installed locally -GOL_PATH="" -if command -v gol &> /dev/null; then - GOL_PATH=$(which gol) - echo -e "${GREEN}Using local gol:${NC} $GOL_PATH" -else - echo -e "${RED}✗ gol CLI not found${NC}" - echo "Install from: https://www.geodesk.com/download/" - exit 1 -fi - -echo "" -echo "🚀 Starting tile generation ..." -echo "" - -# Run in Docker -$DOCKER_CMD run --rm \ - -v "$INPUT_ABS:/input/$INPUT_NAME:ro" \ - -v "$CONFIG_ABS:/config/$CONFIG_NAME:ro" \ - -v "$GOL_PATH:/gol:ro" \ - -v "$OUTPUT_ABS:/output" \ - -v "$(pwd)/tile_generator.py:/work/tile_generator.py:ro" \ - tile-generator:light bash -c " - set -e - - echo '✓ Python environment ready' - echo '' - - # Step 1: Convert PBF to GOL - echo '🔄 Step 1/2: Converting PBF to GOL...' - echo \" PBF: /input/$INPUT_NAME\" - echo \" GOL: input.gol\" - echo '' - - # Build GOL with progress monitoring - /gol build input.gol /input/$INPUT_NAME >/dev/null 2>&1 & - GOL_PID=\$! - - # Monitor progress - while kill -0 \$GOL_PID 2>/dev/null; do - if [ -f input.gol ]; then - SIZE=\$(du -h input.gol 2>/dev/null | cut -f1 || echo \"0B\") - echo -ne \"\\r GOL size: \${SIZE} (building...)\" - fi - sleep 1 - done - - echo '' - wait \$GOL_PID - - if [ ! -f input.gol ]; then - echo '✗ GOL file not created' - exit 1 - fi - - GOL_SIZE=\$(du -h input.gol | cut -f1) - echo '✓ GOL created successfully' - echo \" Size: \$GOL_SIZE\" - echo '' - - # Step 2: Generate tiles - echo '🗺️ Step 2/2: Generating tiles ...' - echo '' - - python3 /work/tile_generator.py \ - input.gol \ - /output \ - /config/$CONFIG_NAME \ - $ZOOM_ARGS - - echo '' - - # Show results - if [ -d \"/output\" ]; then - echo '✓ Tiles generation completed' - fi - " - -# Docker cleanup logic -if [ "$FORCE_CLEAN" = true ]; then - echo "" - echo -e "${YELLOW}Cleaning up Docker resources...${NC}" - - if [ -n "$IMAGE_EXISTS" ]; then - echo " Removing Docker image: tile-generator:light" - $DOCKER_CMD rmi tile-generator:light > /dev/null 2>&1 || true - fi - - # Clean dangling images - DANGLING=$($DOCKER_CMD images -f "dangling=true" -q 2>/dev/null | wc -l) - if [ "$DANGLING" -gt 0 ]; then - echo " Removing dangling images..." - $DOCKER_CMD rmi $($DOCKER_CMD images -f "dangling=true" -q) > /dev/null 2>&1 || true - fi - - echo -e "${GREEN}✓ Docker cleanup completed${NC}" -else - echo "" - echo -e "${GREEN}✓ Docker image kept for reuse${NC}" - echo " Use --clean-docker to rebuild" - IMAGE_SIZE=$($DOCKER_CMD images tile-generator:light --format "{{.Size}}" 2>/dev/null || echo "unknown") - echo " Image size: $IMAGE_SIZE " -fi - -echo "" -echo -e "${GREEN}✓ Tiles generated in: $OUTPUT_DIR${NC}" -echo "" -echo -e "${BLUE}Summary:${NC}" -echo " • Docker image kept for reuse" -echo " • Use --clean-docker to rebuild" diff --git a/nav_viewer.py b/nav_viewer.py new file mode 100644 index 0000000..fd73b27 --- /dev/null +++ b/nav_viewer.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +""" +NAV Tile Viewer - ESP32 Map Simulator + +Displays NAV binary tiles in a 768x768 viewport (3x3 tiles of 256px). + +Usage: + python nav_viewer.py nav_dir --lat 42.5063 --lon 1.5218 [--zoom 14] +""" + +import os +import sys +import math +import struct +import argparse +import logging +from typing import Dict, List, Tuple, Optional, Set + +try: + import pygame + PYGAME_AVAILABLE = True +except ImportError: + PYGAME_AVAILABLE = False + print("Warning: pygame not found. Install with: pip install pygame") + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# Constants +TILE_SIZE = 256 +VIEWPORT_SIZE = 768 +TOOLBAR_WIDTH = 200 +STATUSBAR_HEIGHT = 60 +WINDOW_WIDTH = VIEWPORT_SIZE + TOOLBAR_WIDTH +WINDOW_HEIGHT = VIEWPORT_SIZE + STATUSBAR_HEIGHT + +# NAV format constants +NAV_MAGIC = b'NAV1' +COORD_SCALE = 10000000 # 1e7 + +# Geometry types +GEOM_POINT = 1 +GEOM_LINESTRING = 2 +GEOM_POLYGON = 3 + + +def deg2num(lat_deg: float, lon_deg: float, zoom: int) -> Tuple[float, float]: + """Convert lat/lon to tile numbers.""" + lat_rad = math.radians(lat_deg) + n = 2.0 ** zoom + xtile = (lon_deg + 180.0) / 360.0 * n + ytile = (1.0 - math.log(math.tan(lat_rad) + 1.0 / math.cos(lat_rad)) / math.pi) / 2.0 * n + return xtile, ytile + + +def num2deg(xtile: float, ytile: float, zoom: int) -> Tuple[float, float]: + """Convert tile numbers to lat/lon.""" + n = 2.0 ** zoom + lon_deg = xtile / n * 360.0 - 180.0 + lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n))) + lat_deg = math.degrees(lat_rad) + return lat_deg, lon_deg + + +def get_bbox_for_viewport(center_lat: float, center_lon: float, zoom: int) -> Tuple[float, float, float, float]: + """Calculate bounding box for 3x3 tile viewport.""" + center_x, center_y = deg2num(center_lat, center_lon, zoom) + center_tile_x = int(center_x) + center_tile_y = int(center_y) + + min_tile_x = center_tile_x - 1 + max_tile_x = center_tile_x + 2 + min_tile_y = center_tile_y - 1 + max_tile_y = center_tile_y + 2 + + max_lat, min_lon = num2deg(min_tile_x, min_tile_y, zoom) + min_lat, max_lon = num2deg(max_tile_x, max_tile_y, zoom) + return (min_lon, min_lat, max_lon, max_lat) + + +def latlon_to_pixel(lat: float, lon: float, bbox: Tuple[float, float, float, float]) -> Tuple[int, int]: + """Convert lat/lon to pixel coordinates.""" + min_lon, min_lat, max_lon, max_lat = bbox + x_norm = (lon - min_lon) / (max_lon - min_lon) + y_norm = (max_lat - lat) / (max_lat - min_lat) + px = int(x_norm * VIEWPORT_SIZE) + py = int(y_norm * VIEWPORT_SIZE) + return px, py + + +def rgb565_to_rgb888(c: int) -> Tuple[int, int, int]: + """Convert RGB565 to RGB888.""" + r = ((c >> 11) & 0x1F) << 3 + g = ((c >> 5) & 0x3F) << 2 + b = (c & 0x1F) << 3 + return (r, g, b) + + +def darken_color(rgb: Tuple[int, int, int], amount: float = 0.3) -> Tuple[int, int, int]: + """Darken RGB color.""" + return tuple(max(0, int(v * (1 - amount))) for v in rgb) + + +class NavFeature: + """Parsed NAV feature.""" + def __init__(self): + self.geom_type = 0 + self.color_rgb565 = 0xFFFF + self.zoom_priority = 0 + self.width = 1 # Line width in pixels (NAV v2) + self.coords = [] # List of (lon, lat) floats + + @property + def min_zoom(self): + return self.zoom_priority >> 4 + + @property + def priority(self): + return (self.zoom_priority & 0x0F) * 7 + + +def read_nav_tile(path: str) -> List[NavFeature]: + """Read NAV tile file and return list of features.""" + features = [] + + try: + with open(path, 'rb') as f: + # Read header (22 bytes) + magic = f.read(4) + if magic != NAV_MAGIC: + logger.warning(f"Invalid magic in {path}") + return features + + feature_count = struct.unpack(' List[Tuple[int, int]]: + """Get list of tiles for current viewport.""" + center_x, center_y = deg2num(self.center_lat, self.center_lon, self.zoom) + center_tile_x = int(center_x) + center_tile_y = int(center_y) + + tiles = [] + for dy in range(-1, 2): + for dx in range(-1, 2): + tile_x = center_tile_x + dx + tile_y = center_tile_y + dy + max_tile = 2 ** self.zoom - 1 + if 0 <= tile_x <= max_tile and 0 <= tile_y <= max_tile: + tiles.append((tile_x, tile_y)) + + return tiles + + def _get_tile_path(self, x: int, y: int) -> str: + """Get file path for a tile.""" + return os.path.join(self.nav_dir, str(self.zoom), str(x), f"{y}.nav") + + def set_center(self, lat: float, lon: float, zoom: int = None): + """Set viewport center.""" + self.center_lat = lat + self.center_lon = lon + if zoom is not None: + self.zoom = zoom + self.bbox = get_bbox_for_viewport(self.center_lat, self.center_lon, self.zoom) + + def query_features(self) -> List[NavFeature]: + """Query features from tiles.""" + if self.bbox is None: + return [] + + import time + start = time.time() + + tiles = self._get_tiles_for_viewport() + all_features = [] + tiles_loaded = 0 + tiles_missing = 0 + + for tile_x, tile_y in tiles: + tile_path = self._get_tile_path(tile_x, tile_y) + if os.path.exists(tile_path): + features = read_nav_tile(tile_path) + # Filter by zoom + features = [f for f in features if f.min_zoom <= self.zoom] + all_features.extend(features) + tiles_loaded += 1 + else: + tiles_missing += 1 + + elapsed = (time.time() - start) * 1000 + + self.last_query_stats = { + 'tiles_loaded': tiles_loaded, + 'tiles_missing': tiles_missing, + 'features': len(all_features), + 'time_ms': elapsed + } + + return all_features + + def render_to_surface(self, surface: pygame.Surface): + """Render features to pygame surface.""" + if self.bbox is None: + return + + surface.fill(self.background_color) + + features = self.query_features() + if not features: + self.cached_features = None + return + + # Sort by priority (needed when combining multiple tiles) + features.sort(key=lambda f: f.priority) + self.cached_features = features + + for feature in features: + self._render_feature(surface, feature) + + if self.show_tile_grid: + self._draw_tile_grid(surface) + + def _render_feature(self, surface: pygame.Surface, feature: NavFeature): + """Render a single feature.""" + if not feature.coords: + return + + color = rgb565_to_rgb888(feature.color_rgb565) + + if feature.geom_type == GEOM_POINT: + self._render_point(surface, feature, color) + elif feature.geom_type == GEOM_LINESTRING: + self._render_linestring(surface, feature, color) + elif feature.geom_type == GEOM_POLYGON: + self._render_polygon(surface, feature, color) + + def _render_point(self, surface: pygame.Surface, feature: NavFeature, color: Tuple[int, int, int]): + """Render point.""" + if feature.coords: + lon, lat = feature.coords[0] + px, py = latlon_to_pixel(lat, lon, self.bbox) + if 0 <= px < VIEWPORT_SIZE and 0 <= py < VIEWPORT_SIZE: + pygame.draw.circle(surface, color, (px, py), 3) + + def _render_linestring(self, surface: pygame.Surface, feature: NavFeature, color: Tuple[int, int, int]): + """Render linestring with width and round joins.""" + points = [] + for lon, lat in feature.coords: + px, py = latlon_to_pixel(lat, lon, self.bbox) + points.append((px, py)) + + if len(points) >= 2: + width = max(1, feature.width) + pygame.draw.lines(surface, color, False, points, width) + # Add round joins for thick lines + if width > 2: + radius = width // 2 + for px, py in points: + pygame.draw.circle(surface, color, (px, py), radius) + + def _render_polygon(self, surface: pygame.Surface, feature: NavFeature, color: Tuple[int, int, int]): + """Render polygon.""" + points = [] + for lon, lat in feature.coords: + px, py = latlon_to_pixel(lat, lon, self.bbox) + points.append((px, py)) + + if len(points) >= 3: + if self.fill_polygons: + pygame.draw.polygon(surface, color, points) + border_color = darken_color(color, 0.4) + pygame.draw.polygon(surface, border_color, points, 1) + else: + pygame.draw.polygon(surface, color, points, 1) + + def _draw_tile_grid(self, surface: pygame.Surface): + """Draw tile grid overlay.""" + grid_color = (100, 100, 100) + font = pygame.font.SysFont(None, 14) + + for i in range(4): + x = i * TILE_SIZE + pygame.draw.line(surface, grid_color, (x, 0), (x, VIEWPORT_SIZE), 1) + for i in range(4): + y = i * TILE_SIZE + pygame.draw.line(surface, grid_color, (0, y), (VIEWPORT_SIZE, y), 1) + + tiles = self._get_tiles_for_viewport() + center_x, center_y = deg2num(self.center_lat, self.center_lon, self.zoom) + center_tile_x = int(center_x) + center_tile_y = int(center_y) + + for dy in range(-1, 2): + for dx in range(-1, 2): + tile_x = center_tile_x + dx + tile_y = center_tile_y + dy + screen_x = (dx + 1) * TILE_SIZE + 5 + screen_y = (dy + 1) * TILE_SIZE + 5 + tile_path = self._get_tile_path(tile_x, tile_y) + exists = os.path.exists(tile_path) + color = (0, 100, 0) if exists else (150, 50, 50) + label = font.render(f"{tile_x}/{tile_y}", True, color) + surface.blit(label, (screen_x, screen_y)) + + def identify_feature_at(self, pixel_x: int, pixel_y: int) -> Optional[dict]: + """Identify feature at pixel coordinates.""" + if self.cached_features is None or self.bbox is None: + return None + + min_lon, min_lat, max_lon, max_lat = self.bbox + lon = min_lon + (pixel_x / VIEWPORT_SIZE) * (max_lon - min_lon) + lat = max_lat - (pixel_y / VIEWPORT_SIZE) * (max_lat - min_lat) + + # Search in reverse priority order + for feature in reversed(self.cached_features): + if self._point_in_feature(lon, lat, feature): + return { + 'color': f"#{rgb565_to_rgb888(feature.color_rgb565)[0]:02x}{rgb565_to_rgb888(feature.color_rgb565)[1]:02x}{rgb565_to_rgb888(feature.color_rgb565)[2]:02x}", + 'min_zoom': feature.min_zoom, + 'priority': feature.priority, + 'geom_type': ['?', 'Point', 'Line', 'Polygon'][feature.geom_type], + 'coords': len(feature.coords) + } + + return None + + def _point_in_feature(self, lon: float, lat: float, feature: NavFeature) -> bool: + """Check if point is in/near feature.""" + if feature.geom_type == GEOM_POLYGON: + # Simple point-in-polygon test + n = len(feature.coords) + inside = False + j = n - 1 + for i in range(n): + xi, yi = feature.coords[i] + xj, yj = feature.coords[j] + if ((yi > lat) != (yj > lat)) and (lon < (xj - xi) * (lat - yi) / (yj - yi) + xi): + inside = not inside + j = i + return inside + elif feature.geom_type == GEOM_LINESTRING: + # Check distance to line segments + tolerance = (self.bbox[2] - self.bbox[0]) / VIEWPORT_SIZE * 5 + for i in range(len(feature.coords) - 1): + x1, y1 = feature.coords[i] + x2, y2 = feature.coords[i + 1] + dist = self._point_to_segment_dist(lon, lat, x1, y1, x2, y2) + if dist < tolerance: + return True + elif feature.geom_type == GEOM_POINT and feature.coords: + x, y = feature.coords[0] + tolerance = (self.bbox[2] - self.bbox[0]) / VIEWPORT_SIZE * 10 + if abs(lon - x) < tolerance and abs(lat - y) < tolerance: + return True + return False + + def _point_to_segment_dist(self, px, py, x1, y1, x2, y2): + """Calculate distance from point to line segment.""" + dx = x2 - x1 + dy = y2 - y1 + if dx == 0 and dy == 0: + return math.sqrt((px - x1)**2 + (py - y1)**2) + t = max(0, min(1, ((px - x1) * dx + (py - y1) * dy) / (dx * dx + dy * dy))) + proj_x = x1 + t * dx + proj_y = y1 + t * dy + return math.sqrt((px - proj_x)**2 + (py - proj_y)**2) + + +def draw_button(surface, text, rect, bg_color, fg_color, border_color, font): + """Draw a button.""" + pygame.draw.rect(surface, bg_color, rect, border_radius=8) + pygame.draw.rect(surface, border_color, rect, 2, border_radius=8) + label = font.render(text, True, fg_color) + text_rect = label.get_rect(center=rect.center) + surface.blit(label, text_rect) + + +def main(): + parser = argparse.ArgumentParser(description='NAV Tile Viewer') + parser.add_argument('nav_dir', help='Directory with NAV tiles') + parser.add_argument('--lat', type=float, required=True, help='Center latitude') + parser.add_argument('--lon', type=float, required=True, help='Center longitude') + parser.add_argument('--zoom', type=int, default=14, help='Zoom level') + + args = parser.parse_args() + + if not PYGAME_AVAILABLE: + logger.error("pygame required") + sys.exit(1) + + viewer = NAVViewer(args.nav_dir) + viewer.set_center(args.lat, args.lon, args.zoom) + + pygame.init() + screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) + pygame.display.set_caption(f"NAV Viewer - {os.path.basename(args.nav_dir)}") + + font = pygame.font.SysFont(None, 18) + font_small = pygame.font.SysFont(None, 14) + clock = pygame.time.Clock() + + viewport_surface = pygame.Surface((VIEWPORT_SIZE, VIEWPORT_SIZE)) + + button_margin = 10 + button_height = 35 + button_width = TOOLBAR_WIDTH - 20 + + bg_button_rect = pygame.Rect(VIEWPORT_SIZE + 10, button_margin, button_width, button_height) + fill_button_rect = pygame.Rect(VIEWPORT_SIZE + 10, button_margin * 2 + button_height, button_width, button_height) + grid_button_rect = pygame.Rect(VIEWPORT_SIZE + 10, button_margin * 3 + button_height * 2, button_width, button_height) + + dragging = False + drag_start = None + drag_center_start = None + need_redraw = True + running = True + pan_speed_base = 0.01 + + while running: + for event in pygame.event.get(): + if event.type == pygame.QUIT: + running = False + + elif event.type == pygame.KEYDOWN: + pan_speed = pan_speed_base / (2 ** (viewer.zoom - 10)) + + if event.key == pygame.K_LEFT: + viewer.set_center(viewer.center_lat, viewer.center_lon - pan_speed) + need_redraw = True + elif event.key == pygame.K_RIGHT: + viewer.set_center(viewer.center_lat, viewer.center_lon + pan_speed) + need_redraw = True + elif event.key == pygame.K_UP: + viewer.set_center(viewer.center_lat + pan_speed, viewer.center_lon) + need_redraw = True + elif event.key == pygame.K_DOWN: + viewer.set_center(viewer.center_lat - pan_speed, viewer.center_lon) + need_redraw = True + elif event.key == pygame.K_LEFTBRACKET: + if viewer.zoom > 1: + viewer.set_center(viewer.center_lat, viewer.center_lon, viewer.zoom - 1) + need_redraw = True + elif event.key == pygame.K_RIGHTBRACKET: + if viewer.zoom < 19: + viewer.set_center(viewer.center_lat, viewer.center_lon, viewer.zoom + 1) + need_redraw = True + elif event.key == pygame.K_b: + viewer.background_color = (0, 0, 0) if viewer.background_color == (255, 255, 255) else (255, 255, 255) + need_redraw = True + elif event.key == pygame.K_f: + viewer.fill_polygons = not viewer.fill_polygons + need_redraw = True + elif event.key == pygame.K_g: + viewer.show_tile_grid = not viewer.show_tile_grid + need_redraw = True + elif event.key in (pygame.K_q, pygame.K_ESCAPE): + running = False + + elif event.type == pygame.MOUSEBUTTONDOWN: + mx, my = event.pos + + if event.button == 1: + if bg_button_rect.collidepoint(mx, my): + viewer.background_color = (0, 0, 0) if viewer.background_color == (255, 255, 255) else (255, 255, 255) + need_redraw = True + elif fill_button_rect.collidepoint(mx, my): + viewer.fill_polygons = not viewer.fill_polygons + need_redraw = True + elif grid_button_rect.collidepoint(mx, my): + viewer.show_tile_grid = not viewer.show_tile_grid + need_redraw = True + elif mx < VIEWPORT_SIZE and my < VIEWPORT_SIZE: + dragging = True + drag_start = (mx, my) + drag_center_start = (viewer.center_lat, viewer.center_lon) + + elif event.button == 3: + if mx < VIEWPORT_SIZE and my < VIEWPORT_SIZE: + feature_info = viewer.identify_feature_at(mx, my) + viewer.selected_feature = feature_info + need_redraw = True + + elif event.button == 4: + if viewer.zoom < 19: + viewer.set_center(viewer.center_lat, viewer.center_lon, viewer.zoom + 1) + need_redraw = True + elif event.button == 5: + if viewer.zoom > 1: + viewer.set_center(viewer.center_lat, viewer.center_lon, viewer.zoom - 1) + need_redraw = True + + elif event.type == pygame.MOUSEBUTTONUP: + if event.button == 1: + dragging = False + + elif event.type == pygame.MOUSEMOTION: + if dragging and drag_start and drag_center_start: + dx = event.pos[0] - drag_start[0] + dy = event.pos[1] - drag_start[1] + + if viewer.bbox: + min_lon, min_lat, max_lon, max_lat = viewer.bbox + lon_per_pixel = (max_lon - min_lon) / VIEWPORT_SIZE + lat_per_pixel = (max_lat - min_lat) / VIEWPORT_SIZE + + new_lon = drag_center_start[1] - dx * lon_per_pixel + new_lat = drag_center_start[0] + dy * lat_per_pixel + + viewer.set_center(new_lat, new_lon) + need_redraw = True + + if need_redraw: + viewer.render_to_surface(viewport_surface) + screen.fill((50, 50, 50)) + screen.blit(viewport_surface, (0, 0)) + + # Toolbar + pygame.draw.rect(screen, (30, 30, 30), (VIEWPORT_SIZE, 0, TOOLBAR_WIDTH, VIEWPORT_SIZE)) + + button_bg = (50, 50, 50) + button_fg = (255, 255, 255) + button_border = (100, 100, 100) + + bg_text = "Background: White" if viewer.background_color == (255, 255, 255) else "Background: Black" + draw_button(screen, bg_text, bg_button_rect, button_bg, button_fg, button_border, font_small) + + fill_text = "Fill: ON" if viewer.fill_polygons else "Fill: OFF" + draw_button(screen, fill_text, fill_button_rect, button_bg, button_fg, button_border, font_small) + + grid_text = "Grid: ON" if viewer.show_tile_grid else "Grid: OFF" + draw_button(screen, grid_text, grid_button_rect, button_bg, button_fg, button_border, font_small) + + info_y = button_margin * 4 + button_height * 3 + 20 + info_color = (200, 200, 200) + + screen.blit(font_small.render(f"Lat: {viewer.center_lat:.6f}", True, info_color), (VIEWPORT_SIZE + 10, info_y)) + screen.blit(font_small.render(f"Lon: {viewer.center_lon:.6f}", True, info_color), (VIEWPORT_SIZE + 10, info_y + 18)) + screen.blit(font_small.render(f"Zoom: {viewer.zoom}", True, info_color), (VIEWPORT_SIZE + 10, info_y + 36)) + + # Query stats + stats_y = info_y + 70 + screen.blit(font_small.render("Query Stats:", True, info_color), (VIEWPORT_SIZE + 10, stats_y)) + if viewer.last_query_stats: + s = viewer.last_query_stats + screen.blit(font_small.render(f" Tiles: {s.get('tiles_loaded', 0)}/{s.get('tiles_loaded', 0) + s.get('tiles_missing', 0)}", True, (150, 150, 150)), (VIEWPORT_SIZE + 10, stats_y + 18)) + screen.blit(font_small.render(f" Features: {s.get('features', 0)}", True, (150, 150, 150)), (VIEWPORT_SIZE + 10, stats_y + 32)) + screen.blit(font_small.render(f" Time: {s.get('time_ms', 0):.0f}ms", True, (150, 150, 150)), (VIEWPORT_SIZE + 10, stats_y + 46)) + + # Selected feature + feature_y = stats_y + 80 + screen.blit(font_small.render("Click Feature:", True, info_color), (VIEWPORT_SIZE + 10, feature_y)) + if viewer.selected_feature: + line_y = feature_y + 18 + for key, value in viewer.selected_feature.items(): + text = f" {key}: {value}" + screen.blit(font_small.render(text, True, (100, 200, 100)), (VIEWPORT_SIZE + 10, line_y)) + line_y += 14 + else: + screen.blit(font_small.render(" (right-click)", True, (100, 100, 100)), (VIEWPORT_SIZE + 10, feature_y + 18)) + + # Status bar + pygame.draw.rect(screen, (30, 30, 30), (0, VIEWPORT_SIZE, WINDOW_WIDTH, STATUSBAR_HEIGHT)) + screen.blit(font_small.render("NAV Format - IceNav Navigation Tiles", True, (200, 200, 200)), (10, VIEWPORT_SIZE + 10)) + + if viewer.available_zooms: + zooms_str = f"Available: {min(viewer.available_zooms)}-{max(viewer.available_zooms)}" + screen.blit(font_small.render(zooms_str, True, (150, 150, 150)), (10, VIEWPORT_SIZE + 30)) + + pygame.display.flip() + need_redraw = False + + clock.tick(30) + + pygame.quit() + + +if __name__ == '__main__': + main() diff --git a/pbf_to_nav.py b/pbf_to_nav.py new file mode 100644 index 0000000..450c763 --- /dev/null +++ b/pbf_to_nav.py @@ -0,0 +1,739 @@ +#!/usr/bin/env python3 +""" +PBF to NAV Tile Converter + +Converts OpenStreetMap .pbf files to NAV binary format (.nav) with tile structure. +NAV format uses int32 coordinates (scaled by 1e7) for ~50% size reduction vs FlatGeobuf. + +Usage: + python pbf_to_nav.py input.pbf output_dir features.json [--zoom 6-17] + +Output structure: + output_dir/ + ├── 13/ + │ ├── 4123/ + │ │ ├── 2456.nav + │ │ └── ... + │ └── ... + └── 14/ + └── ... +""" + +import os +import sys +import json +import argparse +import logging +import math +import struct +from typing import Dict, List, Tuple, Set +from collections import defaultdict +import time + +try: + import osmium + from osmium import osm + import osmium.geom +except ImportError: + print("Error: osmium not found. Install with: pip install osmium") + sys.exit(1) + +try: + from shapely.geometry import Polygon + import shapely.wkb + SHAPELY_AVAILABLE = True +except ImportError: + SHAPELY_AVAILABLE = False + print("Warning: shapely not found. Multipolygon support disabled.") + +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +# NAV format constants +NAV_MAGIC = b'NAV1' +COORD_SCALE = 10000000 # 1e7 for ~1cm precision + +# Geometry types +GEOM_POINT = 1 +GEOM_LINESTRING = 2 +GEOM_POLYGON = 3 + +# Tags that support width (LineStrings only) +WIDTH_TAGS = {'highway', 'railway', 'waterway'} + + +def meters_to_pixels(width_meters: float, zoom: int, lat: float = 45.0) -> int: + """Convert width in meters to pixels at given zoom level. + + Uses approximation for given latitude (default 45° for Europe). + Formula: meters_per_pixel ≈ 156543 * cos(lat) / 2^zoom + """ + import math + meters_per_pixel = 156543.0 * math.cos(math.radians(lat)) / (2 ** zoom) + pixels = int(width_meters / meters_per_pixel + 0.5) + return max(1, min(15, pixels)) # Clamp to 1-15 + + +# Layer rendering priority (lower = rendered first = behind) +LAYER_PRIORITY = { + 'water': 10, + 'landuse': 20, + 'terrain': 30, + 'railways': 40, + 'roads': 50, + 'infrastructure': 60, + 'buildings': 70, + 'amenities': 80, + 'places': 90 +} + +# Layer definitions based on feature types +LAYER_MAPPING = { + 'water': [ + 'natural=water', 'natural=coastline', 'natural=bay', + 'waterway=riverbank', 'waterway=dock', 'waterway=boatyard', + 'waterway=river', 'waterway=stream', 'waterway=canal', + 'natural=spring', 'natural=wetland' + ], + 'landuse': [ + 'natural=beach', 'natural=sand', 'natural=wood', + 'landuse=forest', 'natural=forest', 'natural=scrub', + 'natural=heath', 'natural=grassland', 'landuse=meadow', + 'landuse=grass', 'landuse=orchard', 'landuse=vineyard', + 'landuse=farmland', 'landuse=park', 'leisure=park', + 'leisure=nature_reserve', 'leisure=garden', 'leisure=pitch', + 'leisure=golf_course', 'leisure=recreation_ground', 'landuse=recreation_ground', + 'landuse=residential', 'place=suburb', + 'landuse=commercial', 'landuse=retail', 'landuse=industrial', + 'landuse=construction', 'landuse=cemetery', 'landuse=allotments', + 'leisure=stadium', 'leisure=sports_centre', 'leisure=playground', + 'amenity=parking' + ], + 'roads': [ + 'highway=motorway', 'highway=motorway_link', + 'highway=trunk', 'highway=trunk_link', + 'highway=primary', 'highway=primary_link', + 'highway=secondary', 'highway=secondary_link', + 'highway=tertiary', 'highway=tertiary_link', + 'highway=residential', 'highway=living_street', + 'highway=unclassified', 'highway=service', + 'highway=pedestrian', 'highway=track', + 'highway=path', 'highway=footway', + 'highway=cycleway', 'highway=steps', + 'highway=crossing', 'highway=bus_stop' + ], + 'railways': [ + 'railway=rail', 'railway=subway', 'railway=tram' + ], + 'buildings': [ + 'building', 'man_made=tower' + ], + 'amenities': [ + 'amenity=hospital', + 'amenity=school', 'amenity=university', + 'amenity=place_of_worship' + ], + 'infrastructure': [ + 'bridge=yes', 'man_made=bridge', + 'aeroway=runway', 'aeroway=taxiway', 'aeroway=apron', + 'tunnel=yes' + ], + 'terrain': [ + 'natural=peak', 'natural=ridge', + 'natural=volcano', 'natural=cliff', + 'natural=tree_row', 'natural=tree' + ], + 'places': [ + 'place=state', 'place=town', + 'place=village', 'place=hamlet' + ] +} + + +def lon_to_tile_x(lon: float, zoom: int) -> int: + """Convert longitude to tile X coordinate.""" + n = 2.0 ** zoom + return int((lon + 180.0) / 360.0 * n) + + +def lat_to_tile_y(lat: float, zoom: int) -> int: + """Convert latitude to tile Y coordinate.""" + n = 2.0 ** zoom + lat_rad = math.radians(lat) + return int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n) + + +def get_feature_tiles(coords: List[Tuple[float, float]], zoom: int, is_polygon: bool = False) -> Set[Tuple[int, int]]: + """Get all tiles that a feature intersects at given zoom level.""" + tiles = set() + + if is_polygon and len(coords) >= 3: + lons = [c[0] for c in coords] + lats = [c[1] for c in coords] + min_lon, max_lon = min(lons), max(lons) + min_lat, max_lat = min(lats), max(lats) + + min_x = lon_to_tile_x(min_lon, zoom) + max_x = lon_to_tile_x(max_lon, zoom) + min_y = lat_to_tile_y(max_lat, zoom) + max_y = lat_to_tile_y(min_lat, zoom) + + for x in range(min_x, max_x + 1): + for y in range(min_y, max_y + 1): + tiles.add((x, y)) + else: + for lon, lat in coords: + x = lon_to_tile_x(lon, zoom) + y = lat_to_tile_y(lat, zoom) + tiles.add((x, y)) + + return tiles + + +def get_layer_for_tags(tags: Dict[str, str]) -> str: + """Determine which layer a feature belongs to based on its tags.""" + for layer_name, feature_keys in LAYER_MAPPING.items(): + for feature_key in feature_keys: + if '=' in feature_key: + key, value = feature_key.split('=', 1) + if key in tags and tags[key] == value: + return layer_name + else: + if feature_key in tags: + return layer_name + return None + + +def get_zoom_for_tags(tags: Dict[str, str], config: Dict) -> int: + """Get minimum zoom level for feature based on config.""" + for key, value in tags.items(): + feature_key = f"{key}={value}" + if feature_key in config and isinstance(config[feature_key], dict): + return config[feature_key].get('zoom', 6) + if key in config and isinstance(config[key], dict): + return config[key].get('zoom', 6) + return 6 + + +def get_color_for_tags(tags: Dict[str, str], config: Dict) -> str: + """Get color for feature based on config.""" + for key, value in tags.items(): + feature_key = f"{key}={value}" + if feature_key in config and isinstance(config[feature_key], dict): + return config[feature_key].get('color', '#FFFFFF') + if key in config and isinstance(config[key], dict): + return config[key].get('color', '#FFFFFF') + return '#FFFFFF' + + +def get_priority_for_tags(tags: Dict[str, str], config: Dict) -> int: + """Get rendering priority for feature based on config.""" + for key, value in tags.items(): + feature_key = f"{key}={value}" + if feature_key in config and isinstance(config[feature_key], dict): + return config[feature_key].get('priority', 50) + if key in config and isinstance(config[key], dict): + return config[key].get('priority', 50) + return 50 + + +def hex_to_rgb565(hex_color: str) -> int: + """Convert hex color to RGB565 format.""" + try: + if not hex_color or not hex_color.startswith("#"): + return 0xFFFF + r = int(hex_color[1:3], 16) + g = int(hex_color[3:5], 16) + b = int(hex_color[5:7], 16) + return ((r >> 3) << 11) | ((g >> 2) << 5) | (b >> 3) + except: + return 0xFFFF + + +def pack_zoom_priority(min_zoom: int, priority: int) -> int: + """Pack min_zoom and priority into a single byte.""" + zoom_nibble = min(min_zoom, 15) & 0x0F + priority_nibble = min(priority // 7, 15) & 0x0F + return (zoom_nibble << 4) | priority_nibble + + +def get_simplify_tolerance(zoom: int) -> float: + """Calculate simplification tolerance based on zoom level.""" + tile_width_degrees = 360.0 / (2.0 ** zoom) + pixel_size_degrees = tile_width_degrees / 256.0 + return pixel_size_degrees + + +class OSMHandler(osmium.SimpleHandler): + """Handler for processing OSM data from PBF files.""" + + def __init__(self, config: Dict, zoom_range: Tuple[int, int]): + super().__init__() + self.config = config + self.min_zoom, self.max_zoom = zoom_range + self.features: List[Dict] = [] + self.stats = { + 'ways_processed': 0, + 'areas_processed': 0, + 'features_extracted': 0, + 'features_filtered': 0 + } + self.start_time = time.time() + self.last_progress_time = time.time() + self.progress_interval = 5 + self.interesting_tags = self._build_interesting_tags() + self.processed_way_ids: Set[int] = set() + self.wkbfab = osmium.geom.WKBFactory() + + def _build_interesting_tags(self) -> Set[str]: + """Build set of tag keys we're interested in.""" + tags = set() + for key in self.config: + if isinstance(self.config[key], dict): + if '=' in key: + tag_key = key.split('=')[0] + tags.add(tag_key) + else: + tags.add(key) + return tags + + def _log_progress(self): + """Log progress periodically.""" + current_time = time.time() + if current_time - self.last_progress_time >= self.progress_interval: + self.last_progress_time = current_time + ways = self.stats['ways_processed'] + extracted = self.stats['features_extracted'] + elapsed = current_time - self.start_time + mins, secs = divmod(int(elapsed), 60) + print(f"\r Progress: {ways:,} ways, {extracted:,} features [{mins}m {secs:02d}s]", end='', flush=True) + + def _has_interesting_tags(self, tags) -> bool: + """Check if tags contain any interesting keys.""" + for tag in tags: + if tag.k in self.interesting_tags: + return True + return False + + def _tags_to_dict(self, tags) -> Dict[str, str]: + """Convert osmium tags to dictionary.""" + return {tag.k: tag.v for tag in tags} + + def _is_feature_in_config(self, tags: Dict[str, str]) -> bool: + """Check if feature matches any entry in config.""" + for key, value in tags.items(): + feature_key = f"{key}={value}" + if feature_key in self.config and isinstance(self.config[feature_key], dict): + return True + if key in self.config and isinstance(self.config[key], dict): + return True + return False + + def way(self, w): + """Process way - extract roads and linear features.""" + self.stats['ways_processed'] += 1 + self._log_progress() + + if not self._has_interesting_tags(w.tags): + self.stats['features_filtered'] += 1 + return + + tags = self._tags_to_dict(w.tags) + + if not self._is_feature_in_config(tags): + self.stats['features_filtered'] += 1 + return + + layer = get_layer_for_tags(tags) + if layer is None: + self.stats['features_filtered'] += 1 + return + + min_zoom = get_zoom_for_tags(tags, self.config) + if min_zoom > self.max_zoom: + self.stats['features_filtered'] += 1 + return + + coords = [] + for node in w.nodes: + if node.location.valid(): + coords.append((node.location.lon, node.location.lat)) + + if len(coords) < 2: + self.stats['features_filtered'] += 1 + return + + is_closed = len(coords) >= 4 and coords[0] == coords[-1] + is_area_tags = ( + 'building' in tags or + 'landuse' in tags or + ('natural' in tags and tags.get('natural') in ['water', 'wood', 'forest', 'beach', 'sand', 'wetland', 'grassland', 'scrub', 'heath']) or + ('leisure' in tags and tags.get('leisure') in ['park', 'garden', 'pitch', 'golf_course', 'nature_reserve', 'playground', 'sports_centre', 'stadium', 'common']) or + ('amenity' in tags and tags.get('amenity') in ['parking', 'school', 'university', 'hospital', 'marketplace']) or + ('waterway' in tags and tags.get('waterway') in ['riverbank', 'dock', 'boatyard']) or + tags.get('area') == 'yes' + ) + + color = get_color_for_tags(tags, self.config) + priority = get_priority_for_tags(tags, self.config) + color_rgb565 = hex_to_rgb565(color) + layer_base_priority = LAYER_PRIORITY.get(layer, 50) + combined_priority = layer_base_priority + (priority % 10) + + if is_closed and is_area_tags and 'highway' not in tags: + feature = { + 'geom_type': GEOM_POLYGON, + 'coords': coords, + 'color_rgb565': color_rgb565, + 'zoom_priority': pack_zoom_priority(min_zoom, combined_priority), + 'width_meters': 0.0 # Polygons don't use width + } + self.features.append(feature) + self.stats['features_extracted'] += 1 + self.processed_way_ids.add(w.id) + return + + # Extract width in meters for roads/railways/waterways + width_meters = 0.0 + if any(tag in tags for tag in WIDTH_TAGS): + width_meters = self._get_width_meters(tags) + + feature = { + 'geom_type': GEOM_LINESTRING, + 'coords': coords, + 'color_rgb565': color_rgb565, + 'zoom_priority': pack_zoom_priority(min_zoom, combined_priority), + 'width_meters': width_meters + } + self.features.append(feature) + self.stats['features_extracted'] += 1 + + def _get_width_meters(self, tags: Dict[str, str]) -> float: + """Extract width in meters from OSM tags. + + Priority: + 1. width=* tag (meters) + 2. lanes=* tag (lanes × 3.5m) + 3. Return 0 (will become 1 pixel default) + """ + # Check for explicit width tag + if 'width' in tags: + try: + width_str = tags['width'].replace('m', '').replace(' ', '').strip() + return float(width_str) + except (ValueError, TypeError): + pass + + # Check for lanes tag + if 'lanes' in tags: + try: + lanes = int(tags['lanes']) + return lanes * 3.5 # Standard lane width + except (ValueError, TypeError): + pass + + return 0.0 + + def area(self, a): + """Process area - handles multipolygon relations.""" + self.stats['areas_processed'] += 1 + self._log_progress() + + if not self._has_interesting_tags(a.tags): + self.stats['features_filtered'] += 1 + return + + tags = self._tags_to_dict(a.tags) + + if not self._is_feature_in_config(tags): + self.stats['features_filtered'] += 1 + return + + layer = get_layer_for_tags(tags) + if layer is None: + self.stats['features_filtered'] += 1 + return + + if 'highway' in tags: + self.stats['features_filtered'] += 1 + return + + min_zoom = get_zoom_for_tags(tags, self.config) + if min_zoom > self.max_zoom: + self.stats['features_filtered'] += 1 + return + + if a.from_way() and a.orig_id() in self.processed_way_ids: + return + + try: + wkb = self.wkbfab.create_multipolygon(a) + geom = shapely.wkb.loads(wkb, hex=True) + + color = get_color_for_tags(tags, self.config) + priority = get_priority_for_tags(tags, self.config) + color_rgb565 = hex_to_rgb565(color) + layer_base_priority = LAYER_PRIORITY.get(layer, 50) + combined_priority = layer_base_priority + (priority % 10) + + polygons = [] + if geom.geom_type == 'Polygon': + polygons = [geom] + elif geom.geom_type == 'MultiPolygon': + polygons = list(geom.geoms) + + for poly in polygons: + if poly.is_empty or not poly.exterior: + continue + + coords = list(poly.exterior.coords) + if len(coords) < 4: + continue + + feature = { + 'geom_type': GEOM_POLYGON, + 'coords': coords, + 'color_rgb565': color_rgb565, + 'zoom_priority': pack_zoom_priority(min_zoom, combined_priority), + 'width_meters': 0.0 # Polygons don't use width + } + self.features.append(feature) + self.stats['features_extracted'] += 1 + + except Exception as e: + self.stats['features_filtered'] += 1 + + +def simplify_coords(coords: List[Tuple[float, float]], tolerance: float) -> List[Tuple[float, float]]: + """Simple Douglas-Peucker-like simplification.""" + if len(coords) <= 2: + return coords + + # Use shapely for simplification if available + if SHAPELY_AVAILABLE: + from shapely.geometry import LineString + line = LineString(coords) + simplified = line.simplify(tolerance, preserve_topology=True) + return list(simplified.coords) + + return coords + + +def write_nav_tile(features: List[Dict], output_path: str, zoom: int) -> bool: + """Write features to NAV binary tile format.""" + if not features: + return False + + tolerance = get_simplify_tolerance(zoom) + + # Calculate bounding box + all_lons = [] + all_lats = [] + for f in features: + for lon, lat in f['coords']: + all_lons.append(lon) + all_lats.append(lat) + + if not all_lons: + return False + + min_lon = min(all_lons) + min_lat = min(all_lats) + max_lon = max(all_lons) + max_lat = max(all_lats) + + # Create output directory + os.makedirs(os.path.dirname(output_path), exist_ok=True) + + with open(output_path, 'wb') as f: + # Write header (22 bytes) + f.write(NAV_MAGIC) # 4 bytes + f.write(struct.pack(' 2: + coords = simplify_coords(coords, tolerance) + + # Skip if too few coordinates after simplification + if feature['geom_type'] == GEOM_LINESTRING and len(coords) < 2: + continue + if feature['geom_type'] == GEOM_POLYGON and len(coords) < 4: + continue + + # Calculate width in pixels (convert meters to pixels at this zoom) + width_meters = feature.get('width_meters', 0.0) + if width_meters > 0: + width_pixels = meters_to_pixels(width_meters, zoom) + else: + width_pixels = 1 # Default width + + # Write feature header + f.write(struct.pack('> 4 + if min_zoom > zoom: + continue + + is_polygon = feature['geom_type'] == GEOM_POLYGON + tiles = get_feature_tiles(feature['coords'], zoom, is_polygon) + + for tile in tiles: + tile_features[tile].append(feature) + + if not tile_features: + continue + + num_tiles = len(tile_features) + tiles_written = 0 + tile_items = list(tile_features.items()) + + for i, ((x, y), features) in enumerate(tile_items): + progress = (i + 1) / num_tiles + bar_width = 30 + filled = int(bar_width * progress) + bar = '█' * filled + '░' * (bar_width - filled) + print(f"\r Zoom {zoom:2d}: [{bar}] {i+1}/{num_tiles} tiles", end='', flush=True) + + tile_dir = os.path.join(output_dir, str(zoom), str(x)) + tile_path = os.path.join(tile_dir, f"{y}.nav") + + # Pre-sort by priority (low nibble) for streaming render on ESP32 + features.sort(key=lambda f: f['zoom_priority'] & 0x0F) + + if write_nav_tile(features, tile_path, zoom): + tiles_written += 1 + total_size += os.path.getsize(tile_path) + + print(f"\r Zoom {zoom:2d}: {tiles_written} tiles written" + " " * 30) + total_tiles += tiles_written + + total_time = time.time() - start_time + hours, remainder = divmod(int(total_time), 3600) + minutes, seconds = divmod(remainder, 60) + if hours > 0: + time_str = f"{hours}h {minutes:02d}m {seconds:02d}s" + elif minutes > 0: + time_str = f"{minutes}m {seconds:02d}s" + else: + time_str = f"{total_time:.2f}s" + + logger.info("=" * 50) + logger.info("Conversion Summary") + logger.info("=" * 50) + logger.info(f"Input: {input_pbf}") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Format: NAV binary tiles (.nav)") + logger.info(f"Total tiles: {total_tiles}") + logger.info(f"Total size: {total_size / (1024 * 1024):.2f} MB") + logger.info(f"Total time: {time_str}") + logger.info("=" * 50) + + return total_tiles + + +def main(): + parser = argparse.ArgumentParser( + description='Convert OpenStreetMap PBF to NAV binary tile format', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +NAV Format - IceNav Navigation Tiles: + - int32 coordinates (scaled by 1e7) for ~50% size reduction + - Simple binary format optimized for ESP32 + - Same z/x/y tile structure as standard map tiles + +Examples: + python pbf_to_nav.py andorra.pbf ./output features.json + python pbf_to_nav.py spain.pbf ./output features.json --zoom 10-17 + """ + ) + + parser.add_argument('input_pbf', help='Input PBF file path') + parser.add_argument('output_dir', help='Output directory for NAV tiles') + parser.add_argument('config_file', help='Features configuration JSON file') + parser.add_argument('--zoom', default='6-17', + help='Zoom level range (e.g., "6-17" or "12")') + + args = parser.parse_args() + + if not os.path.exists(args.input_pbf): + logger.error(f"Input file not found: {args.input_pbf}") + sys.exit(1) + + if not os.path.exists(args.config_file): + logger.error(f"Config file not found: {args.config_file}") + sys.exit(1) + + if '-' in args.zoom: + min_zoom, max_zoom = map(int, args.zoom.split('-')) + else: + min_zoom = max_zoom = int(args.zoom) + + convert_pbf_to_nav(args.input_pbf, args.output_dir, args.config_file, (min_zoom, max_zoom)) + + +if __name__ == '__main__': + main() diff --git a/tile_generator.py b/tile_generator.py deleted file mode 100644 index 40103cf..0000000 --- a/tile_generator.py +++ /dev/null @@ -1,1323 +0,0 @@ -#!/usr/bin/env python3 - -import math -import struct -import json -import os -import gc -import subprocess -import ijson -import time -import re -from decimal import Decimal -import argparse -import sys -import logging -from typing import List, Dict, Tuple, Optional -from concurrent.futures import ThreadPoolExecutor, as_completed - -logging.basicConfig(level=logging.INFO, format='%(message)s') -logger = logging.getLogger(__name__) - -# Constants -TILE_SIZE = 256 -UINT16_TILE_SIZE = 65536 - -DRAW_COMMANDS = { - 'POLYLINE': 2, - 'STROKE_POLYGON': 3, - 'SET_COLOR': 0x80, - 'SET_COLOR_INDEX': 0x81, -} - -# Global color palette -GLOBAL_COLOR_PALETTE = {} -GLOBAL_INDEX_TO_RGB332 = {} - -# --- CARTO-LIKE STYLE DEFINITIONS (Pixels per Zoom) --- -# OPTIMIZED FOR SMALL SCREENS: Thinner lines at low zoom, reduced max width at high zoom. -CARTO_STYLES = { - # Major roads: Visible but not overwhelming at low zoom - 'motorway': {6: 1, 10: 2, 13: 3, 15: 6, 17: 10, 18: 16}, - 'trunk': {6: 1, 10: 2, 13: 3, 15: 5, 17: 9, 18: 14}, - 'primary': {8: 1, 11: 2, 13: 3, 15: 5, 17: 9, 18: 14}, - - # Connecting roads: Keep thin until zoomed in - 'secondary': {11: 1, 13: 2, 15: 4, 17: 8, 18: 12}, - 'tertiary': {12: 1, 14: 2, 16: 5, 18: 10}, - - # Minor roads: Hairline (1px) until very close - 'residential': {13: 0.5, 15: 1, 16: 3, 18: 8}, - 'unclassified': {13: 0.5, 15: 1, 16: 3, 18: 8}, - 'service': {15: 1, 17: 3, 18: 6}, - 'track': {14: 0.5, 16: 1, 18: 4}, - - # Paths: Very subtle - 'footway': {15: 0.5, 17: 1, 19: 3}, - 'path': {15: 0.5, 17: 1, 19: 3}, - 'cycleway': {15: 0.5, 17: 1, 19: 3}, - - # Transport - 'railway': {10: 1, 14: 2, 16: 3, 18: 5}, - 'subway': {12: 1, 14: 2, 16: 3}, - - # Waterways: Drastically reduced to prevent blocking features - 'river': {8: 1, 12: 2, 14: 4, 16: 7, 18: 10}, - 'canal': {10: 1, 13: 2, 15: 4, 17: 6}, - 'stream': {14: 1, 16: 2, 18: 3}, - 'drain': {15: 1, 18: 2}, - - # Aero: Reduced so they don't cover the airport terminal buildings - 'runway': {10: 1, 12: 3, 14: 8, 16: 16, 18: 24}, - 'taxiway': {13: 1, 15: 4, 17: 8}, -} - - -def interpolate_width(styles: Dict[int, int], zoom: int) -> float: - """Linearly interpolate width between defined zoom levels.""" - zooms = sorted(styles.keys()) - - if not zooms: - return 1.0 - - if zoom <= zooms[0]: - return styles[zooms[0]] - if zoom >= zooms[-1]: - return styles[zooms[-1]] - - # Find the interval [z1, z2] containing zoom - for i in range(len(zooms) - 1): - z1, z2 = zooms[i], zooms[i+1] - if z1 <= zoom <= z2: - w1, w2 = styles[z1], styles[z2] - # Linear interpolation - return w1 + (w2 - w1) * (zoom - z1) / (z2 - z1) - - return 1.0 - - -def get_available_memory_mb() -> int: - """Detect available system memory in MB.""" - try: - if os.name == 'posix': - with open('/proc/meminfo', 'r') as f: - for line in f: - if 'MemAvailable' in line: - return int(line.split()[1]) // 1024 - elif os.name == 'nt': - import ctypes - kernel32 = ctypes.windll.kernel32 - kernel32.GetPhysicallyInstalledSystemMemory.argtypes = [ctypes.POINTER(ctypes.c_ulonglong)] - memory = ctypes.c_ulonglong() - if kernel32.GetPhysicallyInstalledSystemMemory(ctypes.byref(memory)): - return memory.value // (1024 * 1024) - except: - pass - return 4096 - - -def get_optimal_workers(cpu_count: int, available_memory_mb: int) -> int: - """Calculate optimal number of worker threads based on system resources.""" - memory_workers = max(1, min(available_memory_mb // 1024, 6)) - cpu_workers = max(1, cpu_count - 1) - - if available_memory_mb <= 8192: - max_workers = min(3, memory_workers, cpu_workers) - else: - max_workers = min(6, memory_workers, cpu_workers) - - logger.info(f"System: {cpu_count} CPU cores, {available_memory_mb}MB RAM -> Using {max_workers} workers") - return max_workers - - -def get_adaptive_batch_size(zoom: int, base_batch_size: int, available_memory_mb: int) -> int: - """Calculate adaptive batch size based on zoom level and available memory.""" - memory_factor = min(2.0, available_memory_mb / 4096) - - if zoom < 10: - adjusted_size = max(2000, int(base_batch_size * 0.6 * memory_factor)) - elif zoom < 13: - adjusted_size = max(1000, int(base_batch_size * 0.4 * memory_factor)) - elif zoom < 15: - adjusted_size = max(500, int(base_batch_size * 0.3 * memory_factor)) - else: - adjusted_size = max(250, int(base_batch_size * 0.2 * memory_factor)) - - return adjusted_size - - -def precompute_global_color_palette(config: Dict) -> int: - """Build global color palette from configuration.""" - global GLOBAL_COLOR_PALETTE, GLOBAL_INDEX_TO_RGB332 - logger.info("Building color palette...") - - unique_colors = set() - for feature_config in config.values(): - if isinstance(feature_config, dict) and 'color' in feature_config: - hex_color = feature_config['color'] - if hex_color and isinstance(hex_color, str) and hex_color.startswith("#"): - unique_colors.add(hex_color) - - sorted_colors = sorted(list(unique_colors)) - GLOBAL_COLOR_PALETTE.clear() - GLOBAL_INDEX_TO_RGB332.clear() - - for index, hex_color in enumerate(sorted_colors): - rgb332_value = hex_to_rgb332(hex_color) - GLOBAL_COLOR_PALETTE[hex_color] = index - GLOBAL_INDEX_TO_RGB332[index] = rgb332_value - - logger.info(f"Palette: {len(unique_colors)} colors") - return len(unique_colors) - - -def write_palette_bin(output_dir: str): - """Write color palette to binary file.""" - os.makedirs(output_dir, exist_ok=True) - palette_path = os.path.join(output_dir, "palette.bin") - num_colors = len(GLOBAL_INDEX_TO_RGB332) - - palette_data = bytearray() - palette_data.extend(struct.pack('> 5) & 0x07 - # Expansión estándar de 3 a 8 bits - r = (red_3bit << 5) | (red_3bit << 2) | (red_3bit >> 1) - - # Extrae el valor de 3 bits de G (bits 4, 3, 2) y lo expande a 8 bits (G888) - green_3bit = (rgb332 >> 2) & 0x07 - # Expansión estándar de 3 a 8 bits - g = (green_3bit << 5) | (green_3bit << 2) | (green_3bit >> 1) - - # Extrae el valor de 2 bits de B (bits 1, 0) y lo expande a 8 bits (B888) - blue_2bit = rgb332 & 0x03 - # Expansión estándar de 2 a 8 bits - b = (blue_2bit << 6) | (blue_2bit << 4) | (blue_2bit << 2) | blue_2bit - - palette_data.extend([r, g, b]) - else: - palette_data.extend([255, 255, 255]) - - with open(palette_path, 'wb') as f: - f.write(bytes(palette_data)) - - logger.info("Palette written") - - -def hex_to_color_index(hex_color: str) -> Optional[int]: - """Convert hex color to palette index.""" - return GLOBAL_COLOR_PALETTE.get(hex_color) - - -def hex_to_rgb332(hex_color: str) -> int: - """Convert hex color (RGB888) to RGB332 format (8-bit index).""" - try: - if not hex_color or not hex_color.startswith("#"): - return 0xFF - r = int(hex_color[1:3], 16) - g = int(hex_color[3:5], 16) - b = int(hex_color[5:7], 16) - - # RRR (3 bits), GGG (3 bits), YY (2 bits para B) - return ((r & 0xE0) | ((g & 0xE0) >> 3) | (b >> 6)) - except: - return 0xFF - - -def parse_width_value(width_str: str) -> Optional[float]: - """ - Parse width value from OSM tags, handling various formats. - Returns width in meters, or None if parsing fails. - """ - if not width_str: - return None - - width_str = str(width_str).lower().strip().replace(',', '.') - - pattern = r'([0-9]+\.?[0-9]*)\s*(m|meter|meters|metre|metres|ft|feet|foot|\'|"|in|inch|inches)?' - match = re.match(pattern, width_str) - - if not match: - return None - - try: - value = float(match.group(1)) - unit = match.group(2) if match.group(2) else 'm' - - if unit in ['ft', 'feet', 'foot', '\'']: - value = value * 0.3048 - elif unit in ['"', 'in', 'inch', 'inches']: - value = value * 0.0254 - - return value - except (ValueError, AttributeError): - return None - - -def meters_to_pixels_physical(meters: float, zoom: int, lat: float) -> float: - """Convertir metros físicos a píxeles sin escalado agresivo (factor de escala eliminado).""" - base_resolution = 156543.03392 - lat_rad = math.radians(lat) - resolution = base_resolution * math.cos(lat_rad) / (2 ** zoom) - pixels = meters / resolution - return pixels - - -def apply_width_constraints(pixels: float, feature_type: str, zoom: int) -> int: - """Apply STRICT min/max pixel constraints based on feature type for small screens.""" - - # Constraints (min_pixels, max_pixels) - # MAX values reduced by ~40-50% from original standard - constraints = { - 'motorway': (1, 18), 'trunk': (1, 16), 'primary': (1, 16), - 'secondary': (1, 12), 'tertiary': (1, 10), - 'residential': (0.5, 10), 'unclassified': (0.5, 10), - 'service': (0.5, 6), 'track': (0.5, 5), - 'footway': (0.5, 3), 'path': (0.5, 3), 'cycleway': (0.5, 3), - - 'railway': (1, 6), - - # Waterways clamped tightly - 'river': (1, 10), - 'waterway': (0.5, 8), - - 'runway': (1, 24), 'taxiway': (1, 10), - 'pipeline': (0.5, 4), 'power': (0.5, 2), - } - - min_width, max_width = constraints.get(feature_type, (0.5, 8)) - - # Low zoom overrides to ensure clean rendering - if zoom <= 12: - max_width = min(max_width, 4) # Nothing should be thicker than 4px at zoom 12 - - if zoom <= 10: - max_width = min(max_width, 3) # Nothing thicker than 3px at zoom 10 - - clamped = max(min_width, min(max_width, pixels)) - - # Round to nearest integer but ensure at least 1px if calculation > 0.5 - if clamped < 1.0 and clamped >= 0.5: - return 1 - return max(1, int(round(clamped))) - - -def get_feature_category(tags: Dict) -> str: - """Determine the feature category for styling lookups.""" - if 'highway' in tags: return tags['highway'] - elif 'railway' in tags: - r = tags['railway'] - return r if r in ['rail', 'subway', 'tram', 'light_rail'] else 'railway' - elif 'waterway' in tags: return tags['waterway'] - elif 'aeroway' in tags: return tags['aeroway'] - elif 'power' in tags: return 'power' - elif 'man_made' in tags and tags['man_made'] in ['pipeline', 'embankment']: return 'pipeline' - return 'default' - - -def get_line_width_from_tags(tags: Dict, zoom: int, coordinates, geom_type: str) -> int: - """ - Calculate line width in pixels based on: - 1. Explicit OSM tags (width, maxwidth) -> Physical Calculation - 2. If no tags -> CartoCSS-style Zoom-based Defaults - """ - if not tags: - return 1 - - # Polygons usually don't have stroke width unless specified - if geom_type in ["Polygon", "MultiPolygon"]: - if not any(k in tags for k in ['width', 'maxwidth', 'est_width']): - return 1 - - feature_category = get_feature_category(tags) - calculated_pixels = 0.0 - width_found = False - - # 1. Try explicit physical width tags - width_tags = ['width', 'maxwidth', 'est_width', 'diameter', 'gauge'] - for width_tag in width_tags: - if width_tag in tags: - width_meters = parse_width_value(tags[width_tag]) - if width_meters is not None and width_meters > 0: - # Calculate average latitude for projection - avg_lat = 0.0 - try: - if geom_type in ["LineString", "MultiLineString"]: - coords = coordinates if geom_type == "LineString" else (coordinates[0] if coordinates else []) - valid_coords = [c for c in coords if len(c) >= 2] - if valid_coords: - avg_lat = sum(c[1] for c in valid_coords) / len(valid_coords) - elif geom_type in ["Polygon", "MultiPolygon"]: - # Use first point of exterior ring - c = coordinates[0] if geom_type == "Polygon" else coordinates[0][0] - if c and len(c) > 0: avg_lat = c[0][1] - except: - avg_lat = 0.0 - - calculated_pixels = meters_to_pixels_physical(width_meters, zoom, avg_lat) - width_found = True - break - - # 2. If no explicit tag, use Carto-style defaults (Pixels based on Zoom) - if not width_found: - if feature_category in CARTO_STYLES: - calculated_pixels = interpolate_width(CARTO_STYLES[feature_category], zoom) - else: - # Fallback for unknown features - calculated_pixels = 1.5 if zoom > 14 else 1.0 - - # 3. Apply Clamping (Min/Max constraints) regardless of source - # This ensures a 500m wide river doesn't cover the screen (clamped tightly) - final_width = apply_width_constraints(calculated_pixels, feature_category, zoom) - - return final_width - - -def interpolate_curve_points(points: List, max_segment_distance: float = 0.00005) -> List: - """Interpolate additional points to smooth curves - works on geographic coordinates.""" - if len(points) < 2: - return points - - result = [points[0]] - - for i in range(len(points) - 1): - p1 = points[i] - p2 = points[i + 1] - - dx = p2[0] - p1[0] - dy = p2[1] - p1[1] - distance = math.sqrt(dx**2 + dy**2) - - # Calculate number of segments needed - if distance > max_segment_distance: - num_segments = int(math.ceil(distance / max_segment_distance)) - # Add intermediate points - for j in range(1, num_segments): - t = j / num_segments - result.append([ - p1[0] + dx * t, - p1[1] + dy * t - ]) - - result.append(p2) - - return result - - -def smooth_geometry_coords(coordinates, geom_type: str, zoom: int, tags: Dict = None): - """Smooth geometry by adding intermediate points - applies to geographic coordinates.""" - # Iniciar suavizado a partir de zoom 16 - if zoom < 16: - return coordinates - - # Determine maximum segment distance based on zoom - is_rbt = tags and tags.get('junction') == 'roundabout' - - if zoom >= 19: - max_dist = 0.000008 if is_rbt else 0.00001 - elif zoom >= 18: - max_dist = 0.00001 if is_rbt else 0.000015 - elif zoom >= 17: - max_dist = 0.000015 if is_rbt else 0.00002 - else: # zoom 16 - max_dist = 0.00002 if is_rbt else 0.00003 - - def smooth_linestring(coords): - if len(coords) < 2: - return coords - return interpolate_curve_points(coords, max_dist) - - def smooth_ring(ring): - if not ring or len(ring) < 3: - return ring - # For closed rings, remove last point, interpolate, then re-close - is_closed = ring[0] == ring[-1] - work_ring = ring[:-1] if is_closed else ring - smoothed = interpolate_curve_points(work_ring, max_dist) - if is_closed and smoothed: - smoothed.append(smoothed[0]) - return smoothed - - if geom_type == "LineString": - return smooth_linestring(coordinates) - elif geom_type == "Polygon": - if not coordinates or not coordinates[0]: - return coordinates - return [smooth_ring(coordinates[0])] - elif geom_type == "MultiLineString": - return [smooth_linestring(line) for line in coordinates] - elif geom_type == "MultiPolygon": - result = [] - for poly in coordinates: - if poly and poly[0]: - result.append([smooth_ring(poly[0])]) - return result - - return coordinates - - -def geojson_bounds(coordinates, geom_type: str) -> Tuple[float, float, float, float]: - """Calculate bounding box for geometry coordinates.""" - def flatten_coords(coords): - if not coords: - return [] - if isinstance(coords, (int, float, Decimal)): - return [float(coords)] - if len(coords) == 2: - try: - lon = float(coords[0]) if not isinstance(coords[0], (int, float)) else coords[0] - lat = float(coords[1]) if not isinstance(coords[1], (int, float)) else coords[1] - if isinstance(lon, (int, float)) and isinstance(lat, (int, float)): - return [[lon, lat]] - except (TypeError, ValueError, IndexError): - pass - result = [] - for item in coords: - result.extend(flatten_coords(item)) - return result - - flat = flatten_coords(coordinates) - if not flat: - return (0, 0, 0, 0) - - lons = [c[0] for c in flat] - lats = [c[1] for c in flat] - return (min(lons), min(lats), max(lons), max(lats)) - - -def coords_intersect_tile(coordinates, geom_type: str, tile_bbox: Tuple[float, float, float, float]) -> bool: - """Check if geometry intersects with tile bounding box.""" - minx, miny, maxx, maxy = geojson_bounds(coordinates, geom_type) - tile_minx, tile_miny, tile_maxx, tile_maxy = tile_bbox - - if maxx < tile_minx or minx > tile_maxx or maxy < tile_miny or miny > tile_maxy: - return False - return True - - -def normalize_coordinates_robust(coordinates): - """Normalize coordinates to ensure all values are floats.""" - if isinstance(coordinates, (int, float, Decimal)): - return float(coordinates) - - if isinstance(coordinates, (list, tuple)): - if len(coordinates) == 2: - first, second = coordinates[0], coordinates[1] - if (isinstance(first, (int, float, Decimal)) and - isinstance(second, (int, float, Decimal))): - return [float(first), float(second)] - - return [normalize_coordinates_robust(item) for item in coordinates] - - return coordinates - - -def clip_polygon_sutherland_hodgman(vertices, minx, miny, maxx, maxy): - """Clip polygon to bounding box using Sutherland-Hodgman algorithm.""" - if not vertices or len(vertices) < 3: - return [] - - def inside(p, edge): - if edge == 'left': - return p[0] >= minx - elif edge == 'right': - return p[0] <= maxx - elif edge == 'bottom': - return p[1] >= miny - elif edge == 'top': - return p[1] <= maxy - return False - - def compute_intersection(p1, p2, edge): - x1, y1 = float(p1[0]), float(p1[1]) - x2, y2 = float(p2[0]), float(p2[1]) - - # Evitar división por cero - if edge == 'left': - x = minx - y = y1 + (y2 - y1) * (minx - x1) / (x2 - x1) if abs(x2 - x1) > 1e-9 else y1 - elif edge == 'right': - x = maxx - y = y1 + (y2 - y1) * (maxx - x1) / (x2 - x1) if abs(x2 - x1) > 1e-9 else y1 - elif edge == 'bottom': - y = miny - x = x1 + (x2 - x1) * (miny - y1) / (y2 - y1) if abs(y2 - y1) > 1e-9 else x1 - elif edge == 'top': - y = maxy - x = x1 + (x2 - x1) * (maxy - y1) / (y2 - y1) if abs(y2 - y1) > 1e-9 else x1 - return [x, y] - - output = list(vertices) - for edge in ['left', 'right', 'bottom', 'top']: - if not output: - break - input_list = output - output = [] - if not input_list: - continue - prev_vertex = input_list[-1] - for curr_vertex in input_list: - curr_inside = inside(curr_vertex, edge) - prev_inside = inside(prev_vertex, edge) - if curr_inside: - if not prev_inside: - intersection = compute_intersection(prev_vertex, curr_vertex, edge) - output.append(intersection) - output.append(curr_vertex) - elif prev_inside: - intersection = compute_intersection(prev_vertex, curr_vertex, edge) - output.append(intersection) - prev_vertex = curr_vertex - - if output and len(output) >= 3: - if output[0] != output[-1]: - output.append(output[0]) - - return output if len(output) >= 4 else [] - - -def clip_line_to_bbox(p1, p2, bbox): - """Clip line segment to bounding box using Cohen-Sutherland algorithm.""" - minx, miny, maxx, maxy = bbox - x1, y1 = float(p1[0]), float(p1[1]) - x2, y2 = float(p2[0]), float(p2[1]) - - def compute_outcode(x, y): - code = 0 - if x < minx: - code |= 1 - if x > maxx: - code |= 2 - if y < miny: - code |= 4 - if y > maxy: - code |= 8 - return code - - outcode1 = compute_outcode(x1, y1) - outcode2 = compute_outcode(x2, y2) - - while True: - if outcode1 == 0 and outcode2 == 0: - return ([x1, y1], [x2, y2]) - if (outcode1 & outcode2) != 0: - return None - - outcode = outcode1 if outcode1 != 0 else outcode2 - - # Evitar división por cero - if outcode & 8: - x = x1 + (x2 - x1) * (maxy - y1) / (y2 - y1) if abs(y2 - y1) > 1e-9 else x1 - y = maxy - elif outcode & 4: - x = x1 + (x2 - x1) * (miny - y1) / (y2 - y1) if abs(y2 - y1) > 1e-9 else x1 - y = miny - elif outcode & 2: - y = y1 + (y2 - y1) * (maxx - x1) / (x2 - x1) if abs(x2 - x1) > 1e-9 else y1 - x = maxx - elif outcode & 1: - y = y1 + (y2 - y1) * (minx - x1) / (x2 - x1) if abs(x2 - x1) > 1e-9 else y1 - x = minx - - if outcode == outcode1: - x1, y1 = x, y - outcode1 = compute_outcode(x1, y1) - else: - x2, y2 = x, y - outcode2 = compute_outcode(x2, y2) - - -def clip_coordinates_to_tile(coordinates, geom_type: str, tile_bbox: Tuple[float, float, float, float]): - """Clip geometry coordinates to tile bounding box.""" - minx, miny, maxx, maxy = tile_bbox - - def clip_linestring(coords): - if not coords or len(coords) < 2: - return [] - clipped = [] - # TOLERANCIA AUMENTADA A 1e-5 para corregir los problemas de continuidad. - TOLERANCE = 1e-5 - for i in range(len(coords) - 1): - result = clip_line_to_bbox(coords[i], coords[i + 1], tile_bbox) - if result: - p1, p2 = result - - # REVISIÓN DE CLIPPING: Usar distancia para verificar duplicados en el borde del tile. - is_duplicate = False - if clipped: - # Compara la distancia entre el último punto añadido (p2 del segmento anterior) y p1 del segmento actual - dist = math.hypot(clipped[-1][0] - p1[0], clipped[-1][1] - p1[1]) - if dist < TOLERANCE: - is_duplicate = True - - if not is_duplicate: - clipped.append(p1) - - clipped.append(p2) - return clipped if len(clipped) >= 2 else [] - - if geom_type == "Point": - lon, lat = float(coordinates[0]), float(coordinates[1]) - if minx <= lon <= maxx and miny <= lat <= maxy: - return coordinates - return None - elif geom_type == "LineString": - clipped = clip_linestring(coordinates) - return clipped if clipped else None - elif geom_type == "Polygon": - if not coordinates or not coordinates[0]: - return None - exterior = coordinates[0] - clipped_exterior = clip_polygon_sutherland_hodgman(exterior, minx, miny, maxx, maxy) - if not clipped_exterior or len(clipped_exterior) < 4: - return None - return [clipped_exterior] - elif geom_type == "MultiLineString": - clipped_lines = [] - for line in coordinates: - clipped = clip_linestring(line) - if clipped: - clipped_lines.append(clipped) - return clipped_lines if clipped_lines else None - elif geom_type == "MultiPolygon": - clipped_polys = [] - for poly in coordinates: - if poly and poly[0]: - clipped_exterior = clip_polygon_sutherland_hodgman(poly[0], minx, miny, maxx, maxy) - if clipped_exterior and len(clipped_exterior) >= 4: - clipped_polys.append([clipped_exterior]) - return clipped_polys if clipped_polys else None - return None - - -def deg2num(lat_deg: float, lon_deg: float, zoom: int) -> Tuple[int, int]: - """Convert lat/lon to tile numbers.""" - lat_rad = math.radians(lat_deg) - n = 2.0 ** zoom - xtile = int((lon_deg + 180.0) / 360.0 * n) - ytile = int((1.0 - math.log(math.tan(lat_rad) + 1 / math.cos(lat_rad)) / math.pi) / 2.0 * n) - return xtile, ytile - - -def tile_bbox(tile_x: int, tile_y: int, zoom: int) -> Tuple[float, float, float, float]: - """Calculate tile bounding box in lat/lon coordinates.""" - n = 2.0 ** zoom - lon_min = tile_x / n * 360.0 - 180.0 - lon_max = (tile_x + 1) / n * 360.0 - 180.0 - lat_max = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * tile_y / n)))) - lat_min = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (tile_y + 1) / n)))) - return (lon_min, lat_min, lon_max, lat_max) - - -def coords_to_pixels(coordinates, zoom: int, tile_x: int, tile_y: int) -> List[Tuple[int, int]]: - """Convert lat/lon coordinates to pixel coordinates within tile.""" - n = 2.0 ** zoom - pixels = [] - - for coord in coordinates: - lon = float(coord[0]) if not isinstance(coord[0], (int, float)) else coord[0] - lat = float(coord[1]) if not isinstance(coord[1], (int, float)) else coord[1] - - x = ((lon + 180.0) / 360.0 * n - tile_x) * TILE_SIZE - lat_rad = math.radians(lat) - y = ((1.0 - math.log(math.tan(lat_rad) + 1 / math.cos(lat_rad)) / math.pi) / 2.0 * n - tile_y) * TILE_SIZE - - # Scale float pixel (0-256) to uint16 pixel (0-65536) - x_uint16 = int((x * (UINT16_TILE_SIZE - 1)) / (TILE_SIZE - 1)) - y_uint16 = int((y * (UINT16_TILE_SIZE - 1)) / (TILE_SIZE - 1)) - - # Clamp to valid uint16 range but preserve precision - x_uint16 = max(-UINT16_TILE_SIZE, min(UINT16_TILE_SIZE * 2, x_uint16)) - y_uint16 = max(-UINT16_TILE_SIZE, min(UINT16_TILE_SIZE * 2, y_uint16)) - - pixels.append((x_uint16, y_uint16)) - - return pixels - - -def pack_varint(n): - """Pack integer as variable-length integer.""" - out = bytearray() - while True: - byte = n & 0x7F - n >>= 7 - if n: - out.append(byte | 0x80) - else: - out.append(byte) - break - return out - - -def pack_zigzag(n): - """Pack signed integer using zigzag encoding.""" - return pack_varint((n << 1) ^ (n >> 31)) - - -def get_layer_priority(tags: Dict) -> int: - """Calculate rendering priority based on OSM layer tag and feature type.""" - if 'layer' in tags: - try: - layer_val = int(tags['layer']) - return layer_val * 1000 - except (ValueError, TypeError): - pass - - # Water features at bottom - if 'natural' in tags and tags['natural'] in ['water', 'coastline', 'bay']: - return 100 - if 'waterway' in tags and tags.get('waterway') in ['riverbank', 'dock', 'boatyard']: - return 100 - - # Land use and natural areas - if 'landuse' in tags: - return 200 - if 'natural' in tags and tags['natural'] in ['wood', 'forest', 'scrub', 'heath', 'grassland', 'beach', 'sand', 'wetland']: - return 200 - if 'leisure' in tags and tags['leisure'] in ['park', 'nature_reserve', 'garden']: - return 200 - - # Waterways - if 'waterway' in tags and tags['waterway'] in ['river', 'stream', 'canal']: - return 300 - - # Natural features - if 'natural' in tags and tags['natural'] in ['peak', 'ridge', 'volcano', 'cliff']: - return 400 - - # Underground features - if 'tunnel' in tags and tags['tunnel'] == 'yes': - return 500 - - # Railways - if 'railway' in tags: - return 600 - - # Minor roads - if 'highway' in tags and tags['highway'] in ['path', 'footway', 'cycleway', 'steps', 'pedestrian', 'track']: - return 700 - - # Tertiary roads - if 'highway' in tags and tags['highway'] in ['tertiary', 'tertiary_link']: - return 800 - - # Secondary roads - if 'highway' in tags and tags['highway'] in ['secondary', 'secondary_link']: - return 900 - - # Primary roads - if 'highway' in tags and tags['highway'] in ['primary', 'primary_link']: - return 1000 - - # Trunk roads - if 'highway' in tags and tags['highway'] in ['trunk', 'trunk_link']: - return 1100 - - # Motorways - if 'highway' in tags and tags['highway'] in ['motorway', 'motorway_link']: - return 1200 - - # Above-ground structures - if 'bridge' in tags and tags['bridge'] == 'yes': - return 1300 - if 'aeroway' in tags: - return 1300 - - # Buildings - if 'building' in tags: - return 1400 - - # Points of interest - if 'amenity' in tags: - return 1500 - - return 200 - - -def geometry_to_commands(geom_type: str, coordinates, color: int, zoom: int, tile_x: int, tile_y: int, - color_index: Optional[int] = None, tags: Dict = None) -> List[Dict]: - """Convert geometry to draw commands with appropriate line widths.""" - commands = [] - - if color_index is not None: - commands.append({'type': DRAW_COMMANDS['SET_COLOR_INDEX'], 'color_index': color_index}) - else: - commands.append({'type': DRAW_COMMANDS['SET_COLOR'], 'color': color}) - - line_width = get_line_width_from_tags(tags, zoom, coordinates, geom_type) - - def process_linestring(coords, width): - if len(coords) < 2: - return [] - - pixels = coords_to_pixels(coords, zoom, tile_x, tile_y) - - # Don't remove duplicate pixels - they're needed for smooth wide lines - unique_pixels = pixels - - if len(unique_pixels) < 2: - return [] - - # Always use POLYLINE for better rendering of wide lines - return [{'type': DRAW_COMMANDS['POLYLINE'], 'points': unique_pixels, 'width': width}] - - def process_polygon(coords, width): - if not coords or not coords[0] or len(coords[0]) < 3: - return [] - - exterior = coords[0] - pixels = coords_to_pixels(exterior, zoom, tile_x, tile_y) - - # Don't remove duplicates for polygons either - unique_pixels = pixels - - # Close the ring if it's not already closed - if unique_pixels and unique_pixels[0] != unique_pixels[-1]: - unique_pixels.append(unique_pixels[0]) - - if len(set(unique_pixels)) < 3: - return [] - - return [{'type': DRAW_COMMANDS['STROKE_POLYGON'], 'points': unique_pixels, 'width': width}] - - if geom_type == "LineString": - commands.extend(process_linestring(coordinates, line_width)) - elif geom_type == "Polygon": - commands.extend(process_polygon(coordinates, line_width)) - elif geom_type == "MultiLineString": - for line in coordinates: - line_commands = process_linestring(line, line_width) - if line_commands: - commands.extend(line_commands) - elif geom_type == "MultiPolygon": - for poly in coordinates: - poly_commands = process_polygon(poly, line_width) - if poly_commands: - commands.extend(poly_commands) - - return commands - - -def pack_draw_commands(commands: List[Dict]) -> bytes: - """Pack draw commands into binary format.""" - out = bytearray() - out += pack_varint(len(commands)) - - for cmd in commands: - cmd_type = cmd['type'] - out += pack_varint(cmd_type) - - if cmd_type == DRAW_COMMANDS['SET_COLOR']: - out += struct.pack("B", cmd.get('color', 0xFF)) - elif cmd_type == DRAW_COMMANDS['SET_COLOR_INDEX']: - out += pack_varint(cmd.get('color_index', 0)) - elif cmd_type in [DRAW_COMMANDS['POLYLINE'], DRAW_COMMANDS['STROKE_POLYGON']]: - points = cmd['points'] - width = cmd.get('width', 1) - out += pack_varint(width) - out += pack_varint(len(points)) - prev_x, prev_y = 0, 0 - for i, (x, y) in enumerate(points): - if i == 0: - out += pack_zigzag(x) - out += pack_zigzag(y) - else: - out += pack_zigzag(x - prev_x) - out += pack_zigzag(y - prev_y) - prev_x, prev_y = x, y - - return bytes(out) - - -def compress_goql_queries(config: Dict) -> str: - """Compress configuration into optimized GOQL query.""" - META = {'tile_size', 'viewport_size', 'toolbar_width', 'statusbar_height', 'max_cache_size', - 'thread_pool_size', 'background_colors', 'log_level', 'config_file', 'fps_limit', 'fill_polygons'} - LINEAR_TAGS = {'highway', 'railway'} - LINEAR_VALUES = {'natural': {'coastline', 'tree_row'}, 'waterway': {'river', 'stream', 'canal'}} - AREA_VALUES = {'waterway': {'riverbank', 'dock', 'boatyard'}} - - way_q = [] - area_q = [] - - for key in config: - if key in META or not isinstance(config[key], dict): - continue - if '=' in key: - tag, val = key.split('=', 1) - is_lin = (tag in LINEAR_VALUES and val in LINEAR_VALUES[tag]) or tag in LINEAR_TAGS - is_area = tag in AREA_VALUES and val in AREA_VALUES[tag] - - if is_lin and not is_area: - way_q.append(tag + "=" + val) - else: - area_q.append(tag + "=" + val) - else: - area_q.append(key) - - def group(queries): - grp = {} - single = [] - for q in queries: - if '=' in q: - t, v = q.split('=', 1) - if t not in grp: - grp[t] = [] - grp[t].append(v) - else: - single.append(q) - return grp, single - - parts = [] - way_grp, _ = group(way_q) - for tag, vals in sorted(way_grp.items()): - if len(vals) == 1: - parts.append("w[" + tag + "=" + vals[0] + "]") - else: - pat = '|'.join(vals) - parts.append("w[" + tag + "~\"^(" + pat + ")$\"]") - - area_grp, area_single = group(area_q) - for tag, vals in sorted(area_grp.items()): - if len(vals) == 1: - parts.append("nwa[" + tag + "=" + vals[0] + "]") - else: - pat = '|'.join(vals) - parts.append("nwa[" + tag + "~\"^(" + pat + ")$\"]") - - for tag in sorted(area_single): - parts.append("nwa[" + tag + "]") - - return ", ".join(parts) if parts else "*" - - -def process_feature(feature: Dict, config: Dict, zoom: int, tiles_data: Dict): - """Process a single GeoJSON feature and add it to tiles_data.""" - geom = feature.get('geometry') - props = feature.get('properties', {}) - - if not geom or geom['type'] not in ['LineString', 'Polygon', 'MultiLineString', 'MultiPolygon']: - return - - geom_type = geom['type'] - coordinates = geom['coordinates'] - - style = get_style_for_tags(props, config) - if not style: - return - - zoom_filter = style.get('zoom', 6) - if zoom < zoom_filter: - return - - coordinates = normalize_coordinates_robust(coordinates) - - # 1. Aplicar suavizado (smoothing) para alta fidelidad en curvas (Z >= 16). - coordinates = smooth_geometry_coords(coordinates, geom_type, zoom, props) - - hex_color = style.get('color', '#FFFFFF') - color = hex_to_rgb332(hex_color) - color_index = hex_to_color_index(hex_color) - priority = style.get('priority', 50) - layer_priority = get_layer_priority(props) - - minx, miny, maxx, maxy = geojson_bounds(coordinates, geom_type) - tile_x_min, tile_y_min = deg2num(miny, minx, zoom) - tile_x_max, tile_y_max = deg2num(maxy, maxx, zoom) - - # 3. Iterar y clip (Cohen-Sutherland/Sutherland-Hodgman) - for tx in range(min(tile_x_min, tile_x_max), max(tile_x_min, tile_x_max) + 1): - for ty in range(min(tile_y_min, tile_y_max), max(tile_y_min, tile_y_max) + 1): - tile_bounds = tile_bbox(tx, ty, zoom) - - if not coords_intersect_tile(coordinates, geom_type, tile_bounds): - continue - - clipped = clip_coordinates_to_tile(coordinates, geom_type, tile_bounds) - if not clipped: - continue - - commands = geometry_to_commands(geom_type, clipped, color, zoom, tx, ty, color_index, props) - if commands: - tile_key = (tx, ty) - if tile_key not in tiles_data: - tiles_data[tile_key] = [] - combined_priority = layer_priority + priority - tiles_data[tile_key].append({'commands': commands, 'priority': combined_priority}) - - -def get_style_for_tags(tags: Dict, config: Dict) -> Optional[Dict]: - """Find style configuration for given OSM tags.""" - for k, v in tags.items(): - keyval = k + "=" + str(v) - if keyval in config and isinstance(config[keyval], dict): - return config[keyval] - - for k in tags: - if k in config and isinstance(config[k], dict): - return config[k] - - return None - - -def sort_and_flatten_commands(tile_entries: List[Dict]) -> List[Dict]: - """Sort tile entries by priority and flatten to command list.""" - sorted_entries = sorted(tile_entries, key=lambda x: x['priority']) - all_commands = [] - for entry in sorted_entries: - all_commands.extend(entry['commands']) - return all_commands - - -def write_single_tile(job): - """Write a single tile to disk.""" - tx, ty, tile_entries, zoom, output_dir, max_file_size = job - commands = sort_and_flatten_commands(tile_entries) - - tile_dir = os.path.join(output_dir, str(zoom), str(tx)) - os.makedirs(tile_dir, exist_ok=True) - filepath = os.path.join(tile_dir, str(ty) + ".bin") - - data = pack_draw_commands(commands) - if len(data) > max_file_size: - data = data[:max_file_size] - - with open(filepath, 'wb') as f: - f.write(data) - - -def format_time(seconds: float) -> str: - """Format seconds into human-readable time string.""" - if seconds < 60: - return f"{seconds:.2f}s" - elif seconds < 3600: - minutes = int(seconds // 60) - secs = seconds % 60 - return f"{minutes}m {secs:.2f}s" - else: - hours = int(seconds // 3600) - minutes = int((seconds % 3600) // 60) - secs = seconds % 60 - return f"{hours}h {minutes}m {secs:.2f}s" - - -def merge_tiles_data(target: Dict, source: Dict): - """Merge source tiles data into target dictionary.""" - for tile_key, entries in source.items(): - if tile_key in target: - target[tile_key].extend(entries) - else: - target[tile_key] = entries - - -def write_tiles_batch(tiles_data: Dict, zoom: int, output_dir: str, max_file_size: int, - max_workers: int, persistent_tiles: Dict): - """Merge current batch into persistent tiles storage.""" - merge_tiles_data(persistent_tiles, tiles_data) - return len(tiles_data) - - -def write_final_tiles(persistent_tiles: Dict, zoom: int, output_dir: str, max_file_size: int, max_workers: int): - """Write all accumulated tiles to disk using thread pool.""" - if not persistent_tiles: - return 0 - - tile_jobs = [(tx, ty, entries, zoom, output_dir, max_file_size) - for (tx, ty), entries in persistent_tiles.items()] - tiles_written = len(tile_jobs) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(write_single_tile, job) for job in tile_jobs] - for future in as_completed(futures): - future.result() - - return tiles_written - - -def generate_tiles(gol_file: str, output_dir: str, config_file: str, zoom_levels: List[int], - max_file_size: int = 65536, base_batch_size: int = 10000): - """Main tile generation function.""" - with open(config_file) as f: - config = json.load(f) - - available_memory_mb = get_available_memory_mb() - cpu_count = os.cpu_count() or 4 - max_workers = get_optimal_workers(cpu_count, available_memory_mb) - - logger.info(f"System detected: {cpu_count} CPU cores, {available_memory_mb}MB RAM") - logger.info(f"Resource settings: {max_workers} workers, base batch size: {base_batch_size}") - - precompute_global_color_palette(config) - write_palette_bin(output_dir) - query = compress_goql_queries(config) - - gol_cmd = "/gol" if os.path.exists("/gol") else "gol" - - for zoom in zoom_levels: - zoom_start = time.time() - adaptive_batch = get_adaptive_batch_size(zoom, base_batch_size, available_memory_mb) - logger.info(f"Processing zoom {zoom} (batch size: {adaptive_batch}, workers: {max_workers})...") - - process = subprocess.Popen( - [gol_cmd, "query", gol_file, query, "-f", "geojson"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - bufsize=65536 - ) - - tiles_data = {} - persistent_tiles = {} - feature_count = 0 - batch_count = 0 - stderr_content = "" - - try: - for feature in ijson.items(process.stdout, "features.item"): - feature_count += 1 - process_feature(feature, config, zoom, tiles_data) - - if feature_count % adaptive_batch == 0: - batch_count += 1 - write_tiles_batch(tiles_data, zoom, output_dir, max_file_size, max_workers, persistent_tiles) - print(f"\rZoom {zoom}: {feature_count} features, {len(persistent_tiles)} unique tiles (batch {batch_count})...", - end='', flush=True) - tiles_data.clear() - gc.collect() - elif feature_count % 5000 == 0: - print(f"\rZoom {zoom}: {feature_count} features, {len(tiles_data)} tiles in buffer, {len(persistent_tiles)} persistent...", - end='', flush=True) - - if tiles_data: - write_tiles_batch(tiles_data, zoom, output_dir, max_file_size, max_workers, persistent_tiles) - - print(f"\rZoom {zoom}: {feature_count} features processed, writing {len(persistent_tiles)} tiles...{' ' * 20}") - sys.stdout.flush() - - total_tiles_written = write_final_tiles(persistent_tiles, zoom, output_dir, max_file_size, max_workers) - - print(f"\rZoom {zoom}: {feature_count} features processed, {total_tiles_written} tiles written{' ' * 20}") - sys.stdout.flush() - - except ijson.common.IncompleteJSONError: - pass - finally: - if process.stdout: - process.stdout.close() - if process.stderr: - stderr_content = process.stderr.read() if process.stderr else "" - if process.stderr: - process.stderr.close() - - process.wait() - if process.returncode != 0: - raise RuntimeError("gol query failed: " + stderr_content) - - zoom_elapsed = time.time() - zoom_start - logger.info(f"Zoom {zoom} completed in {format_time(zoom_elapsed)}") - - del tiles_data - del persistent_tiles - del process - gc.collect() - - -def main(): - """Command-line entry point for tile generator.""" - start_time = time.time() - - parser = argparse.ArgumentParser( - description='Generate map tiles from GoL file with OSM-compliant width handling' - ) - parser.add_argument("gol_file", help="Input GoL file path") - parser.add_argument("output_dir", help="Output directory for tiles") - parser.add_argument("config_file", help="JSON config file with feature definitions") - parser.add_argument("--zoom", default="6-17", help="Zoom level(s) to generate (e.g., '12' or '10-15')") - parser.add_argument("--max-file-size", type=int, default=128, help="Maximum tile file size in KB") - parser.add_argument("--batch-size", type=int, default=10000, - help="Base batch size (auto-adjusted per zoom level and system RAM)") - - args = parser.parse_args() - - if '-' in args.zoom: - start, end = map(int, args.zoom.split('-')) - zoom_levels = list(range(start, end + 1)) - else: - zoom_levels = [int(args.zoom)] - - max_file_size_bytes = args.max_file_size * 1024 - - logger.info("=" * 50) - logger.info("OSM-Compliant Tile Generator (CLEANED and CLIPPING FIXED)") - logger.info("=" * 50) - logger.info(f"Input: {args.gol_file}") - logger.info(f"Output: {args.output_dir}") - logger.info(f"Config: {args.config_file}") - logger.info(f"Zoom levels: {zoom_levels}") - logger.info(f"Max tile size: {args.max_file_size}KB") - logger.info(f"Base batch size: {args.batch_size} features (adaptive per zoom and system RAM)") - logger.info(f"Simplification: DISABLED (Geometry preserved)") - logger.info(f"Width handling: Hybrid (OSM Tags + CartoCSS Defaults) - SCALED DOWN for Small Screens") - logger.info("=" * 50) - - generate_tiles( - args.gol_file, - args.output_dir, - args.config_file, - zoom_levels, - max_file_size_bytes, - args.batch_size - ) - - total_count = 0 - total_size = 0 - for zoom in zoom_levels: - zoom_dir = os.path.join(args.output_dir, str(zoom)) - if os.path.exists(zoom_dir): - for x_dir in os.listdir(zoom_dir): - x_path = os.path.join(zoom_dir, x_dir) - if os.path.isdir(x_path): - for tile_file in os.listdir(x_path): - if tile_file.endswith('.bin'): - total_count += 1 - tile_path = os.path.join(x_path, tile_file) - total_size += os.path.getsize(tile_path) - - if total_count > 0: - if total_size < 1024: - size_str = f"{total_size}B" - elif total_size < 1024 * 1024: - size_str = f"{round(total_size / 1024, 1)}KB" - elif total_size < 1024 * 1024 * 1024: - size_str = f"{round(total_size / (1024 * 1024), 1)}MB" - else: - size_str = f"{round(total_size / (1024 * 1024 * 1024), 2)}GB" - - avg_size = round(total_size / total_count) - else: - size_str = "0B" - avg_size = 0 - - end_time = time.time() - elapsed_time = end_time - start_time - - logger.info("=" * 50) - logger.info("Generation Summary") - logger.info("=" * 50) - logger.info(f"Total tiles written: {total_count}") - logger.info(f"Total size: {size_str}") - logger.info(f"Average tile size: {avg_size} bytes") - logger.info(f"Palette file: {os.path.join(args.output_dir, 'palette.bin')}") - logger.info(f"Total processing time: {format_time(elapsed_time)}") - logger.info("=" * 50) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tile_viewer.py b/tile_viewer.py deleted file mode 100644 index 82a8fe2..0000000 --- a/tile_viewer.py +++ /dev/null @@ -1,1490 +0,0 @@ -import struct -import sys -import os -import math -import pygame -import threading -import logging -from concurrent.futures import ThreadPoolExecutor, as_completed -from collections import OrderedDict -from functools import lru_cache - -# Drawing command codes (solo los usados por tile_generator.py) -DRAW_COMMANDS = { - 'LINE': 1, - 'POLYLINE': 2, - 'STROKE_POLYGON': 3, - 'SET_COLOR': 0x80, - 'SET_COLOR_INDEX': 0x81, -} - -# Tile configuration -TILE_SIZE = 256 -UINT16_TILE_SIZE = 65536 - -# Setup logging first -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -# Default configuration -DEFAULT_CONFIG = { - 'tile_size': 256, - 'viewport_size': 768, - 'toolbar_width': 160, - 'statusbar_height': 40, - 'max_cache_size': 1000, - 'thread_pool_size': 4, - 'background_colors': [(0, 0, 0), (255, 255, 255)], - 'log_level': 'INFO', - 'config_file': 'features.json', - 'fps_limit': 30, - 'fill_polygons': False -} - -class Config: - """Configuration management class""" - def __init__(self, config_file=None): - self.config = DEFAULT_CONFIG.copy() - self.config_file = config_file or DEFAULT_CONFIG['config_file'] - self.load_config() - - def load_config(self): - """Load configuration from file if it exists""" - try: - if os.path.exists(self.config_file): - import json - with open(self.config_file, 'r') as f: - file_config = json.load(f) - - # Merge file config with defaults - self.config.update(file_config) - logger.info(f"Loaded configuration from {self.config_file}") - else: - logger.info(f"Config file {self.config_file} not found, using defaults") - except Exception as e: - logger.error(f"Error loading config file {self.config_file}: {e}") - logger.info("Using default configuration") - - def get(self, key, default=None): - """Get configuration value""" - return self.config.get(key, default) - - def set(self, key, value): - """Set configuration value""" - self.config[key] = value - - def save_config(self): - """Save current configuration to file""" - try: - import json - with open(self.config_file, 'w') as f: - json.dump(self.config, f, indent=2) - logger.info(f"Saved configuration to {self.config_file}") - except Exception as e: - logger.error(f"Error saving config file: {e}") - -# Global configuration instance -config = Config() - -# Initialize constants from configuration -TILE_SIZE = config.get('tile_size', 256) -VIEWPORT_SIZE = config.get('viewport_size', 768) -TOOLBAR_WIDTH = config.get('toolbar_width', 160) -STATUSBAR_HEIGHT = config.get('statusbar_height', 40) -WINDOW_WIDTH = VIEWPORT_SIZE + TOOLBAR_WIDTH -WINDOW_HEIGHT = VIEWPORT_SIZE + STATUSBAR_HEIGHT - -# Global variables for palette -GLOBAL_PALETTE = {} - -class TileCache: - """LRU Cache for tile surfaces to limit memory usage""" - def __init__(self, max_size=None): - self.cache = OrderedDict() - self.max_size = max_size or config.get('max_cache_size', 1000) - self.hits = 0 - self.misses = 0 - - def get(self, key): - """Get tile surface from cache""" - if key in self.cache: - self.cache.move_to_end(key) - self.hits += 1 - return self.cache[key] - self.misses += 1 - return None - - def put(self, key, value): - """Put tile surface in cache""" - if key in self.cache: - self.cache.move_to_end(key) - else: - if len(self.cache) >= self.max_size: - removed_key, removed_value = self.cache.popitem(last=False) - logger.debug(f"Evicted tile from cache: {removed_key}") - self.cache[key] = value - - def clear(self): - """Clear all cached surfaces""" - self.cache.clear() - logger.info("Tile cache cleared") - - def get_stats(self): - """Get cache statistics""" - total = self.hits + self.misses - hit_rate = (self.hits / total * 100) if total > 0 else 0 - return { - 'size': len(self.cache), - 'max_size': self.max_size, - 'hits': self.hits, - 'misses': self.misses, - 'hit_rate': hit_rate - } - -# Global tile cache instance -tile_cache = TileCache() - -class TileLoader: - """Persistent thread pool for tile loading operations""" - def __init__(self, max_workers=None): - self.executor = ThreadPoolExecutor(max_workers=max_workers or config.get('thread_pool_size', 4)) - self.active_futures = set() - - def submit_tile_load(self, tile_info, callback=None): - """Submit a tile loading task""" - future = self.executor.submit(self._load_single_tile, tile_info) - if callback: - future.add_done_callback(callback) - self.active_futures.add(future) - return future - - def _load_single_tile(self, tile_info): - """Load a single tile (internal method) with error recovery""" - try: - x, y, zoom_level, directory, bg_color, fill_mode = tile_info - key = (zoom_level, x, y, bg_color, fill_mode) - - # Check cache first - cached_surface = tile_cache.get(key) - if cached_surface is not None: - return key, cached_surface - - tile_file = get_tile_file(directory, x, y) - if not tile_file: - logger.warning(f"No tile file found for ({x}, {y}) in {directory}") - return None, None - - # Validate tile file exists - if not os.path.exists(tile_file): - logger.warning(f"Tile file does not exist: {tile_file}") - return None, None - - surface = render_tile_surface({'x': x, 'y': y, 'file': tile_file}, bg_color, fill_mode, False) - if surface is None: - logger.error(f"Failed to render tile surface for {tile_file}") - return None, None - - tile_cache.put(key, surface) - return key, surface - - except Exception as e: - logger.error(f"Error loading tile {tile_info}: {e}") - return None, None - - def cleanup_completed_futures(self): - """Remove completed futures from active set""" - completed = [f for f in self.active_futures if f.done()] - for future in completed: - self.active_futures.discard(future) - return len(completed) - - def shutdown(self): - """Shutdown the thread pool""" - self.executor.shutdown(wait=True) - logger.info("Tile loader thread pool shutdown") - -# Global tile loader instance -tile_loader = TileLoader() - -def load_global_palette_from_bin(palette_file): - """ - Load the global palette from the binary palette file generated by tile_generator.py. - """ - global GLOBAL_PALETTE - try: - with open(palette_file, 'rb') as f: - # Read 4-byte header for number of colors - header = f.read(4) - if len(header) < 4: - logger.warning(f"Invalid palette file header: {palette_file}") - return False - - num_colors = struct.unpack(' RGB888 - GLOBAL_PALETTE = {} - for index, hex_color in enumerate(sorted_colors): - rgb888 = hex_to_rgb888(hex_color) - GLOBAL_PALETTE[index] = rgb888 - - logger.info(f"Loaded dynamic palette: {len(GLOBAL_PALETTE)} colors") - return True - except Exception as e: - logger.warning(f"Could not load palette from config: {e}") - logger.info("Will use fallback palette if needed") - return False - -def hex_to_rgb888(hex_color): - """Converts hex color to RGB888""" - try: - if not hex_color or not hex_color.startswith("#"): - return (255, 255, 255) - r = int(hex_color[1:3], 16) - g = int(hex_color[3:5], 16) - b = int(hex_color[5:7], 16) - return (r, g, b) - except: - return (255, 255, 255) - -def darken_color(rgb, amount=0.3): - """Darken RGB color by specified amount""" - return tuple(max(0, int(v * (1 - amount))) for v in rgb) - -def skip_command(cmd_type, data, offset): - """Skip command data without processing it (incluyendo ancho)""" - if cmd_type == 1: # LINE - _, offset = read_varint(data, offset) # Saltar ancho - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - elif cmd_type == 2: # POLYLINE - _, offset = read_varint(data, offset) # Saltar ancho - n_pts, offset = read_varint(data, offset) - for _ in range(n_pts): - if _ == 0: - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - else: - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - elif cmd_type == 3: # STROKE_POLYGON - _, offset = read_varint(data, offset) # Saltar ancho - n_pts, offset = read_varint(data, offset) - for _ in range(n_pts): - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - else: - # Para comandos desconocidos, saltar 4 bytes - if offset < len(data) - 4: - offset += 4 - - return offset - -@lru_cache(maxsize=1000) -def uint16_to_tile_pixel(val): - """Convert uint16 coordinate to tile pixel with caching""" - return int(round(val * (TILE_SIZE - 1) / (UINT16_TILE_SIZE - 1))) - -def get_button_icons(): - """Create beautiful, modern button icons""" - - # Background toggle icon (sun/moon) - icon_surface_bg = pygame.Surface((24, 24), pygame.SRCALPHA) - icon_surface_bg.fill((0, 0, 0, 0)) - # Sun rays - for i in range(8): - angle = i * 45 - x1 = 12 + int(8 * math.cos(math.radians(angle))) - y1 = 12 + int(8 * math.sin(math.radians(angle))) - x2 = 12 + int(10 * math.cos(math.radians(angle))) - y2 = 12 + int(10 * math.sin(math.radians(angle))) - pygame.draw.line(icon_surface_bg, (255, 255, 0), (x1, y1), (x2, y2), 2) - # Sun center - pygame.draw.circle(icon_surface_bg, (255, 255, 0), (12, 12), 6, 0) - pygame.draw.circle(icon_surface_bg, (255, 200, 0), (12, 12), 4, 0) - - # Tile labels icon (document with lines) - icon_surface_label = pygame.Surface((24, 24), pygame.SRCALPHA) - icon_surface_label.fill((0, 0, 0, 0)) - # Document background - pygame.draw.rect(icon_surface_label, (255, 255, 255), (6, 4, 12, 16), 0) - pygame.draw.rect(icon_surface_label, (200, 200, 200), (6, 4, 12, 16), 2) - # Document fold - pygame.draw.polygon(icon_surface_label, (240, 240, 240), [(18, 4), (18, 8), (14, 8)], 0) - pygame.draw.polygon(icon_surface_label, (180, 180, 180), [(18, 4), (18, 8), (14, 8)], 1) - # Text lines - pygame.draw.line(icon_surface_label, (100, 100, 100), (8, 8), (16, 8), 1) - pygame.draw.line(icon_surface_label, (100, 100, 100), (8, 10), (14, 10), 1) - pygame.draw.line(icon_surface_label, (100, 100, 100), (8, 12), (16, 12), 1) - pygame.draw.line(icon_surface_label, (100, 100, 100), (8, 14), (12, 14), 1) - - # GPS cursor icon (crosshair with target) - icon_surface_gps = pygame.Surface((24, 24), pygame.SRCALPHA) - icon_surface_gps.fill((0, 0, 0, 0)) - # Outer circle - pygame.draw.circle(icon_surface_gps, (0, 255, 0), (12, 12), 10, 2) - # Inner circle - pygame.draw.circle(icon_surface_gps, (0, 255, 0), (12, 12), 6, 2) - # Crosshair lines - pygame.draw.line(icon_surface_gps, (0, 255, 0), (12, 2), (12, 6), 2) - pygame.draw.line(icon_surface_gps, (0, 255, 0), (12, 18), (12, 22), 2) - pygame.draw.line(icon_surface_gps, (0, 255, 0), (2, 12), (6, 12), 2) - pygame.draw.line(icon_surface_gps, (0, 255, 0), (18, 12), (22, 12), 2) - # Center dot - pygame.draw.circle(icon_surface_gps, (0, 255, 0), (12, 12), 2, 0) - - # Fill polygons icon (paint bucket) - icon_surface_fill = pygame.Surface((24, 24), pygame.SRCALPHA) - icon_surface_fill.fill((0, 0, 0, 0)) - # Paint bucket body - pygame.draw.rect(icon_surface_fill, (255, 100, 100), (8, 6, 8, 10), 0) - pygame.draw.rect(icon_surface_fill, (200, 80, 80), (8, 6, 8, 10), 1) - # Paint bucket spout - pygame.draw.polygon(icon_surface_fill, (255, 100, 100), [(10, 16), (14, 16), (12, 20)], 0) - pygame.draw.polygon(icon_surface_fill, (200, 80, 80), [(10, 16), (14, 16), (12, 20)], 1) - # Paint drops - pygame.draw.circle(icon_surface_fill, (255, 100, 100), (6, 18), 1, 0) - pygame.draw.circle(icon_surface_fill, (255, 100, 100), (18, 20), 1, 0) - pygame.draw.circle(icon_surface_fill, (255, 100, 100), (4, 22), 1, 0) - - return icon_surface_bg, icon_surface_label, icon_surface_gps, icon_surface_fill - -def index_available_tiles(directory, progress_callback=None): - available_tiles = set() - if not os.path.isdir(directory): - logger.error(f"Directory does not exist: {directory}") - return available_tiles - x_dirs = [x_str for x_str in os.listdir(directory) if os.path.isdir(os.path.join(directory, x_str))] - total_x = len(x_dirs) - def index_xdir(x_str): - x_path = os.path.join(directory, x_str) - try: - x = int(x_str) - except: - return [] - files = os.listdir(x_path) - y_dict = {} - for fname in files: - if fname.endswith('.bin') or fname.endswith('.png'): - y_str = fname.split('.')[0] - if y_str.isdigit(): - y = int(y_str) - if y not in y_dict or fname.endswith('.bin'): - y_dict[y] = fname - return [(x, y) for y in y_dict] - results = [] - with ThreadPoolExecutor(min(8, os.cpu_count() or 4)) as pool: - futures = {pool.submit(index_xdir, x_str): i_x for i_x, x_str in enumerate(x_dirs)} - for i, future in enumerate(as_completed(futures)): - tiles = future.result() - results.extend(tiles) - if progress_callback is not None: - percent = (i + 1) / max(total_x, 1) - progress_callback(percent, "Indexing tiles...") - available_tiles.update(results) - return available_tiles - -def get_tile_file(directory, x, y): - """Get tile file path with validation""" - bin_path = f"{directory}/{x}/{y}.bin" - png_path = f"{directory}/{x}/{y}.png" - if os.path.isfile(bin_path): - return bin_path - elif os.path.isfile(png_path): - return png_path - return None - -def is_tile_visible(x, y, viewport_x, viewport_y): - """Check if tile is visible in current viewport""" - tile_px = x * TILE_SIZE - viewport_x - tile_py = y * TILE_SIZE - viewport_y - - # Check if tile intersects with viewport - return not (tile_px + TILE_SIZE < 0 or tile_px > VIEWPORT_SIZE or - tile_py + TILE_SIZE < 0 or tile_py > VIEWPORT_SIZE) - -def get_visible_tiles(available_tiles, viewport_x, viewport_y): - """Get list of tiles that are currently visible""" - min_tile_x = int(viewport_x // TILE_SIZE) - max_tile_x = int((viewport_x + VIEWPORT_SIZE) // TILE_SIZE) - min_tile_y = int(viewport_y // TILE_SIZE) - max_tile_y = int((viewport_y + VIEWPORT_SIZE) // TILE_SIZE) - - visible_tiles = [] - for x in range(min_tile_x, max_tile_x + 1): - for y in range(min_tile_y, max_tile_y + 1): - if (x, y) in available_tiles: - visible_tiles.append((x, y)) - - return visible_tiles - -def read_varint(data, offset): - result = 0 - shift = 0 - while True: - if offset >= len(data): - return result, offset - b = data[offset] - offset += 1 - result |= (b & 0x7F) << shift - if not (b & 0x80): - break - shift += 7 - return result, offset - -def read_zigzag(data, offset): - v, offset = read_varint(data, offset) - return (v >> 1) ^ -(v & 1), offset - -def rgb332_to_rgb888(c): - r = (c & 0xE0) - g = (c & 0x1C) << 3 - b = (c & 0x03) << 6 - return (r, g, b) - -def is_tile_border_point(pt): - x, y = pt - return (x == 0 or x == TILE_SIZE-1) or (y == 0 or y == TILE_SIZE-1) - -def is_polygon_on_tile_border(pts): - """Check if a polygon has any points on the tile border""" - for pt in pts: - if is_tile_border_point(pt): - return True - return False - -def get_polygon_border_segments(pts): - """Get border segments of a polygon, distinguishing between tile border and interior segments""" - border_segments = [] - interior_segments = [] - - for i in range(len(pts)): - pt1 = pts[i] - pt2 = pts[(i + 1) % len(pts)] # Next point (wrapping around) - - # Check if this segment is on the tile border - if is_tile_border_point(pt1) and is_tile_border_point(pt2): - border_segments.append((pt1, pt2)) - else: - interior_segments.append((pt1, pt2)) - - return border_segments, interior_segments - -def get_background_color(surface): - """Get the background color from the surface""" - # Get the color of the top-left pixel as background color - return surface.get_at((0, 0))[:3] # RGB only, ignore alpha - -def create_error_tile(error_message="Error"): - """Create a tile surface indicating an error""" - surface = pygame.Surface((TILE_SIZE, TILE_SIZE)) - surface.fill((255, 0, 0)) # Red background for errors - try: - font = pygame.font.SysFont(None, 24) - text = font.render(error_message, True, (255, 255, 255)) - text_rect = text.get_rect(center=(TILE_SIZE//2, TILE_SIZE//2)) - surface.blit(text, text_rect) - except Exception: - pass # If even error rendering fails, just return red surface - return surface - -def load_png_tile(filepath): - """Load and render PNG tile with error recovery""" - try: - if not os.path.exists(filepath): - logger.warning(f"PNG file does not exist: {filepath}") - return create_error_tile("Missing") - - img = pygame.image.load(filepath) - if img is None: - logger.error(f"Failed to load PNG image: {filepath}") - return create_error_tile("Load Failed") - - img = pygame.transform.scale(img, (TILE_SIZE, TILE_SIZE)) - surface = pygame.Surface((TILE_SIZE, TILE_SIZE)) - surface.blit(img, (0, 0)) - return surface - except pygame.error as e: - logger.error(f"Pygame error loading PNG {filepath}: {e}") - return create_error_tile("Pygame Error") - except Exception as e: - logger.error(f"Unexpected error loading PNG {filepath}: {e}") - return create_error_tile("Unknown Error") - -def parse_command_header(data, offset): - """Parse the command header and return number of commands""" - try: - num_cmds, offset = read_varint(data, offset) - return num_cmds, offset - except Exception as e: - logger.error(f"Error parsing command header: {e}") - return 0, offset - -def handle_color_command(cmd_type, data, offset, current_color): - """Handle color setting commands""" - if cmd_type == 0x80: # SET_COLOR (direct RGB332) - if offset >= len(data): - return current_color, offset, True - current_color = data[offset] - offset += 1 - return current_color, offset, True - elif cmd_type == 0x81: # SET_COLOR_INDEX (palette index) - if offset >= len(data): - return current_color, offset, True - color_index, offset = read_varint(data, offset) - - # Convert index to RGB using global palette - if color_index in GLOBAL_PALETTE: - rgb = GLOBAL_PALETTE[color_index] - # Simulate RGB332 to maintain compatibility - current_color = ((rgb[0] & 0xE0) | ((rgb[1] & 0xE0) >> 3) | (rgb[2] >> 6)) - else: - # Fallback if we don't have the palette - current_color = 255 # Default color - logger.warning(f"Unknown palette index {color_index}") - return current_color, offset, True - - return current_color, offset, False - -def draw_thick_line_rounded(surface, color, start_pos, end_pos, width): - """Draw a line with rounded ends""" - # Calculate direction vector - dx = end_pos[0] - start_pos[0] - dy = end_pos[1] - start_pos[1] - length = max(1, math.sqrt(dx*dx + dy*dy)) # Avoid division by zero - - # Normalize direction vector - dx /= length - dy /= length - - # Calculate perpendicular vector - px = -dy * width / 2 - py = dx * width / 2 - - # Create the rectangle points for the thick line - points = [ - (start_pos[0] + px, start_pos[1] + py), - (start_pos[0] - px, start_pos[1] - py), - (end_pos[0] - px, end_pos[1] - py), - (end_pos[0] + px, end_pos[1] + py) - ] - - # Draw the rectangle (main line body) - pygame.draw.polygon(surface, color, points) - - # Draw rounded ends (circles at start and end) - pygame.draw.circle(surface, color, (int(start_pos[0]), int(start_pos[1])), width // 2) - pygame.draw.circle(surface, color, (int(end_pos[0]), int(end_pos[1])), width // 2) - -def draw_thick_polyline_rounded(surface, color, points, width): - """Draw a polyline with rounded joins and ends""" - if len(points) < 2: - return - - # Draw the line segments with rounded joins - for i in range(len(points) - 1): - draw_thick_line_rounded(surface, color, points[i], points[i + 1], width) - - # Draw rounded joints at intermediate points - for i in range(1, len(points) - 1): - pygame.draw.circle(surface, color, (int(points[i][0]), int(points[i][1])), width // 2) - -def render_polygon_command(data, offset, surface, rgb, fill_polygons, line_width): - """Render STROKE_POLYGON command con ancho de línea""" - n_pts, offset = read_varint(data, offset) - pts = [] - x, y = 0, 0 - for i in range(n_pts): - if i == 0: - x, offset = read_zigzag(data, offset) - y, offset = read_zigzag(data, offset) - else: - dx, offset = read_zigzag(data, offset) - dy, offset = read_zigzag(data, offset) - x += dx - y += dy - pts.append((uint16_to_tile_pixel(x), uint16_to_tile_pixel(y))) - - if fill_polygons and len(pts) >= 3: - pygame.draw.polygon(surface, rgb, pts, 0) - # Dibujar contorno con el ancho especificado - border_segments, interior_segments = get_polygon_border_segments(pts) - - # Dibujar segmentos de borde con color original - for pt1, pt2 in border_segments: - pygame.draw.line(surface, rgb, pt1, pt2, line_width) - - # Dibujar segmentos interiores con color oscurecido - for pt1, pt2 in interior_segments: - pygame.draw.line(surface, darken_color(rgb, 0.4), pt1, pt2, line_width) - elif not fill_polygons and len(pts) >= 2: - # Sin relleno: dibujar contorno con color original en interior, color de fondo en bordes - border_segments, interior_segments = get_polygon_border_segments(pts) - bg_color = get_background_color(surface) - - # Dibujar segmentos de borde con color de fondo - for pt1, pt2 in border_segments: - pygame.draw.line(surface, bg_color, pt1, pt2, line_width) - - # Dibujar segmentos interiores con color original - for pt1, pt2 in interior_segments: - pygame.draw.line(surface, rgb, pt1, pt2, line_width) - return offset - -def render_geometry_command(cmd_type, data, offset, surface, current_color, fill_mode, current_position, movement_vector, fill_polygons=False, show_tile_labels=False): - """Render geometry commands with rounded line caps and joins""" - rgb = rgb332_to_rgb888(current_color) if current_color is not None else (255, 255, 255) - - # Leer ancho de línea (por defecto 1) - line_width = 1 - - # Si surface es None, solo saltar el renderizado pero procesar para obtener offset correcto - if surface is None: - if cmd_type == 1: # LINE - line_width, offset = read_varint(data, offset) # Leer ancho - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - return offset, current_position, movement_vector - elif cmd_type == 2: # POLYLINE - line_width, offset = read_varint(data, offset) # Leer ancho - n_pts, offset = read_varint(data, offset) - for _ in range(n_pts): - if _ == 0: - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - else: - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - return offset, current_position, movement_vector - elif cmd_type == 3: # STROKE_POLYGON - line_width, offset = read_varint(data, offset) # Leer ancho - n_pts, offset = read_varint(data, offset) - for _ in range(n_pts): - _, offset = read_zigzag(data, offset) - _, offset = read_zigzag(data, offset) - return offset, current_position, movement_vector - return offset, current_position, movement_vector - - if cmd_type == 1: # LINE - line_width, offset = read_varint(data, offset) # Leer ancho - x1, offset = read_zigzag(data, offset) - y1, offset = read_zigzag(data, offset) - dx, offset = read_zigzag(data, offset) - dy, offset = read_zigzag(data, offset) - x2 = x1 + dx - y2 = y1 + dy - p1 = (uint16_to_tile_pixel(x1), uint16_to_tile_pixel(y1)) - p2 = (uint16_to_tile_pixel(x2), uint16_to_tile_pixel(y2)) - - if line_width == 1: - pygame.draw.line(surface, rgb, p1, p2, line_width) - else: - draw_thick_line_rounded(surface, rgb, p1, p2, line_width) - current_position = (x2, y2) - movement_vector = (dx, dy) - elif cmd_type == 2: # POLYLINE - line_width, offset = read_varint(data, offset) # Leer ancho - n_pts, offset = read_varint(data, offset) - pts = [] - x, y = 0, 0 - for i in range(n_pts): - if i == 0: - x, offset = read_zigzag(data, offset) - y, offset = read_zigzag(data, offset) - else: - dx, offset = read_zigzag(data, offset) - dy, offset = read_zigzag(data, offset) - x += dx - y += dy - pts.append((uint16_to_tile_pixel(x), uint16_to_tile_pixel(y))) - if len(pts) >= 2: - if line_width == 1: - pygame.draw.lines(surface, rgb, False, pts, line_width) - else: - # Usar el método de líneas gruesas con joins redondos - draw_thick_polyline_rounded(surface, rgb, pts, line_width) - current_position = (x, y) - elif cmd_type == 3: # STROKE_POLYGON - line_width, offset = read_varint(data, offset) # Leer ancho - offset = render_polygon_command(data, offset, surface, rgb, fill_polygons, line_width) - else: - logger.warning(f"Unknown command type: {cmd_type} (0x{cmd_type:02x})") - - return offset, current_position, movement_vector - -def render_tile_surface(tile, bg_color, fill_mode, show_tile_labels=False): - """Main function to render tile surface from binary data with error recovery""" - try: - surface = pygame.Surface((TILE_SIZE, TILE_SIZE)) - surface.fill(bg_color) - filepath = tile['file'] - - if not filepath: - logger.error("No file path provided for tile") - return create_error_tile("No File") - - if filepath.endswith('.png'): - png_surface = load_png_tile(filepath) - if png_surface: - return png_surface - return surface - - # Validate file exists and is readable - if not os.path.exists(filepath): - logger.warning(f"Tile file does not exist: {filepath}") - return create_error_tile("Missing") - - if not os.access(filepath, os.R_OK): - logger.error(f"Tile file is not readable: {filepath}") - return create_error_tile("No Access") - - try: - with open(filepath, "rb") as f: - data = f.read() - except PermissionError as e: - logger.error(f"Permission denied reading {filepath}: {e}") - return create_error_tile("Permission") - except OSError as e: - logger.error(f"OS error reading {filepath}: {e}") - return create_error_tile("OS Error") - except Exception as e: - logger.error(f"Unexpected error reading {filepath}: {e}") - return create_error_tile("Read Error") - - if len(data) < 1: - logger.warning(f"Empty tile file: {filepath}") - return surface - - offset = 0 - current_color = None - - # Initialize rendering state - current_position = (0, 0) - movement_vector = (0, 0) - - try: - num_cmds, offset = parse_command_header(data, offset) - - # Process commands in order - for cmd_idx in range(num_cmds): - if offset >= len(data): - logger.warning(f"Command {cmd_idx} extends beyond data length in {filepath}") - break - - cmd_type, offset = read_varint(data, offset) - - # Handle color commands - current_color, offset, is_color_cmd = handle_color_command(cmd_type, data, offset, current_color) - if is_color_cmd: - continue - - # Render geometry commands - fill_polygons = config.config.get('fill_polygons', False) - offset, current_position, movement_vector = render_geometry_command( - cmd_type, data, offset, surface, current_color, fill_mode, current_position, movement_vector, fill_polygons, show_tile_labels - ) - - except Exception as e: - logger.error(f"Error parsing commands in {filepath}: {e}") - logger.error(f"Error at offset: {offset}, data length: {len(data)}") - # Return partial surface instead of error tile for parsing errors - return surface - - return surface - - except Exception as e: - logger.error(f"Critical error in render_tile_surface: {e}") - return create_error_tile("Critical") - -def center_viewport_on_central_tile(available_tiles): - if not available_tiles: - return 0, 0 - xs = [x for x, y in available_tiles] - ys = [y for x, y in available_tiles] - min_x, max_x = min(xs), max(xs) - min_y, max_y = min(ys), max(ys) - center_x = (min_x + max_x) // 2 - center_y = (min_y + max_y) // 2 - viewport_x = center_x * TILE_SIZE - VIEWPORT_SIZE // 2 - viewport_y = center_y * TILE_SIZE - VIEWPORT_SIZE // 2 - return viewport_x, viewport_y - -def clamp_viewport(viewport_x, viewport_y, available_tiles): - if not available_tiles: - return viewport_x, viewport_y - xs = [x for x, y in available_tiles] - ys = [y for x, y in available_tiles] - min_x, max_x = min(xs), max(xs) - min_y, max_y = min(ys), max(ys) - viewport_x = max(min_x * TILE_SIZE, min(viewport_x, (max_x * TILE_SIZE + TILE_SIZE) - VIEWPORT_SIZE)) - viewport_y = max(min_y * TILE_SIZE, min(viewport_y, (max_y * TILE_SIZE + TILE_SIZE) - VIEWPORT_SIZE)) - return viewport_x, viewport_y - -def draw_button(surface, text, rect, bg_color, fg_color, border_color, font, icon=None, pressed=False): - """Draw a button with improved text handling and multi-line support""" - radius = 16 - pygame.draw.rect(surface, bg_color, rect, border_radius=radius) - pygame.draw.rect(surface, border_color, rect, 2, border_radius=radius) - if pressed: - pygame.draw.rect(surface, border_color, rect, 4, border_radius=radius) - - # Calculate content area - content_x = rect.left + 12 - content_y = rect.centery - icon_width = 0 - - if icon is not None: - icon_rect = icon.get_rect() - icon_rect.centery = rect.centery - icon_rect.left = rect.left + 12 - surface.blit(icon, icon_rect) - content_x = icon_rect.right + 8 - icon_width = icon_rect.width + 8 - - # Calculate available text width - max_text_width = rect.width - icon_width - 24 # 12px margin on each side - - # Split text into words for multi-line support - words = text.split() - if not words: - return - - # Use consistent font size (14px for better readability) - button_font = pygame.font.SysFont(None, 14) - line_height = button_font.get_height() - - # Calculate how many lines we need - lines = [] - current_line = [] - current_width = 0 - - for word in words: - word_width = button_font.size(word + " ")[0] - if current_width + word_width <= max_text_width: - current_line.append(word) - current_width += word_width - else: - if current_line: - lines.append(" ".join(current_line)) - current_line = [word] - current_width = word_width - else: - # Single word is too long, force it - lines.append(word) - current_line = [] - current_width = 0 - - if current_line: - lines.append(" ".join(current_line)) - - # Limit to 2 lines maximum - if len(lines) > 2: - lines = lines[:2] - # If we have more than 2 lines, truncate the last line with "..." - if len(lines) == 2: - last_line = lines[1] - while button_font.size(last_line + "...")[0] > max_text_width and len(last_line) > 3: - last_line = last_line[:-1] - lines[1] = last_line + "..." - - # Calculate total text height - total_text_height = len(lines) * line_height - - # Center the text vertically - start_y = content_y - total_text_height // 2 - - # Render each line - for i, line in enumerate(lines): - label = button_font.render(line, True, fg_color) - text_rect = label.get_rect(midleft=(content_x, start_y + i * line_height + line_height // 2)) - surface.blit(label, text_rect) - -def show_status_progress_bar(surface, percent, text, font): - bar_max_width = WINDOW_WIDTH // 3 - bar_height = 18 - bar_margin_right = 24 - bar_x = WINDOW_WIDTH - bar_max_width - bar_margin_right - bar_y = VIEWPORT_SIZE + STATUSBAR_HEIGHT // 2 - bar_height // 2 - pygame.draw.rect(surface, (80, 80, 80), (bar_x, bar_y, bar_max_width, bar_height)) - pygame.draw.rect(surface, (30, 160, 220), (bar_x, bar_y, int(bar_max_width * percent), bar_height)) - pygame.draw.rect(surface, (120, 120, 120), (bar_x, bar_y, bar_max_width, bar_height), 2) - label = font.render(text, True, (255,255,255)) - label_rect = label.get_rect(midleft=(bar_x + 8, bar_y + bar_height//2 - label.get_height()//2)) - surface.blit(label, label_rect) - -def draw_tile_labels( - screen, font, available_tiles, viewport_x, viewport_y, zoom_level, background_color, show_tile_labels, directory, fill_polygons=False -): - if not show_tile_labels: - return - fg = (0, 0, 0) if background_color == (255,255,255) else (255,255,255) - label_bg = (240,240,240) if background_color == (255,255,255) else (32,32,32) - border = (180,180,180) if background_color == (255,255,255) else (64,64,64) - outline = (120,120,120) if background_color == (255,255,255) else (220,220,220) - for x, y in available_tiles: - px = x * TILE_SIZE - viewport_x - py = y * TILE_SIZE - viewport_y - if px + TILE_SIZE < 0 or px > VIEWPORT_SIZE or py + TILE_SIZE < 0 or py > VIEWPORT_SIZE: - continue - filename = None - if os.path.isfile(f"{directory}/{x}/{y}.bin"): - filename = f"{y}.bin" - elif os.path.isfile(f"{directory}/{x}/{y}.png"): - filename = f"{y}.png" - else: - filename = f"{y}" - txt = f"x={x} y={y} z={zoom_level} {filename}" - label_surfs = [font.render(txt, True, fg)] - lw = max(s.get_width() for s in label_surfs) - lh = sum(s.get_height() for s in label_surfs) - margin = 2 - label_rect = pygame.Rect( - px + margin, py + margin, - lw + margin * 2, lh + margin * 2 - ) - pygame.draw.rect(screen, label_bg, label_rect) - pygame.draw.rect(screen, border, label_rect, 1) - offset_y = label_rect.top + margin - for surf in label_surfs: - screen.blit(surf, (label_rect.left + margin, offset_y)) - offset_y += surf.get_height() - # Draw tile borders when show_tile_labels is enabled - draw_dashed_rect(screen, pygame.Rect(px, py, TILE_SIZE, TILE_SIZE), outline, width=1) - -def draw_dashed_rect(surface, rect, color, dash_length=6, gap_length=4, width=1): - x = rect.left - while x < rect.right: - end_x = min(x + dash_length, rect.right) - pygame.draw.line(surface, color, (x, rect.top), (end_x, rect.top), width) - x += dash_length + gap_length - x = rect.left - while x < rect.right: - end_x = min(x + dash_length, rect.right) - pygame.draw.line(surface, color, (x, rect.bottom-1), (end_x, rect.bottom-1), width) - x += dash_length + gap_length - y = rect.top - while y < rect.bottom: - end_y = min(y + dash_length, rect.bottom) - pygame.draw.line(surface, color, (rect.left, y), (rect.left, end_y), width) - y += dash_length + gap_length - y = rect.top - while y < rect.bottom: - end_y = min(y + dash_length, rect.bottom) - pygame.draw.line(surface, color, (rect.right-1, y), (rect.right-1, end_y), width) - y += dash_length + gap_length - -def pixel_to_latlon(px, py, viewport_x, viewport_y, zoom): - """Convert pixel coordinates to lat/lon""" - map_px = viewport_x + px - map_py = viewport_y + py - tile_x = map_px / TILE_SIZE - tile_y = map_py / TILE_SIZE - n = 2.0 ** zoom - lon_deg = tile_x / n * 360.0 - 180.0 - lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * tile_y / n))) - lat_deg = math.degrees(lat_rad) - return lat_deg, lon_deg - -def latlon_to_pixel(lat, lon, zoom): - """Convert lat/lon to pixel coordinates""" - n = 2.0 ** zoom - x = (lon + 180.0) / 360.0 * n - y = (1 - math.asinh(math.tan(math.radians(lat))) / math.pi) / 2 * n - map_px = x * TILE_SIZE - map_py = y * TILE_SIZE - return map_px, map_py - -def decimal_to_gms(decimal, is_latitude=True): - sign = "" - if is_latitude: - sign = "N" if decimal >= 0 else "S" - else: - sign = "E" if decimal >= 0 else "W" - decimal = abs(decimal) - degrees = int(decimal) - minutes_full = (decimal - degrees) * 60 - minutes = int(minutes_full) - seconds = (minutes_full - minutes) * 60 - return f"{degrees}°{minutes}'{seconds:.2f}\" {sign}" - -def main(base_dir): - # Try to load palette from palette.bin first, then fallback to features.json - palette_file = "palette.bin" - config_file = "features.json" - - if os.path.exists(palette_file): - if load_global_palette_from_bin(palette_file): - logger.info("Loaded palette from palette.bin") - else: - logger.warning("Failed to load palette.bin, trying features.json") - if os.path.exists(config_file): - load_global_palette_from_config(config_file) - else: - logger.info(f"Features file {config_file} not found, using fallback colors") - elif os.path.exists(config_file): - load_global_palette_from_config(config_file) - else: - logger.info(f"Features file {config_file} not found, using fallback colors") - - zoom_dirs = [d for d in os.listdir(base_dir) if os.path.isdir(os.path.join(base_dir, d)) and d.isdigit()] - zoom_levels_list = sorted([int(d) for d in zoom_dirs]) - if not zoom_levels_list: - logger.error(f"No zoom level directories found in {base_dir}") - sys.exit(1) - min_zoom = zoom_levels_list[0] - max_zoom = zoom_levels_list[-1] - zoom_levels = list(range(min_zoom, max_zoom+1)) - zoom_idx = 0 - - background_color = (0, 0, 0) - button_color = (0, 0, 0) - button_fg = (255, 255, 255) - button_border = (100,100,100) - - toolbar_x = VIEWPORT_SIZE - toolbar_y = 0 - button_height = 40 - button_margin = 16 - button_rect = pygame.Rect(toolbar_x + 30, toolbar_y + button_margin, 100, button_height) - tile_label_button_rect = pygame.Rect(toolbar_x + 30, toolbar_y + button_margin * 2 + button_height, 100, button_height) - gps_button_rect = pygame.Rect(toolbar_x + 30, toolbar_y + button_margin * 3 + button_height * 2, 100, button_height) - fill_button_rect = pygame.Rect(toolbar_x + 30, toolbar_y + button_margin * 4 + button_height * 3, 100, button_height) - - button_text_black = "Black" - button_text_white = "White" - button_pressed = False - - show_tile_labels = False - show_gps_tooltip = False - - pygame.init() - screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT)) - pygame.display.set_caption(f"Tile viewer - MAP {os.path.basename(base_dir)}") - font = pygame.font.SysFont(None, 16) - font_main = pygame.font.SysFont(None, 18) - font_b = pygame.font.SysFont(None, 16) - font_status = pygame.font.SysFont(None, 14) - clock = pygame.time.Clock() - - available_tiles = set() - - icon_bg, icon_label, icon_gps, icon_fill = get_button_icons() - - mouse_gps_coords = None - mouse_gps_rect = None - - show_index_progress = False - index_progress_percent = 0.0 - index_progress_text = "" - index_progress_done_drawn = False - show_render_progress = False - render_progress_percent = 0.0 - render_progress_text = "" - render_progress_done_drawn = False - - tiles_loading = False - tiles_loading_lock = threading.Lock() - need_redraw = True - zoom_change_pending = False - zoom_change_params = None - - def status_index_progress_callback(percent, text): - nonlocal show_index_progress, index_progress_percent, index_progress_text, need_redraw - show_index_progress = True - index_progress_percent = percent - index_progress_text = text - need_redraw = True - - def hide_index_progress(): - nonlocal show_index_progress, index_progress_percent, index_progress_text, need_redraw, index_progress_done_drawn - if index_progress_done_drawn: - show_index_progress = False - index_progress_percent = 0.0 - index_progress_text = "" - index_progress_done_drawn = False - need_redraw = True - - def status_render_progress_callback(percent, text): - nonlocal show_render_progress, render_progress_percent, render_progress_text, need_redraw - show_render_progress = True - render_progress_percent = percent - render_progress_text = text - need_redraw = True - - def hide_render_progress(): - nonlocal show_render_progress, render_progress_percent, render_progress_text, need_redraw, render_progress_done_drawn - if render_progress_done_drawn: - show_render_progress = False - render_progress_percent = 0.0 - render_progress_text = "" - render_progress_done_drawn = False - need_redraw = True - - def load_available_tiles(level, progress_callback=None): - directory = os.path.join(base_dir, str(level)) - available = index_available_tiles(directory, progress_callback) - return available, directory - - def get_tile_surface(x, y, zoom_level, directory, bg_color, fill_mode, show_tile_labels=False): - """Get tile surface with lazy loading - only load if visible""" - key = (zoom_level, x, y, bg_color, fill_mode) - if (x, y) not in available_tiles: - return None - - # Try to get from cache first - cached_surface = tile_cache.get(key) - if cached_surface is not None: - return cached_surface - - # Only load if tile is visible (lazy loading) - if not is_tile_visible(x, y, viewport_x, viewport_y): - logger.debug(f"Skipping lazy load for non-visible tile ({x}, {y})") - return None - - # Load and cache the surface - tile_file = get_tile_file(directory, x, y) - if not tile_file: - return None - - surface = render_tile_surface({'x': x, 'y': y, 'file': tile_file}, bg_color, fill_mode, show_tile_labels) - tile_cache.put(key, surface) - return surface - - def preload_tile_surfaces_threaded(tile_list, zoom_level, directory, bg_color, fill_mode, progress_callback=None, done_callback=None): - """Preload tiles using the persistent thread pool""" - total = len(tile_list) - completed_count = 0 - - def progress_callback_wrapper(future): - nonlocal completed_count - completed_count += 1 - if progress_callback is not None: - percent = completed_count / max(total, 1) - progress_callback(percent, "Loading visible tiles...") - - if completed_count >= total and done_callback: - done_callback() - - # Submit all tiles to the persistent thread pool - for x, y in tile_list: - tile_info = (x, y, zoom_level, directory, bg_color, fill_mode) - tile_loader.submit_tile_load(tile_info, progress_callback_wrapper) - - return None # No thread to return since we use persistent pool - - def set_tiles_loading(flag): - nonlocal tiles_loading, need_redraw - with tiles_loading_lock: - tiles_loading = flag - need_redraw = True - - def start_zoom_change(idx, last_mouse_pos, viewport_x, viewport_y, old_zoom_idx): - nonlocal zoom_change_pending, zoom_change_params - zoom_change_pending = True - zoom_change_params = (idx, last_mouse_pos, viewport_x, viewport_y, old_zoom_idx) - - show_index_progress = True - index_progress_percent = 0.0 - index_progress_text = "Indexing initial tiles..." - available_tiles, directory = load_available_tiles(zoom_levels[zoom_idx], status_index_progress_callback) - hide_index_progress() - viewport_x, viewport_y = center_viewport_on_central_tile(available_tiles) - - dragging = False - drag_start = None - running = True - last_mouse_pos = (VIEWPORT_SIZE // 2, VIEWPORT_SIZE // 2) - - while running: - if zoom_change_pending and not tiles_loading: - idx, last_mouse_pos_z, old_vx, old_vy, old_zoom_idx = zoom_change_params - show_index_progress = True - index_progress_percent = 0.0 - index_progress_text = "Indexing tiles..." - index_progress_done_drawn = False - available_tiles, directory = load_available_tiles(zoom_levels[idx], status_index_progress_callback) - if idx > old_zoom_idx: - lat, lon = pixel_to_latlon(last_mouse_pos_z[0], last_mouse_pos_z[1], old_vx, old_vy, zoom_levels[idx-1]) - else: - lat, lon = pixel_to_latlon(last_mouse_pos_z[0], last_mouse_pos_z[1], old_vx, old_vy, zoom_levels[idx+1]) - map_px, map_py = latlon_to_pixel(lat, lon, zoom_levels[idx]) - viewport_x, viewport_y = int(map_px - last_mouse_pos_z[0]), int(map_py - last_mouse_pos_z[1]) - viewport_x, viewport_y = clamp_viewport(viewport_x, viewport_y, available_tiles) - zoom_idx = idx - zoom_change_pending = False - need_redraw = True - logger.info(f"Changed to zoom level {zoom_levels[zoom_idx]}") - - mx, my = pygame.mouse.get_pos() - if show_gps_tooltip and 0 <= mx < VIEWPORT_SIZE and 0 <= my < VIEWPORT_SIZE: - lat, lon = pixel_to_latlon(mx, my, viewport_x, viewport_y, zoom_levels[zoom_idx]) - mouse_gps_coords = (lat, lon) - mouse_gps_rect = (mx, my) - else: - mouse_gps_coords = None - mouse_gps_rect = None - - screen.fill((70,70,70)) - pygame.draw.rect(screen, background_color, (0,0,VIEWPORT_SIZE,VIEWPORT_SIZE)) - - if available_tiles: - xs = [x for x, y in available_tiles] - ys = [y for x, y in available_tiles] - min_x = max_x = min_y = max_y = 0 - if xs and ys: - min_x, max_x = min(xs), max(xs) - min_y, max_y = min(ys), max(ys) - else: - min_x = max_x = min_y = max_y = 0 - - # Get visible tiles using the new function - visible_tiles = get_visible_tiles(available_tiles, viewport_x, viewport_y) - - # Only check for uncached tiles among visible ones (lazy loading) - uncached_tiles = [] - fill_polygons = config.config.get('fill_polygons', False) - for x, y in visible_tiles: - key = (zoom_levels[zoom_idx], x, y, background_color, fill_polygons) - if tile_cache.get(key) is None: - uncached_tiles.append((x, y)) - - if uncached_tiles and not tiles_loading: - set_tiles_loading(True) - show_render_progress = True - render_progress_percent = 0.0 - render_progress_text = "Loading visible tiles..." - def done_callback(): - set_tiles_loading(False) - preload_tile_surfaces_threaded( - uncached_tiles, zoom_levels[zoom_idx], directory, background_color, fill_polygons, - status_render_progress_callback, done_callback - ) - - for x, y in visible_tiles: - surf = get_tile_surface(x, y, zoom_levels[zoom_idx], directory, background_color, fill_polygons, show_tile_labels) - if surf: - px = x * TILE_SIZE - viewport_x - py = y * TILE_SIZE - viewport_y - screen.blit(surf, (px, py)) - - draw_tile_labels( - screen, font, available_tiles, viewport_x, viewport_y, zoom_levels[zoom_idx], background_color, show_tile_labels, directory, fill_polygons - ) - pygame.draw.rect(screen, (0,0,0), (toolbar_x, toolbar_y, TOOLBAR_WIDTH, VIEWPORT_SIZE)) - pygame.draw.line(screen, (160,160,160), (toolbar_x,0), (toolbar_x, VIEWPORT_SIZE)) - draw_button( - screen, - button_text_black if background_color == (255, 255, 255) else button_text_white, - button_rect, button_color, button_fg, button_border, font_b, - icon=icon_bg, pressed=button_pressed - ) - label_btn_text = "Tile labels ON" if show_tile_labels else "Tile labels OFF" - draw_button( - screen, label_btn_text, tile_label_button_rect, button_color, button_fg, button_border, font_b, - icon=icon_label, pressed=False - ) - gps_btn_text = "GPS Cursor ON" if show_gps_tooltip else "GPS Cursor OFF" - draw_button( - screen, gps_btn_text, gps_button_rect, button_color, button_fg, button_border, font_b, - icon=icon_gps, pressed=False - ) - - # Polygon fill button - fill_polygons = config.config.get('fill_polygons', False) - fill_btn_text = "Fill Polygons ON" if fill_polygons else "Fill Polygons OFF" - draw_button( - screen, fill_btn_text, fill_button_rect, button_color, button_fg, button_border, font_b, - icon=icon_fill, pressed=False - ) - pygame.draw.rect(screen, (0,0,0), (0, VIEWPORT_SIZE, WINDOW_WIDTH, STATUSBAR_HEIGHT)) - pygame.draw.line(screen, (160,160,160), (0, VIEWPORT_SIZE), (WINDOW_WIDTH, VIEWPORT_SIZE)) - zoom_text = f"Zoom level: {zoom_levels[zoom_idx]}" - zoom_img = font_status.render(zoom_text, True, (255,255,255)) - screen.blit(zoom_img, (16, VIEWPORT_SIZE + STATUSBAR_HEIGHT//2 - zoom_img.get_height()//2)) - - if show_gps_tooltip and mouse_gps_coords is not None: - lat, lon = mouse_gps_coords[0], mouse_gps_coords[1] - lat_gms = decimal_to_gms(lat, is_latitude=True) - lon_gms = decimal_to_gms(lon, is_latitude=False) - txt = f"lat: {lat:.6f} ({lat_gms})\nlon: {lon:.6f} ({lon_gms})" - tooltip_lines = txt.split('\n') - tooltip_surfs = [font.render(line, True, (255,255,255)) for line in tooltip_lines] - tw = max(s.get_width() for s in tooltip_surfs) - th = sum(s.get_height() for s in tooltip_surfs) - tm = 4 - mx, my = mouse_gps_rect[:2] - tooltip_rect = pygame.Rect(mx+10, my+10, tw+tm*2, th+tm*2) - pygame.draw.rect(screen, (0,0,0), tooltip_rect) - pygame.draw.rect(screen, (200,200,200), tooltip_rect, 1) - yoff = tooltip_rect.top + tm - for surf in tooltip_surfs: - screen.blit(surf, (tooltip_rect.left + tm, yoff)) - yoff += surf.get_height() - - if show_index_progress: - show_status_progress_bar(screen, index_progress_percent, index_progress_text, font_main) - if index_progress_percent >= 1.0: - index_progress_done_drawn = True - else: - index_progress_done_drawn = False - if show_render_progress: - show_status_progress_bar(screen, render_progress_percent, render_progress_text, font_main) - if render_progress_percent >= 1.0 and not uncached_tiles and not tiles_loading: - render_progress_done_drawn = True - else: - render_progress_done_drawn = False - - pygame.display.flip() - need_redraw = False - - # Cleanup completed futures periodically - completed_futures = tile_loader.cleanup_completed_futures() - if completed_futures > 0: - logger.debug(f"Cleaned up {completed_futures} completed futures") - - if index_progress_done_drawn: - hide_index_progress() - if render_progress_done_drawn: - hide_render_progress() - - can_interact = not show_index_progress and not show_render_progress and not tiles_loading - - for event in pygame.event.get(): - if event.type == pygame.QUIT: - running = False - if event.type == pygame.KEYDOWN: - base_step = VIEWPORT_SIZE // 4 - zoom_factor = 1 + ((zoom_levels[zoom_idx] - min_zoom) * 0.23) if zoom_levels[zoom_idx] > min_zoom else 1 - step = int(base_step * zoom_factor) - if event.key == pygame.K_LEFT and can_interact: - viewport_x = max(min_x * TILE_SIZE, viewport_x - step) - need_redraw = True - elif event.key == pygame.K_RIGHT and can_interact: - viewport_x = min(viewport_x + step, (max_x * TILE_SIZE + TILE_SIZE) - VIEWPORT_SIZE) - need_redraw = True - elif event.key == pygame.K_UP and can_interact: - viewport_y = max(min_y * TILE_SIZE, viewport_y - step) - need_redraw = True - elif event.key == pygame.K_DOWN and can_interact: - viewport_y = min(viewport_y + step, (max_y * TILE_SIZE + TILE_SIZE) - VIEWPORT_SIZE) - need_redraw = True - elif event.key == pygame.K_LEFTBRACKET and can_interact and not zoom_change_pending: - if zoom_idx > 0: - start_zoom_change(zoom_idx-1, last_mouse_pos, viewport_x, viewport_y, zoom_idx) - elif event.key == pygame.K_RIGHTBRACKET and can_interact and not zoom_change_pending: - if zoom_idx < len(zoom_levels) - 1: - start_zoom_change(zoom_idx+1, last_mouse_pos, viewport_x, viewport_y, zoom_idx) - elif event.key == pygame.K_l and can_interact: - show_tile_labels = not show_tile_labels - need_redraw = True - elif event.key == pygame.K_f and can_interact: - # Toggle polygon filling - config.config['fill_polygons'] = not config.config.get('fill_polygons', False) - config.save_config() - tile_cache.clear() # Clear cache to force redraw with new fill setting - need_redraw = True - logger.info(f"Polygon filling: {'ON' if config.config['fill_polygons'] else 'OFF'}") - elif event.type == pygame.MOUSEBUTTONDOWN: - if event.button == 1 and can_interact: - if tile_label_button_rect.collidepoint(event.pos): - show_tile_labels = not show_tile_labels - need_redraw = True - elif button_rect.collidepoint(event.pos): - background_color = (255, 255, 255) if background_color == (0, 0, 0) else (0, 0, 0) - need_redraw = True - logger.info(f"Background color changed to {'white' if background_color == (255, 255, 255) else 'black'}") - elif gps_button_rect.collidepoint(event.pos): - show_gps_tooltip = not show_gps_tooltip - need_redraw = True - elif fill_button_rect.collidepoint(event.pos): - # Toggle polygon filling - config.config['fill_polygons'] = not config.config.get('fill_polygons', False) - config.save_config() - tile_cache.clear() # Clear cache to force redraw with new fill setting - need_redraw = True - logger.info(f"Polygon filling: {'ON' if config.config['fill_polygons'] else 'OFF'}") - else: - dragging = True - drag_start = event.pos - drag_viewport_start = (viewport_x, viewport_y) - elif event.button == 4 and can_interact and not zoom_change_pending: - if zoom_idx < len(zoom_levels) - 1: - start_zoom_change(zoom_idx+1, last_mouse_pos, viewport_x, viewport_y, zoom_idx) - elif event.button == 5 and can_interact and not zoom_change_pending: - if zoom_idx > 0: - start_zoom_change(zoom_idx-1, last_mouse_pos, viewport_x, viewport_y, zoom_idx) - elif event.type == pygame.MOUSEBUTTONUP: - if event.button == 1: - dragging = False - drag_start = None - elif event.type == pygame.MOUSEMOTION: - mx, my = event.pos - last_mouse_pos = (mx, my) - if dragging: - dx = drag_start[0] - event.pos[0] - dy = drag_start[1] - event.pos[1] - viewport_x = drag_viewport_start[0] + dx - viewport_y = drag_viewport_start[1] + dy - viewport_x, viewport_y = clamp_viewport(viewport_x, viewport_y, available_tiles) - need_redraw = True - - clock.tick(config.get('fps_limit', 30)) - - # Shutdown thread pool before quitting - tile_loader.shutdown() - pygame.quit() - -if __name__ == "__main__": - if len(sys.argv) < 2: - logger.info("Usage: python tile_viewer.py VECTORMAP") - logger.info("Keys: [arrows] move, [ ] [ ] zoom level, mouse scroll: zoom level") - logger.info("Mouse: drag to pan, buttons for background, tile labels, GPS cursor, fill polygons. [l] toggle labels") - logger.info("Example: python tile_viewer.py VECTORMAP") - logger.info("Note: Place features.json in current directory for dynamic palette support") - sys.exit(1) - main(sys.argv[1]) \ No newline at end of file