mb-util

#!/usr/bin/env python

# MBUtil: a tool for MBTiles files
# Supports importing, exporting, and more
#
# (c) Development Seed 2012
# Licensed under BSD

import logging, os, sys
from optparse import OptionParser, OptionGroup

from mbutil import mbtiles_to_disk, disk_to_mbtiles, mbtiles_create, merge_mbtiles, optimize_database, check_mbtiles, clean_mbtiles, test_mbtiles, fill_mbtiles, execute_commands_on_mbtiles, convert_string, mbtiles_tilelist, expire_mbtiles, expire_tiles_bbox, update_mbtiles

if __name__ == '__main__':

    logging.basicConfig(level=logging.INFO)

    parser = OptionParser(usage="""usage: %prog [command] [options] file|directory [file|directory ...]

    Examples:

    Export an mbtiles database to a directory of files:
    $ mb-util --export world.mbtiles tiles

    Import a directory of tiles into an mbtiles database:
    $ mb-util --import tiles world.mbtiles

    Create an empty mbtiles file:
    $ mb-util --create empty.mbtiles

    Execute commands on all tiles in the mbtiles file:
    $ mb-util --process --execute "COMMAND ARGUMENTS" [--execute "SECOND COMMAND"] world.mbtiles

    Merge two or more mbtiles files (receiver will be the first file):
    $ mb-util --merge receiver.mbtiles file1.mbtiles [file2.mbtiles ...]

    Remove tiles older than 30 days from the database:
    $ mb-util --expire=30 world.mbtiles

    Fill a database with a given tile image
    $ mb-util --fill --min-zoom=7 --max-zoom=12 world.mbtiles transparent.png

    Check if a mbtiles file contains all tiles at a specific zoom level:
    $ mb-util --check --zoom=7 world.mbtiles

    Clean a database (remove unused images, etc.):
    $ mb-util --clean world.mbtiles

    Test tiles with a command, print tile coordinates for non-zero return values
    $ mb-util --test --execute "COMMAND ARGUMENTS" world.mbtiles

    Dumps a list of tiles to the console:
    $ mb-util --tilelist --as-bboxes --zoom=11 world.mbtiles

    Convert tile coordinates and bounding boxes:
    $ mb-util --convert="13/4328/2861"
    $ mb-util --convert="10.195312,47.546872,10.239258,47.576526" --min-zoom=12 --max-zoom=13
    """)

    group = OptionGroup(parser, "Commands", "These are the commands to use on mbtiles databases")

    group.add_option("-e", "--export",
        dest='export_tiles', action="store_true",
        help='''Export an mbtiles database to a directory of files. If the directory exists, any already existing tiles will be overwritten.''',
        default=False)

    group.add_option("-i", "--import",
        dest='import_tiles', action="store_true",
        help='''Import a directory of tiles into an mbtiles database. If the mbtiles database already exists, existing tiles will be overwritten with the imported tiles.''',
        default=False)

    group.add_option("-m", "--merge",
        dest='merge_tiles', action="store_true",
        help='''Merge two or more databases. The receiver will be created if it doesn\'t yet exist.''',
        default=False)

    group.add_option("-u", "--update",
        dest='update_tiles', action="store_true",
        help='''Update one database from another, based on updated_at timestamps. The receiver will be created if it doesn\'t yet exist.''',
        default=False)

    group.add_option("-p", "--process",
        action="store_true", dest="process", default=False,
        help='''Processes a mbtiles databases. Only usefull together with one or more --execute.''')

    group.add_option("--expire", metavar="DAYS",
        dest="expire", type="int", default="0",
        help='''Remove tiles older than DAYS from the database.''')

    group.add_option("--expire-tiles",
        dest='expire_tiles', action="store_true",
        help='''Remove tiles within a bounding box from the database (must be used with --bbox/--tile-bbox and --zoom/--min-zoom/--max-zoom).''',
        default=False)

    group.add_option("--check",
        dest='check', action="store_true",
        help='''Check the database for missing tiles.''',
        default=False)

    group.add_option("--clean",
        dest='clean', action="store_true",
        help='''Remove unused images from the database.''',
        default=False)

    group.add_option("--test",
        dest='test', action="store_true",
        help='''Test every tile with the given command, print the tile coordinate if the command returns anything non-zero.''',
        default=False)

    group.add_option("--fill",
        dest='fill', action="store_true",
        help='''Fill a database with tile images where it doesn\'t already contain tiles. Only usefull with --min-zoom/--max-zoom and --tile-bbox/--bbox.''',
        default=False)

    group.add_option("--create",
        action="store_true", dest="create", default=False,
        help='''Create an empty mbtiles database.''')

    group.add_option("--tilelist",
        action="store_true", dest="tilelist", default=False,
        help='''Dumps a list of tiles to the console.''')

    group.add_option('--convert',
        dest='convert', type="string", default=None,
        help='''Convert tile coordinates 'y/x/z' to bounding box 'left,bottom,right,top' or vice versa.''')

    parser.add_option_group(group)

    group = OptionGroup(parser, "Options", "")

    group.add_option("--execute",
        dest="command_list", type="string", metavar="COMMAND",
        action="append", default=None,
        help='''Commands to execute for each tile image. %s will be replaced with the file name. This argument may be repeated several times and can be used together with --import/--export/--merge/--compact/--process.''')

    group.add_option("--scale", dest="tile_scale",
        help='''The scale factor for the tiles (1 or 2). Default is to work on all available tiles.''',
        type="int", default=None)

    group.add_option('--flip-y', dest='flip_y',
        help='''Flip the y tile coordinate during --export/--import/--merge/--update/--convert/--tilelist.''',
        action="store_true", default=False)

    group.add_option('--min-zoom', dest='min_zoom',
        help='''Minimum zoom level for --export/--import/--merge/--update/--process/--check/--convert.''',
        type="int", default=0)

    group.add_option('--max-zoom', dest='max_zoom',
        help='''Maximum zoom level for --export/--import/--merge/--update/--process/--check/--convert.''',
        type="int", default=18)

    group.add_option('--zoom', dest='zoom',
        help='''Zoom level for --export/--import/--merge/--update/--process/--check/--convert. (Overrides --min-zoom and --max-zoom)''',
        type='int', default=-1)

    group.add_option('--min-timestamp', dest='min_timestamp',
        help='''Minimum numerical timestamp for --export/--merge/--process/--test.''',
        type="long", default=0)

    group.add_option('--max-timestamp', dest='max_timestamp',
        help='''Maximum numerical timestamp for --export/--merge/--process/--test.''',
        type="long", default=0)

    group.add_option('--bbox', dest='bbox',
        help='''Bounding box in coordinates 'left,bottom,right,top' (10.195312,47.546872,10.239258,47.576526)''',
        type='string', default=None)

    group.add_option('--tile-bbox', dest='tile_bbox',
        help='''Bounding box in tile coordinates 'left,bottom,right,top' (10,10,20,20). Can only be used with --zoom.''',
        type='string', default=None)

    group.add_option("--revert-test",
        action="store_true", dest="revert_test", default=False,
        help='''For --test, print the tile coordinates if the command returns zero.''')

    group.add_option("--as-bboxes",
        action="store_true", dest="as_bboxes", default=False,
        help='''For --tilelist, print the bounding boxes for tiles.''')

    group.add_option("--auto-commit",
        action="store_true", dest="auto_commit", default=False,
        help='''Enable auto commit.''')

    group.add_option("--synchronous-off",
        action="store_true", dest="synchronous_off", default=False,
        help='''DANGEROUS!!! Set synchronous=OFF for SQLite database connections.''')

    group.add_option("--use-wal-journal",
        action="store_true", dest="wal_journal", default=False,
        help='''Use journal_mode=WAL for SQLite databases. [DEPRECATED]''')

    group.add_option("--journal_mode",
        type="string", default="wal",
        help='''SQLite journal mode to use (wal|delete|memory|truncate|persist|off). Defaults to 'wal'.''')

    group.add_option("--check-before-merge",
        action="store_true", dest="check_before_merge", default=False,
        help='''Runs some basic checks (like --check) on databases before merging them.''')

    group.add_option("--delete-after-export",
        action="store_true", dest="delete_after_export", default=False,
        help='''DANGEROUS!!! After a --merge or --export, this option will delete all the merged/exported tiles from the (sending) database. Only really usefull with --min-zoom/--max-zoom or --zoom since it would remove all tiles from the database otherwise.''')

    group.add_option("--delete-vanished-tiles",
        action="store_true", dest="delete_vanished_tiles", default=False,
        help='''DANGEROUS!!! If a tile vanishes during --execute then delete it also from the database or ignore it during --merge/--process.''')

    group.add_option("--poolsize",
        type="int", default=-1,
        help="""Pool size for processing tiles with --process/--merge. Default is to use a pool size equal to the number of cpu cores.""")

    group.add_option('--tmp-dir',
        dest='tmp_dir', type="string", default=None,
        help='''Temporary directory to use for --execute (e.g. /dev/shm).''')

    group.add_option("--vacuum",
        action="store_false", dest="skip_vacuum", default=True,
        help='''VACUUM the database after --import/--merge/--process/--fill/--expire.''')

    group.add_option("--analyze",
        action="store_false", dest="skip_analyze", default=True,
        help='''ANALYZE the database after --import/--merge/--process/--fill/--expire.''')

    group.add_option("--progress",
        action="store_true", dest="progress", default=False,
        help='''Print progress updates and keep them on one line.''')

    group.add_option("-q", "--quiet",
        action="store_true", dest="quiet", default=False,
        help='''don't print any status messages to stdout except errors.''')

    group.add_option("-d", "--debug",
        action="store_true", dest="debug", default=False,
        help='''print debug messages to stdout (exclusive to --quiet).''')

    parser.add_option_group(group)

    (options, args) = parser.parse_args()

    # Transfer operations
    if len(args) == 0:
        if options.convert:
            convert_string(options.convert, **options.__dict__)
            sys.exit(0)

        parser.print_help()
        sys.exit(1)

    if options.quiet:
        logging.getLogger().setLevel(logging.ERROR)
    elif options.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    logger = logging.getLogger(__name__)


    if options.wal_journal:
        options.journal_mode = "wal";

    options.journal_mode = options.journal_mode.lower()

    if options.journal_mode not in [ 'wal', 'delete', 'memory', 'truncate', 'persist', 'off' ]:
        options.journal_mode = 'wal'

    logger.debug("Using journal_mode=%s" % (options.journal_mode))

    if options.auto_commit:
        logger.debug("Using auto commit (isolation_level = None)")
    if options.flip_y:
        logger.debug("Flipping the y coordinate")

    if options.tmp_dir:
        logger.debug("Using tmp dir: %s" % (options.tmp_dir, ))

    if options.tile_scale not in [None, 1, 2]:
        sys.stderr.write('Tile scale must be 1 or 2.\n')
        sys.exit(1)

    if len(args) == 1:
        # Check the mbtiles db?
        if options.check:
            result = check_mbtiles(args[0], **options.__dict__)
            sys.exit(0) if result else sys.exit(1)

        # Clean the mbtiles db?
        if options.clean:
            result = clean_mbtiles(args[0], **options.__dict__)
            sys.exit(0) if result else sys.exit(1)

        # Execute commands on the tiles in the mbtiles db?
        if options.process:
            if options.command_list == None:
                sys.stderr.write('Need at least one command to execute for each tile.\n')
                sys.exit(1)
            execute_commands_on_mbtiles(args[0], **options.__dict__)
            sys.exit(0)

        if options.test:
            if options.command_list == None or len(options.command_list) != 1:
                sys.stderr.write('Need exactly one command to execute for each tile.\n')
                sys.exit(1)
            test_mbtiles(args[0], **options.__dict__)
            sys.exit(0)

        if options.tilelist:
            mbtiles_tilelist(args[0], **options.__dict__)
            sys.exit(0)

        if options.expire > 0:
            expire_mbtiles(args[0], **options.__dict__)
            sys.exit(0)

        if options.expire_tiles:
            expire_tiles_bbox(args[0], **options.__dict__)
            sys.exit(0)

        # Create an empty mbtiles db?
        if options.create:
            mbtiles_create(args[0], **options.__dict__)
            sys.exit(0)

        sys.stderr.write("No command given, don't know what to do. Exiting...\n")
        sys.exit(0)

    if len(args) == 2:
        if options.fill:
            if not os.path.isfile(args[1]):
                sys.stderr.write('The tile image file must exist.\n')
                sys.exit(1)
            fill_mbtiles(args[0], args[1], **options.__dict__)
            sys.exit(0)

        if options.update_tiles:
            update_mbtiles(args[0], args[1], **options.__dict__)
            optimize_database(args[0], options.auto_commit, options.skip_analyze, options.skip_vacuum, options.journal_mode)
            sys.exit(0)

    # merge mbtiles files
    if options.merge_tiles:
        receiving_mbtiles = args[0]
        for n in range(1, len(args)):
            other_mbtiles = args[n]
            if not options.quiet:
                logging.info("%d: Merging %s" % (n, other_mbtiles))
            merge_mbtiles(receiving_mbtiles, other_mbtiles, **options.__dict__)

        optimize_database(args[0], options.auto_commit, options.skip_analyze, options.skip_vacuum, options.journal_mode)
        sys.exit(0)

    # export from mbtiles to disk
    if options.export_tiles:
        mbtiles_file, directory_path = args
        mbtiles_to_disk(mbtiles_file, directory_path, **options.__dict__)
        sys.exit(0)

    # import from disk to mbtiles
    if options.import_tiles:
        if not os.path.isdir(args[0]):
            sys.stderr.write('The directory to import from must exist.\n')
            sys.exit(1)

        directory_path, mbtiles_file = args
        disk_to_mbtiles(directory_path, mbtiles_file, **options.__dict__)
        sys.exit(0)