Merge pull request #184 from divio/feature/improvements

Various improvements and fixes

divio_cli/cli.py

@@ -19,8 +19,8 @@
 from .cloud import CloudClient, get_endpoint
 from .check_system import check_requirements, check_requirements_human
 from .utils import (
-    hr, table, open_project_cloud_site, get_dashboard_url,
-    get_project_cheatsheet_url, get_git_checked_branch,
+    hr, table, open_project_cloud_site,
+    get_cp_url, get_git_checked_branch,
     print_package_renamed_warning,
 )
 from .validators.addon import validate_addon
@@ -196,7 +196,7 @@ def project_deploy(obj, stage):
 @click.pass_obj
 def project_dashboard(obj):
     """Open project dashboard"""
-    click.launch(get_dashboard_url(obj))
+    click.launch(get_cp_url(obj))
 
 
 @project.command(name='up')
@@ -252,7 +252,7 @@ def project_stop(obj):
 @click.pass_obj
 def project_cheatsheet(obj):
     """Show useful commands for your project"""
-    click.launch(get_project_cheatsheet_url(obj))
+    click.launch(get_cp_url(obj, 'local-development/'))
 
 
 @project.command(name='setup')

divio_cli/config.py

@@ -3,16 +3,19 @@
 import os
 from distutils.version import StrictVersion
 
-from . import utils, __version__
+from . import utils, __version__, settings
+
+
+CONFIG_FILE_NAME = settings.ALDRYN_DOT_FILE
+CONFIG_FILE_PATH = os.path.join(os.path.expanduser('~'), CONFIG_FILE_NAME)
 
 
 class Config(object):
-    config_name = '.aldryn'
+    config_path = CONFIG_FILE_PATH
+    config = {}
 
     def __init__(self):
         super(Config, self).__init__()
-        home = os.path.expanduser('~')
-        self.config_path = os.path.join(home, self.config_name)
         self.read()
 
     def read(self):

divio_cli/localdev/main.py

@@ -67,10 +67,16 @@ def configure_project(website_slug, path, client):
 
 def setup_website_containers(client, stage, path):
     docker_compose = utils.get_docker_compose_cmd(path)
+    docker_compose_config = utils.DockerComposeConfig(docker_compose)
 
-    existing_db_container_id = utils.get_db_container_id(path, False)
+    if docker_compose_config.has_service('db'):
+        has_db_service = True
+        existing_db_container_id = utils.get_db_container_id(path, False)
+    else:
+        has_db_service = False
+        existing_db_container_id = None
 
-    # stop all running for project
+    # stop all running containers for project
     check_call(docker_compose('stop'))
 
     # pull docker images
@@ -86,19 +92,20 @@ def setup_website_containers(client, stage, path):
         check_call(docker_compose('stop', 'db'))
         check_call(docker_compose('rm', '-f', 'db'))
 
-    click.secho('creating new database container', fg='green')
-    ImportRemoteDatabase(client=client, stage=stage, path=path)()
+    if has_db_service:
+        click.secho('creating new database container', fg='green')
+        ImportRemoteDatabase(client=client, stage=stage, path=path)()
 
-    click.secho('sync and migrate database', fg='green')
+        click.secho('syncing and migrating database', fg='green')
 
-    if is_windows():
-        # interactive mode is not yet supported with docker-compose
-        # on windows. that's why we have to call it as daemon
-        # and just wait a sane time
-        check_call(docker_compose('run', '-d', 'web', 'start', 'migrate'))
-        sleep(30)
-    else:
-        check_call(docker_compose('run', 'web', 'start', 'migrate'))
+        if is_windows():
+            # interactive mode is not yet supported with docker-compose
+            # on windows. that's why we have to call it as daemon
+            # and just wait a sane time
+            check_call(docker_compose('run', '-d', 'web', 'start', 'migrate'))
+            sleep(30)
+        else:
+            check_call(docker_compose('run', 'web', 'start', 'migrate'))
 
 
 def create_workspace(client, website_slug, stage, path=None, force_overwrite=False):
@@ -173,6 +180,10 @@ def __init__(self, *args, **kwargs):
         self.website_id = utils.get_aldryn_project_settings(self.path)['id']
         self.website_slug = utils.get_aldryn_project_settings(self.path)['slug']
         self.docker_compose = utils.get_docker_compose_cmd(self.path)
+        docker_compose_config = utils.DockerComposeConfig(self.docker_compose)
+        if not docker_compose_config.has_service('db'):
+            click.secho('No service "db" found in local project', fg='red')
+            sys.exit(1)
 
         self.start_time = time()
 
@@ -368,9 +379,18 @@ def get_db_restore_command(self):
 
 def pull_media(client, stage, path=None):
     project_home = utils.get_project_home(path)
-    path = os.path.join(project_home, 'data', 'media')
-    website_id = utils.get_aldryn_project_settings(path)['id']
-    website_slug = utils.get_aldryn_project_settings(path)['slug']
+    website_id = utils.get_aldryn_project_settings(project_home)['id']
+    website_slug = utils.get_aldryn_project_settings(project_home)['slug']
+
+    docker_compose = utils.get_docker_compose_cmd(project_home)
+    docker_compose_config = utils.DockerComposeConfig(docker_compose)
+
+    local_data_folder = os.path.join(project_home, 'data')
+    remote_data_folder = '/data'
+
+    if not docker_compose_config.has_volume_mount('web', remote_data_folder):
+        click.secho('No mount for /data folder found')
+        return
 
     click.secho(
         ' ===> Pulling media files from {} {} server'.format(
@@ -406,30 +426,31 @@ def pull_media(client, stage, path=None):
         return
     click.echo(' [{}s]'.format(int(time() - start_download)))
 
-    if os.path.isdir(path):
+    media_path = os.path.join(local_data_folder, 'media')
+
+    if os.path.isdir(media_path):
         start_remove = time()
         click.secho(' ---> Removing local files', nl=False)
-        shutil.rmtree(path)
+        shutil.rmtree(media_path)
         click.echo(' [{}s]'.format(int(time() - start_remove)))
 
     if 'linux' in sys.platform:
         # On Linux, Docker typically runs as root, so files and folders
         # created from within the container will be owned by root. As a
         # workaround, make the folder permissions more permissive, to
         # allow the invoking user to create files inside it.
-        docker_compose = utils.get_docker_compose_cmd(project_home)
         check_call(
             docker_compose(
                 'run', '--rm', 'web',
-                'chown', '-R', str(os.getuid()), 'data'
+                'chown', '-R', str(os.getuid()), remote_data_folder
             )
         )
 
-    click.secho(' ---> Extracting files to {}'.format(path), nl=False)
+    click.secho(' ---> Extracting files to {}'.format(media_path), nl=False)
     start_extract = time()
     with open(backup_path, 'rb') as fobj:
         with tarfile.open(fileobj=fobj, mode='r:*') as media_archive:
-            media_archive.extractall(path=path)
+            media_archive.extractall(path=media_path)
     os.remove(backup_path)
     click.echo(' [{}s]'.format(int(time() - start_extract)))
     click.secho('Done', fg='green', nl=False)
@@ -439,6 +460,11 @@ def pull_media(client, stage, path=None):
 def dump_database(dump_filename, archive_filename=None):
     project_home = utils.get_project_home()
     docker_compose = utils.get_docker_compose_cmd(project_home)
+    docker_compose_config = utils.DockerComposeConfig(docker_compose)
+    if not docker_compose_config.has_service('db'):
+        click.secho('No service "db" found in local project', fg='red')
+        sys.exit(1)
+
     utils.start_database_server(docker_compose)
 
     click.secho(' ---> Dumping local database', nl=False)
@@ -624,7 +650,7 @@ def update_local_project(git_branch):
     check_call(docker_compose('pull'))
     click.secho('Building local docker images', fg='green')
     check_call(docker_compose('build'))
-    click.secho('sync and migrate database', fg='green')
+    click.secho('syncing and migrating database', fg='green')
     if is_windows():
         # interactive mode is not yet supported with docker-compose
         # on windows. that's why we have to call it as daemon
@@ -638,6 +664,7 @@ def update_local_project(git_branch):
 def develop_package(package, no_rebuild=False):
     """
     :param package: package name in addons-dev folder
+    :param no_rebuild: skip the rebuild of the container
     """
 
     project_home = utils.get_project_home()

divio_cli/localdev/utils.py

@@ -4,10 +4,12 @@
 from time import time
 
 import click
+import yaml
 
 from ..utils import check_output, is_windows, check_call
 from .. import exceptions
 from .. import settings
+from .. import config
 
 
 def get_aldryn_project_settings(path=None):
@@ -29,7 +31,7 @@ def get_project_home(path=None):
 
         # check if '.aldryn' file exists in current directory
         dotfile = os.path.join(current_path, settings.ALDRYN_DOT_FILE)
-        if os.path.exists(dotfile):
+        if os.path.exists(dotfile) and dotfile != config.CONFIG_FILE_PATH:
             return current_path
 
         # traversing up the tree
@@ -87,8 +89,6 @@ def ensure_windows_docker_compose_file_exists(path):
     Hope that's all. And of course, I'm sorry.
     """
 
-    import yaml
-
     windows_path = os.path.join(path, WINDOWS_DOCKER_COMPOSE_FILENAME)
     if os.path.isfile(windows_path):
         return
@@ -103,9 +103,9 @@ def ensure_windows_docker_compose_file_exists(path):
         sys.exit(1)
 
     with open(unix_path, 'r') as fh:
-        config = yaml.load(fh)
+        conf = yaml.load(fh)
 
-    for component, sections in config.items():
+    for component, sections in conf.items():
         if 'volumes' not in sections:
             continue
         volumes = []
@@ -128,10 +128,10 @@ def ensure_windows_docker_compose_file_exists(path):
                 new_volume.append(mode)
             volumes.append(':'.join(new_volume))
 
-        config[component]['volumes'] = volumes
+        conf[component]['volumes'] = volumes
 
     with open(windows_path, 'w+') as fh:
-        yaml.safe_dump(config, fh)
+        yaml.safe_dump(conf, fh)
 
 
 def get_db_container_id(path, raise_on_missing=True):
@@ -148,3 +148,26 @@ def start_database_server(docker_compose):
     click.secho('      ', nl=False)
     check_call(docker_compose('up', '-d', 'db'))
     click.secho('      [{}s]'.format(int(time() - start_db)))
+
+
+class DockerComposeConfig(object):
+    def __init__(self, docker_compose):
+        super(DockerComposeConfig, self).__init__()
+        self.config = yaml.load(check_output(docker_compose('config')))
+
+    def get_services(self):
+        return self.config.get('services', {})
+
+    def has_service(self, service):
+        return service in self.get_services().keys()
+
+    def has_volume_mount(self, service, remote_path):
+        try:
+            service_config = self.get_services()[service]
+        except KeyError:
+            return False
+
+        for mount in service_config.get('volumes', []):
+            data = mount.split(':')
+            if data[1] == remote_path:
+                return True

divio_cli/utils.py

@@ -5,7 +5,6 @@
 import io
 import os
 import sys
-from subprocess import CalledProcessError
 from contextlib import contextmanager
 from distutils.version import StrictVersion
 from math import log
@@ -19,7 +18,7 @@
 from . import __version__
 
 
-ALDRYN_DEFAULT_BRANCH_NAME = "develop"
+ALDRYN_DEFAULT_BRANCH_NAME = 'develop'
 
 
 def hr(char='-', width=None, **kwargs):
@@ -183,17 +182,17 @@ def open_project_cloud_site(client, stage):
         click.secho('No {} server deployed yet.'.format(stage), fg='yellow')
 
 
-def get_dashboard_url(client):
+def get_cp_url(client, section='dashboard'):
     from .localdev.utils import get_aldryn_project_settings
 
     project_settings = get_aldryn_project_settings()
     project_data = client.get_project(project_settings['id'])
-    return project_data['dashboard_url']
+    url = project_data['dashboard_url']
 
+    if section != 'dashboard':
+        url = urljoin(url, section)
 
-def get_project_cheatsheet_url(client):
-    dashboard = get_dashboard_url(client)
-    return urljoin(dashboard, 'local-development/')
+    return url
 
 
 def is_windows():
@@ -279,7 +278,7 @@ def get_git_checked_branch():
             'git',
             'rev-parse', '--abbrev-ref', 'HEAD'
         ], env=get_subprocess_env()).strip()
-    except CalledProcessError:
+    except subprocess.CalledProcessError:
         return ALDRYN_DEFAULT_BRANCH_NAME
 
 
@@ -290,12 +289,12 @@ def get_user_agent():
     else:
         client = 'divio-cli/{}'.format(__version__)
 
-    os = '{}/{}'.format(platform.system(), platform.release())
+    os_identifier = '{}/{}'.format(platform.system(), platform.release())
     python = '{}/{}'.format(
         platform.python_implementation(),
         platform.python_version(),
     )
-    return '{} ({}; {})'.format(client, os, python)
+    return '{} ({}; {})'.format(client, os_identifier, python)
 
 
 def download_file(url, directory=None, filename=None):

requirements-windows.txt

@@ -1,2 +1 @@
-pyyaml  # converting docker-compose configs
 colorama  # colored output

requirements.txt

@@ -4,3 +4,4 @@ tabulate
 six
 cryptography
 attrs
+yaml