From 4bab58cd62f3a71625d2128347b73e1775d73a11 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 21 Jun 2015 10:20:15 -0700 Subject: [PATCH 001/100] Added 'upload' command --- molns.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/molns.py b/molns.py index 6702a15..a694e94 100755 --- a/molns.py +++ b/molns.py @@ -266,6 +266,32 @@ def ssh_controller(cls, args, config): subprocess.call(cmd) print "SSH process completed" + @classmethod + def upload_controller(cls, args, config): + """ Copy a local file to the controller's home directory. """ + logging.debug("MOLNSController.upload_controller(args={0})".format(args)) + controller_obj = cls._get_controllerobj(args, config) + if controller_obj is None: return + # Check if any instances are assigned to this controller + instance_list = config.get_controller_instances(controller_id=controller_obj.id) + #logging.debug("instance_list={0}".format(instance_list)) + # Check if they are running + ip = None + if len(instance_list) > 0: + for i in instance_list: + status = controller_obj.get_instance_status(i) + logging.debug("instance={0} has status={1}".format(i, status)) + if status == controller_obj.STATUS_RUNNING: + ip = i.ip_address + if ip is None: + print "No active instance for this controller" + return + #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) + #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) + cmd = ['/usr/bin/scp','-r','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] + print " ".join(cmd) + subprocess.call(cmd) + print "SSH process completed" @classmethod def put_controller(cls, args, config): @@ -945,7 +971,8 @@ def clear_instances(cls, args, config): function=MOLNSController.terminate_controller), Command('put', {'name':None, 'file':None}, function=MOLNSController.put_controller), - + Command('upload', {'name':None, 'file':None}, + function=MOLNSController.upload_controller), #Command('local-connect', {'name':None}, # function=MOLNSController.connect_controller_to_local), # Commands to interact with controller From f071d2220b6f7f81e7a7cdf3a7bc1250f3e6e5b8 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 28 Jun 2015 11:56:41 -0700 Subject: [PATCH 002/100] Changing the command from 'instances' to 'instancedb' for clarity --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index a694e94..c9a6ae1 100755 --- a/molns.py +++ b/molns.py @@ -1021,7 +1021,7 @@ def clear_instances(cls, args, config): function=MOLNSProvider.delete_provider), ]), # Commands to interact with the instance DB - SubCommand('instances',[ + SubCommand('instancedb',[ Command('list', {}, function=MOLNSInstances.show_instances), Command('delete', {'ID':None}, From 160a8995df16514fcf7444182684b809912f890d Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 26 Jul 2015 07:51:21 -0700 Subject: [PATCH 003/100] First commit of the Euca provider --- MolnsLib/EucalyptusProvider.py | 682 +++++++++++++++++++++++++++++++++ MolnsLib/installSoftware.py | 2 +- MolnsLib/molns_datastore.py | 2 +- molns.py | 3 - 4 files changed, 684 insertions(+), 5 deletions(-) create mode 100644 MolnsLib/EucalyptusProvider.py diff --git a/MolnsLib/EucalyptusProvider.py b/MolnsLib/EucalyptusProvider.py new file mode 100644 index 0000000..6eb3b6a --- /dev/null +++ b/MolnsLib/EucalyptusProvider.py @@ -0,0 +1,682 @@ +import boto +import boto.ec2 +from boto.exception import EC2ResponseError +from boto.ec2.regioninfo import RegionInfo +import collections +import os +import time +import sys +import logging +from urlparse import urlparse +from collections import OrderedDict +import installSoftware +import ssh_deploy +from molns_provider import ProviderBase, ProviderException + +#logging.getLogger('boto').setLevel(logging.ERROR) +logging.getLogger('boto').setLevel(logging.CRITICAL) + + +########################################## +class EucalyptusBase(ProviderBase): + """ Abstract class for Eucalyptus. """ + + SSH_KEY_EXTENSION = ".pem" + PROVIDER_TYPE = 'Eucalyptus' + +#def EucalyptusProvider_config_get_region(): +# if os.environ.get('AWS_DEFAULT_REGION') is None: +# return 'us-east-1' +# return os.environ.get('AWS_DEFAULT_REGION') + +def EucalyptusProvider_config_get_ubuntu_images_by_region(conf=None): + if conf is not None: + access_key = conf['aws_access_key'] + secret_key = conf['aws_secret_key'] + ec2_url = conf['ec2_url'] + else: + access_key = os.environ['EC2_ACCESS_KEY'] + secret_key = os.environ['EC2_SECRET_KEY'] + ec2_url = os.environ['EC2_URL'] + + try: + o = urlparse(ec2_url) + ec2_host = o.hostname + ec2_port = o.port + ec2_path = o.path + # Setup connection to Eucalyptus + conn = boto.connect_ec2(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + is_secure=False, + region=RegionInfo(name="eucalyptus", endpoint=ec2_host), + port=ec2_port, + path=ec2_path) + + # Run commands + images = conn.get_all_images() + for i in images: + if 'trusty' in i.name.lower(): + return i.id + except Exception as e: + logging.debug('EucalyptusProvider_config_get_ubuntu_images_by_region() caught exception {0}'.format(e)) + return None + +def EucalyptusProvider_default_key_name(): + user = os.environ.get('USER') or 'USER' + return "{0}_molns_sshkey_{1}".format(user, hex(int(time.time())).replace('0x','')) +########################################## +class EucalyptusProvider(EucalyptusBase): + """ Provider handle for an Eucalyptus service. """ + + OBJ_NAME = 'EucalyptusProvider' + + CONFIG_VARS = OrderedDict( + [ + ('aws_access_key', + {'q':'Eucalyptus access Key', 'default':os.environ.get('EC2_ACCESS_KEY'), 'ask':True, 'obfuscate':True}), + ('aws_secret_key', + {'q':'Eucalyptus secret key', 'default':os.environ.get('EC2_SECRET_KEY'), 'ask':True, 'obfuscate':True}), + ('ec2_url', + {'q':'URL of Eucalyptus service (EC2_URL)', 'default':os.environ.get('EC2_URL'), 'ask':True, 'obfuscate':True}), +# ('aws_region', +# {'q':'Eucalyptus AWS region', 'default':EucalyptusProvider_config_get_region(), 'ask':True}), + ('key_name', + {'q':'Eucalyptus Key Pair name', 'default':EucalyptusProvider_default_key_name(), 'ask':True}), + ('group_name', + {'q':'Eucalyptus Security Group name', 'default':'molns', 'ask':True}), + ('ubuntu_image_name', + {'q':'ID of the base Ubuntu image to use', 'default':EucalyptusProvider_config_get_ubuntu_images_by_region, 'ask':True}), + ('molns_image_name', + {'q':'ID of the MOLNs image (leave empty for none)', 'default':None, 'ask':True}), + ('default_instance_type', + {'q':'Default Instance Type', 'default':'c3.large', 'ask':True}), + ('login_username', + {'default':'ubuntu', 'ask':False}) + ]) + + def get_config_credentials(self): + """ Return a dict with the credentials necessary for authentication. """ + return { + 'aws_access_key_id' : self.config['aws_access_key'], + 'aws_secret_access_key' : self.config['aws_secret_key'] + } + + + def check_ssh_key(self): + """ Check that the SSH key is found locally and remotely. + Returns: + True if the key is valid, otherwise False. + """ + ssh_key_dir = os.path.join(self.config_dir, self.name) + logging.debug('ssh_key_dir={0}'.format(ssh_key_dir)) + if not os.path.isdir(ssh_key_dir): + logging.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) + os.makedirs(ssh_key_dir) + ssh_key_file = os.path.join(ssh_key_dir,self.config['key_name']+self.SSH_KEY_EXTENSION) + if not os.path.isfile(ssh_key_file): + logging.debug("ssh_key_file '{0}' not found".format(ssh_key_file)) + return False + self._connect() + return self.eucalyptus.keypair_exists(self.config['key_name']) + + def create_ssh_key(self): + """ Create the ssh key and write the file locally. """ + self._connect() + ssh_key_dir = os.path.join(self.config_dir, self.name) + logging.debug('creating ssh key {0} in dir{1}'.format(self.config['key_name'], ssh_key_dir)) + self.eucalyptus.create_keypair(self.config['key_name'], ssh_key_dir) + + def check_security_group(self): + """ Check if the security group is created. """ + self._connect() + return self.eucalyptus.security_group_exists(self.config['group_name']) + + def create_seurity_group(self): + """ Create the security group. """ + self._connect() + return self.eucalyptus.create_security_group(self.config['group_name']) + + def check_molns_image(self): + """ Check if the molns image is created. """ + if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config['molns_image_name'] != '': + self._connect() + return self.eucalyptus.image_exists(self.config['molns_image_name']) + return False + + def create_molns_image(self): + """ Create the molns image is created. """ + self._connect() + # start vm + instances = self.eucalyptus.start_eucalyptus_instances(image_id=self.config["ubuntu_image_name"]) + instance = instances[0] + # get login ip + ip = instance.public_dns_name + # install software + try: + logging.debug("installing software on server (ip={0})".format(ip)) + install_vm_instance = installSoftware.InstallSW(ip, config=self) + #install_vm_instance.run_with_logging() + # create image + logging.debug("Shutting down instance") + self.eucalyptus.stop_eucalyptus_instances([instance]) + #logging.debug("Creating image") + #image_id = instance.create_image(name=self._get_image_name()) + logging.debug("Finding volume of instance") + vol = None + for v in self.eucalyptus.conn.get_all_volumes(): + if v.attach_data is not None and v.attach_data.instance_id == instance.id: + vol = v + break + if vol is None: + raise Exception("Can not find volume associated with instance. Base image must be an EBS backed image.") + snap = vol.create_snapshot() + logging.debug('Snapshot {0} of volume {1}'.format(snap.id, vol.id)) + #image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id, delete_root_volume_on_termination=True) + #deleteOnTermination + image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id) + logging.debug("Image created: {0}".format(image_id)) + except Exception as e: + logging.exception(e) + raise ProviderException("Failed to create molns image: {0}".format(e)) + finally: + logging.debug("terminating {0}".format(instance)) + self.eucalyptus.terminate_eucalyptus_instances([instance]) + return image_id + + def _connect(self): + if self.connected: return + self.eucalyptus = CreateVM(config=self) + self.connected = True + + def _get_image_name(self): + return "MOLNS_{0}_{1}_{2}".format(self.PROVIDER_TYPE, self.name, int(time.time())) + +########################################## +class EucalyptusController(EucalyptusBase): + """ Provider handle for an open stack controller. """ + + OBJ_NAME = 'EucalyptusController' + + CONFIG_VARS = OrderedDict( + [ + ('instance_type', + {'q':'Default Instance Type', 'default':'c3.large', 'ask':True}), + ]) + + def _connect(self): + if self.connected: return + self.eucalyptus = CreateVM(config=self.provider) + self.connected = True + + def start_instance(self, num=1): + """ Start or resume the controller. """ + try: + self._connect() + instances = self.eucalyptus.start_eucalyptus_instances(image_id=self.provider.config["molns_image_name"], num=int(num), instance_type=self.config["instance_type"]) + ret = [] + for instance in instances: + ip = instance.public_dns_name + i = self.datastore.get_instance(provider_instance_identifier=instance.id, ip_address=ip, provider_id=self.provider.id, controller_id=self.id) + ret.append(i) + if num == 1: + return ret[0] + else: + return ret + except Exception as e: + logging.exception(e) + raise ProviderException("Failed to start molns instance: {0}".format(e)) + + def resume_instance(self, instances): + self._connect() + if isinstance(instances, list): + eucalyptus_instances = [] + for instance in instances: + eucalyptus_instance = self.eucalyptus.get_instance(instance.provider_instance_identifier) + eucalyptus_instances.append(eucalyptus_instance) + new_eucalyptus_instances = self.eucalyptus.resume_eucalyptus_instances(eucalyptus_instances) + instances_to_update = list(instances) + while len(instances_to_update) > 0: + instance = instances_to_update.pop() + success=False + for eucalyptus_inst in new_eucalyptus_instances: + if eucalyptus_inst.id == instance.provider_instance_identifier: + instance.ip_address = eucalyptus_inst.public_dns_name + logging.debug("instance.id={0} updated with ip={1}".format(instance.provider_instance_identifier, instance.ip_address)) + success=True + break + if not success: + raise ProviderException("Could not update the IP of id={0} after resume".format(instance.provider_instance_identifier)) + else: + eucalyptus_instance = self.eucalyptus.get_instance(instances.provider_instance_identifier) + new_instance = self.eucalyptus.resume_eucalyptus_instances([eucalyptus_instance]) + instances.ip_address = new_instance[0].public_dns_name + logging.debug("instance.id={0} updated with ip={1}".format(instances.provider_instance_identifier, instances.ip_address)) + + def stop_instance(self, instances): + self._connect() + if isinstance(instances, list): + eucalyptus_instances = [] + for instance in instances: + eucalyptus_instance = self.eucalyptus.get_instance(instance.provider_instance_identifier) + eucalyptus_instances.append(eucalyptus_instance) + self.eucalyptus.stop_eucalyptus_instances(eucalyptus_instances) + else: + eucalyptus_instance = self.eucalyptus.get_instance(instances.provider_instance_identifier) + self.eucalyptus.stop_eucalyptus_instances([eucalyptus_instance]) + + def terminate_instance(self, instances): + self._connect() + if isinstance(instances, list): + eucalyptus_instances = [] + for instance in instances: + eucalyptus_instance = self.eucalyptus.get_instance(instances.provider_instance_identifier) + eucalyptus_instances.append(eucalyptus_instance) + self.datastore.delete_instance(instance) + self.eucalyptus.terminate_eucalyptus_instances(eucalyptus_instances) + else: + eucalyptus_instance = self.eucalyptus.get_instance(instances.provider_instance_identifier) + self.eucalyptus.terminate_eucalyptus_instances([eucalyptus_instance]) + self.datastore.delete_instance(instances) + + def get_instance_status(self, instance): + self._connect() + try: + status = self.eucalyptus.get_instance_status(instance.provider_instance_identifier) + except Exception as e: + #logging.exception(e) + return self.STATUS_TERMINATED + if status == 'running' or status == 'pending': + return self.STATUS_RUNNING + if status == 'stopped' or status == 'stopping': + return self.STATUS_STOPPED + if status == 'terminated' or status == 'shutting-down': + return self.STATUS_TERMINATED + raise ProviderException("EucalyptusController.get_instance_status() got unknown status '{0}'".format(status)) + + +########################################## +class EucalyptusWorkerGroup(EucalyptusController): + """ Provider handle for an open stack controller. """ + + OBJ_NAME = 'EucalyptusWorkerGroup' + + CONFIG_VARS = OrderedDict( + [ + ('instance_type', + {'q':'Default Instance Type', 'default':'c3.large', 'ask':True}), + ('num_vms', + {'q':'Number of virtual machines in group', 'default':'1', 'ask':True}), + ]) + + def start_instance(self, num=1): + """ Start worker group vms. """ + try: + self._connect() + instances = self.eucalyptus.start_eucalyptus_instances(image_id=self.provider.config["molns_image_name"], num=int(num), instance_type=self.config["instance_type"]) + ret = [] + for instance in instances: + ip = instance.public_dns_name + i = self.datastore.get_instance(provider_instance_identifier=instance.id, ip_address=ip, provider_id=self.provider.id, controller_id=self.controller.id, worker_group_id=self.id) + ret.append(i) + if num == 1: + return ret[0] + else: + return ret + except Exception as e: + logging.exception(e) + raise ProviderException("Failed to start molns instance: {0}".format(e)) + + def terminate_instance(self, instances): + self._connect() + if isinstance(instances, list): + eucalyptus_instances = [] + for instance in instances: + eucalyptus_instance = self.eucalyptus.get_instance(instance.provider_instance_identifier) + eucalyptus_instances.append(eucalyptus_instance) + self.datastore.delete_instance(instance) + self.eucalyptus.terminate_eucalyptus_instances(eucalyptus_instances) + else: + eucalyptus_instance = self.eucalyptus.get_instance(instances.provider_instance_identifier) + self.eucalyptus.terminate_eucalyptus_instances([eucalyptus_instance]) + self.datastore.delete_instance(instances) + + +########################################## +class CreateVM: + ''' + This class is used to create VMs for Eucalyptus + ''' + PENDING_IMAGE_WAITTIME = 60 + + def __init__(self, config=None, connect=True): + if config is not None: + self.config = config + if self.config['aws_access_key'] is None or self.config['aws_secret_key'] is None: + raise ProviderException("AWS_SECRET_KEY or AWS_ACCESS_KEY not set") + if connect: + self.connect() + + def connect(self): + #self.conn = boto.ec2.connect_to_region( + # self.config['aws_region'], + # aws_access_key_id=self.config['aws_access_key'], + # aws_secret_access_key=self.config['aws_secret_key'] + #) + access_key = self.config['aws_access_key'] + secret_key = self.config['aws_secret_key'] + ec2_url = self.config['ec2_url'] + o = urlparse(ec2_url) + ec2_host = o.hostname + ec2_port = o.port + ec2_path = o.path + # Setup connection to Eucalyptus + self.conn = boto.connect_ec2(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + is_secure=False, + region=RegionInfo(name="eucalyptus", endpoint=ec2_host), + port=ec2_port, + path=ec2_path) + + + def get_instance(self, instance_id): + #logging.debug("get_instance(instance_id={0})".format(instance_id)) + try: + reservations = self.conn.get_all_reservations(instance_ids=[instance_id]) + except EC2ResponseError: + raise ProviderException("instance not found {0}".format(instance_id)) + #logging.debug("get_instance() reservations:{0}".format(reservations)) + for reservation in reservations: + #logging.debug("get_instance() reservation.instances:{0}".format(reservation.instances)) + for instance in reservation.instances: + if instance.id == instance_id: + return instance + raise ProviderException("instance not found {0}".format(instance_id)) + + def get_instance_status(self, instance_id): + return self.get_instance(instance_id).state + + + def get_vm_status(self, key_name=None, verbose=False, show_all=False): + if key_name is None: + key_name = self.config['key_name'] + reservations = self.conn.get_all_reservations() + stopped_vms = [] + running_vms = [] + for reservation in reservations: + for instance in reservation.instances: + if verbose and show_all: + print "{0}\t{1}\t{2}\t{3}".format(instance.id,instance.key_name,instance.state,instance.public_dns_name) + if instance.key_name == key_name: + if verbose and not show_all: + print "{0}\t{1}\t{2}\t{3}".format(instance.id,instance.key_name,instance.state,instance.public_dns_name) + if instance.state == 'running': + running_vms.append(instance) + elif instance.state == 'stopped': + stopped_vms.append(instance) + #return (stopped_vms, running_vms) + return (stopped_vms, sorted(running_vms, key=lambda vm: vm.id)) + + def image_exists(self, image_id): + try: + img = self.conn.get_all_images(image_ids=[image_id])[0] + return True + except IndexError: + return False + + def start_vms(self, image_id=None, key_name=None, group_name=None, num=None, instance_type=None): + if key_name is None: + key_name = self.config['key_name'] + if group_name is None: + group_name = self.config['group_name'] + if num is None: + num = 1 + if instance_type is None: + instance_type = self.config['default_instance_type'] + # Check the group + self.create_security_group(group_name) + + #(stopped_vms, running_vms) = self.get_vm_status(key_name) + #if len(running_vms) > 0: + # msg = "Error: {0} VMs are already running with key_name={1}".format(len(running_vms), + # key_name) + # print msg + # raise ProviderException(msg) + + if len(stopped_vms) > 0: + return self.resume_eucalyptus_instances(stopped_vms) + + if image_id is None: + raise ProviderException("Base Ubuntu image not specified.") + else: + self.image_id = image_id + + # Check image + try: + img = self.conn.get_all_images(image_ids=[self.image_id])[0] + except IndexError: + raise ProviderException("Could not find image_id={0}".format(self.image_id)) + + if img.state != "available": + if img.state != "pending": + raise ProviderException("Image {0} is not available, it has state is {1}.".format(self.image_id, img.state)) + while img.state == "pending": + print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(self.image_id, img.state, self.PENDING_IMAGE_WAITTIME) + time.sleep(self.PENDING_IMAGE_WAITTIME) + img.update() + + self.key_name = key_name + self.group_name = group_name + group_list = [] + for _ in range(num): + group_list.append(group_name) + + print "Starting {0} Eucalyptus instance(s). This will take a minute...".format(num) + reservation = self.conn.run_instances(self.image_id, min_count=num, max_count=num, key_name=key_name, security_groups=group_list, instance_type=instance_type) + + instances = reservation.instances + num_instance = len(instances) + num_running = 0 + while num_running < num_instance: + num_running = 0 + for instance in instances: + instance.update() + if instance.state == 'running': + num_running += 1 + if num_running < num_instance: + time.sleep(5) + print "Eucalyptus instances started." + return sorted(instances, key=lambda vm: vm.id) + + def start_eucalyptus_instances(self, image_id=None, key_name=None, group_name=None, num=1, instance_type=None): + if key_name is None: + key_name = self.config['key_name'] + if group_name is None: + group_name = self.config['group_name'] + if num is None: + num = 1 + if instance_type is None: + instance_type = self.config['default_instance_type'] + try: + img = self.conn.get_all_images(image_ids=[image_id])[0] + except IndexError: + raise ProviderException("Could not find image_id={0}".format(image_id)) + if img.state != "available": + if img.state != "pending": + raise ProviderException("Image {0} is not available, it has state is {1}.".format(image_id, img.state)) + while img.state == "pending": + print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(image_id, img.state, self.PENDING_IMAGE_WAITTIME) + time.sleep(self.PENDING_IMAGE_WAITTIME) + img.update() + print "Starting {0} Eucalyptus instance(s). This will take a minute...".format(num) + reservation = self.conn.run_instances(image_id, min_count=num, max_count=num, key_name=key_name, security_groups=[group_name], instance_type=instance_type) + instances = reservation.instances + num_instance = len(instances) + num_running = 0 + while num_running < num_instance: + num_running = 0 + for instance in instances: + instance.update() + if instance.state == 'running': + num_running += 1 + if num_running < num_instance: + time.sleep(5) + print "Eucalyptus instances started." + return sorted(instances, key=lambda vm: vm.id) + + def stop_vms(self, key_name=None): + if key_name is None: + key_name = self.config['key_name'] + (stopped_vms, running_vms) = self.get_vm_status(key_name) + self.stop_eucalyptus_instances(running_vms) + + def terminate_vms(self, key_name=None): + if key_name is None: + key_name = self.config['key_name'] + (stopped_vms, running_vms) = self.get_vm_status(key_name) + self.terminate_eucalyptus_instances(running_vms+stopped_vms) + + def resume_eucalyptus_instances(self, instances): + num_instance = len(instances) + print "Resuming Eucalyptus instance(s). This will take a minute..." + for instance in instances: + print "\t{0}.".format(instance.id) + instance.start() + num_running = 0 + while num_running < num_instance: + num_running = 0 + for instance in instances: + instance.update() + if instance.state == 'running': + num_running += 1 + if num_running < num_instance: + time.sleep(5) + print "Eucalyptus instances resumed." + return instances + + def stop_eucalyptus_instances(self, instances): + num_instance = len(instances) + print "Stopping Eucalyptus instance(s). This will take a minute..." + for instance in instances: + print "\t{0}.".format(instance.id) + instance.stop() + num_stopped = 0 + while num_stopped < num_instance: + num_stopped = 0 + for instance in instances: + instance.update() + if instance.state == 'stopped': + num_stopped += 1 + if num_stopped < num_instance: + time.sleep(5) + print "Eucalyptus instances stopped." + + def terminate_eucalyptus_instances(self, instances): + num_instance = len(instances) + print "Terminating Eucalyptus instance(s). This will take a minute..." + for instance in instances: + print "\t{0}.".format(instance.id) + instance.terminate() + num_terminated = 0 + while num_terminated < num_instance: + num_terminated = 0 + for instance in instances: + instance.update() + if instance.state == 'terminated': + num_terminated += 1 + if num_terminated < num_instance: + time.sleep(5) + print "Eucalyptus instance terminated." + + def create_vm_image(self, image_name=None, key_name=None): + if key_name is None: + key_name = self.config['key_name'] + if image_name is None: + image_name = "MOLNS_{0}_{1}".format(key_name,int(time.time())) + (stopped_vms, running_vms) = self.get_vm_status(key_name) + if len(running_vms) != 1: + raise ProviderException("Expected only one running vm, {0} are running".format(len(running_vms))) + self.stop_eucalyptus_instances(running_vms) + instance = running_vms[0] + image_ami = instance.create_image(image_name) + print "Image created id={0} name={0}".format(image_ami, image_name) + self.terminate_eucalyptus_instances(running_vms) + return image_ami + + + + def keypair_exists(self, key_name): + for sg in self.conn.get_all_key_pairs(): + if sg.name == key_name: + return True + return False + + def keypair_file_exists(cls, key_name, conf_dir): + return os.path.exists(conf_dir + os.sep + key_name + ".pem") + + def create_keypair(self, key_name, conf_dir): + key_pair = self.conn.create_key_pair(key_name) + key_pair.save(conf_dir) + + def security_group_exists(self, group_name): + for sg in self.conn.get_all_security_groups(): + if sg.name == group_name: + return True + return False + + def create_security_group(self, group_name): + security_group = None + for sg in self.conn.get_all_security_groups(): + if sg.name == group_name: + security_group = sg + break + if security_group is None: + print "Security group not found, creating one." + security_group = self.conn.create_security_group(group_name, 'MOLNs Security Group') + self.set_security_group_rules(security_group) + elif not self.check_security_group_rules(security_group): + raise ProviderException("Security group {0} exists, but has the wrong firewall rules. Please delete the group, or choose a different one.") + return security_group + + + def set_security_group_rules(self, group, expected_rules=ProviderBase.FIREWALL_RULES): + for rule in expected_rules: + if not group.authorize(ip_protocol=rule.ip_protocol, + from_port=rule.from_port, + to_port=rule.to_port, + cidr_ip=rule.cidr_ip): + return False + return True + + def check_security_group_rules(self, group, expected_rules=ProviderBase.FIREWALL_RULES): + """ Check to be sure the expected_rules are set for this group. """ + ret = True + + current_rules = [] + for rule in group.rules: + if not rule.grants[0].cidr_ip: + current_rule = self.SecurityGroupRule(rule.ip_protocol, + rule.from_port, + rule.to_port, + "0.0.0.0/0", + rule.grants[0].name) + else: + current_rule = self.SecurityGroupRule(rule.ip_protocol, + rule.from_port, + rule.to_port, + rule.grants[0].cidr_ip, + None) + + if current_rule not in expected_rules: + print "Unexpected Rule: {0}".format(current_rule) + ret = False + else: + #print "Current Rule: {0}".format(current_rule) + current_rules.append(current_rule) + + for rule in expected_rules: + if rule not in current_rules: + print "Rule not found: {0}".format(rule) + ret = False + + return ret + diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 2094ccf..29c53c7 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -32,7 +32,7 @@ class InstallSW: "sudo apt-get -y install python-matplotlib python-numpy python-scipy", "sudo apt-get -y install make", "sudo apt-get -y install python-software-properties", - "sudo add-apt-repository ppa:fenics-packages/fenics", + "sudo add-apt-repository -y ppa:fenics-packages/fenics", "sudo apt-get update", "sudo apt-get -y install fenics", "sudo apt-get -y install cython python-h5py", diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index b54bf43..184e51e 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -9,7 +9,7 @@ import sys ############################################################# #VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] -VALID_PROVIDER_TYPES = ['OpenStack', 'EC2'] +VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] ############################################################# #### SCHEMA ################################################# ############################################################# diff --git a/molns.py b/molns.py index c9a6ae1..edff96b 100755 --- a/molns.py +++ b/molns.py @@ -1056,14 +1056,11 @@ def parseArgs(): #logger.setLevel(logging.INFO) #for Debugging arg_list = arg_list[1:] - #print "config_dir", config_dir - #print "arg_list ", arg_list if len(arg_list) == 0 or arg_list[0] =='help' or arg_list[0] == '-h': printHelp() return if arg_list[0] in COMMAND_LIST: - #print arg_list[0] + " in COMMAND_LIST" for cmd in COMMAND_LIST: if cmd == arg_list[0]: try: From a67db002eab9c2a5a51b4f209bfc5f722b1d6abb Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 26 Jul 2015 23:49:57 -0700 Subject: [PATCH 004/100] Provider import/export --- MolnsLib/molns_datastore.py | 2 + molns.py | 290 ++++++++++++++++++++++-------------- 2 files changed, 179 insertions(+), 113 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 184e51e..bdc8ead 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -148,6 +148,8 @@ def __init__(self, db_file=None, config_dir=None): """ Constructor. """ if db_file is not None: self.engine = create_engine('sqlite:///{0}'.format(db_file)) + if config_dir is None: + self.config_dir = os.path.basename(db_file) elif config_dir is not None: if not os.path.exists(config_dir): os.makedirs(config_dir) diff --git a/molns.py b/molns.py index edff96b..3e3c7a8 100755 --- a/molns.py +++ b/molns.py @@ -8,6 +8,7 @@ import subprocess from MolnsLib.ssh_deploy import SSHDeploy import multiprocessing +import json import logging logger = logging.getLogger() @@ -15,122 +16,11 @@ logger.setLevel(logging.CRITICAL) ############################################### -class CommandException(Exception): - pass -############################################### -def table_print(column_names, data): - column_width = [0]*len(column_names) - for i,n in enumerate(column_names): - column_width[i] = len(str(n)) - for row in data: - if len(row) != len(column_names): - print "len(row) != len(column_names): {0} vs {1}".format(len(row), len(column_names)) - for i,n in enumerate(row): - if len(str(n)) > column_width[i]: - column_width[i] = len(str(n)) - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' - out = " | ".join([ column_names[i].ljust(column_width[i]) for i in range(len(column_names))]) - print '| '+out+' |' - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' - for row in data: - out = " | ".join([ str(n).ljust(column_width[i]) for i,n in enumerate(row)]) - print '| '+out+' |' - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' - -def raw_input_default(q, default=None, obfuscate=False): - if default is None or default == '': - return raw_input("{0}:".format(q)) - else: - if obfuscate: - ret = raw_input("{0} [******]: ".format(q)) - else: - ret = raw_input("{0} [{1}]: ".format(q, default)) - if ret == '': - return default - else: - return ret.strip() - -def raw_input_default_config(q, default=None, obj=None): - """ Ask the user and process the response with a default value. """ - if default is None: - if callable(q['default']): - f1 = q['default'] - try: - default = f1(obj) - except TypeError: - pass - else: - default = q['default'] - if 'ask' in q and not q['ask']: - return default - if 'obfuscate' in q and q['obfuscate']: - return raw_input_default(q['q'], default=default, obfuscate=True) - else: - return raw_input_default(q['q'], default=default, obfuscate=False) - -def setup_object(obj): - """ Setup a molns_datastore object using raw_input_default function. """ - for key, conf, value in obj.get_config_vars(): - obj[key] = raw_input_default_config(conf, default=value, obj=obj) - -############################################### -class SubCommand(): - def __init__(self, command, subcommands): - self.command = command - self.subcommands = subcommands - def __str__(self): - r = '' - for c in self.subcommands: - r += self.command + " " + c.__str__() + "\n" - return r[:-1] - def __eq__(self, other): - return self.command == other - - def run(self, args, config_dir=None): - #print "SubCommand().run({0}, {1})".format(self.command, args) - if len(args) > 0: - cmd = args[0] - for c in self.subcommands: - if c == cmd: - return c.run(args[1:], config_dir=config_dir) - raise CommandException("command not found") - -############################################### -class Command(): - def __init__(self, command, args_defs={}, description=None, function=None): - self.command = command - self.args_defs = args_defs - if function is None: - raise Exception("Command must have a function") - self.function = function - if description is None: - self.description = function.__doc__.strip() - else: - self.description = description - def __str__(self): - ret = self.command+" " - for k,v in self.args_defs.iteritems(): - if v is None: - ret += "[{0}] ".format(k) - else: - ret += "[{0}={1}] ".format(k,v) - ret += "\n\t"+self.description - return ret - - def __eq__(self, other): - return self.command == other - - def run(self, args, config_dir=None): - config = MOLNSConfig(config_dir=config_dir) - self.function(args, config=config) ############################################### class MOLNSConfig(Datastore): - def __init__(self, config_dir): - Datastore.__init__(self,config_dir=config_dir) + def __init__(self, config_dir=None, db_file=None): + Datastore.__init__(self,config_dir=config_dir, db_file=db_file) def __str__(self): return "MOLNSConfig(config_dir={0})".format(self.config_dir) @@ -775,6 +665,58 @@ def terminate_worker_groups(cls, args, config): ############################################### class MOLNSProvider(): + @classmethod + def provider_export(cls, args, config): + """ Export the configuration of a provider. """ + if len(args) < 1: + print "USAGE: molns provider export name [Filename]" + print "\Export the data from the provider with the given name." + return + provider_name = args[0] + # check if provider exists + try: + provider_obj = config.get_object(args[0], kind='Provider') + except DatastoreException as e: + print "provider not found" + return + data = {'name': provider_obj.name, + 'type': provider_obj.type, + 'config': provider_obj.config} + if len(args) > 1 and args[1] is not None: + with open(args[1],'w+') as fd: + json.dump(data, fd) + else: + print json.dumps(data) + + @classmethod + def provider_import(cls, args, config): + """ Import the configuration of a provider. """ + if len(args) < 1: + print "USAGE: molns provider import [Filename.json]" + print "\Import the data from the provider with the given name." + return + filename = args[0] + with open(filename) as fd: + data = json.load(fd) + provider_name = data['name'] + if data['type'] not in VALID_PROVIDER_TYPES: + print "Error: unknown provider type '{0}'".format(data['type']) + return + try: + provider_obj = config.get_object(provider_name, kind='Provider') + print "Found existing provider" + if provider_obj.type != data['type']: + print "Import data has provider type '{0}'. Provier {1} exists with type {2}. Type conversion is not possible.".format(data['type'], provider_obj.name, provider_obj.type) + return + except DatastoreException as e: + provider_obj = config.create_object(name=provider_name, ptype=data['type'], kind='Provider') + print "Creating new provider" + provider_obj.config = data['config'] + config.save_object(provider_obj, kind='Provider') + print "Provider data imported" + + + @classmethod def provider_setup(cls, args, config): """ Setup a new provider. Create the MOLNS image and SSH key if necessary.""" @@ -954,7 +896,125 @@ def clear_instances(cls, args, config): print "No instance found" +############################################################################################## +############################################################################################## +############################################################################################## +############################################################################################## +############################################################################################## +############################################################################################## +# Below is the API for the commmand line execution + +class CommandException(Exception): + pass +############################################### +def table_print(column_names, data): + column_width = [0]*len(column_names) + for i,n in enumerate(column_names): + column_width[i] = len(str(n)) + for row in data: + if len(row) != len(column_names): + print "len(row) != len(column_names): {0} vs {1}".format(len(row), len(column_names)) + for i,n in enumerate(row): + if len(str(n)) > column_width[i]: + column_width[i] = len(str(n)) + out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) + print '|'+out+'|' + out = " | ".join([ column_names[i].ljust(column_width[i]) for i in range(len(column_names))]) + print '| '+out+' |' + out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) + print '|'+out+'|' + for row in data: + out = " | ".join([ str(n).ljust(column_width[i]) for i,n in enumerate(row)]) + print '| '+out+' |' + out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) + print '|'+out+'|' + +def raw_input_default(q, default=None, obfuscate=False): + if default is None or default == '': + return raw_input("{0}:".format(q)) + else: + if obfuscate: + ret = raw_input("{0} [******]: ".format(q)) + else: + ret = raw_input("{0} [{1}]: ".format(q, default)) + if ret == '': + return default + else: + return ret.strip() + +def raw_input_default_config(q, default=None, obj=None): + """ Ask the user and process the response with a default value. """ + if default is None: + if callable(q['default']): + f1 = q['default'] + try: + default = f1(obj) + except TypeError: + pass + else: + default = q['default'] + if 'ask' in q and not q['ask']: + return default + if 'obfuscate' in q and q['obfuscate']: + return raw_input_default(q['q'], default=default, obfuscate=True) + else: + return raw_input_default(q['q'], default=default, obfuscate=False) + +def setup_object(obj): + """ Setup a molns_datastore object using raw_input_default function. """ + for key, conf, value in obj.get_config_vars(): + obj[key] = raw_input_default_config(conf, default=value, obj=obj) + +############################################### +class SubCommand(): + def __init__(self, command, subcommands): + self.command = command + self.subcommands = subcommands + def __str__(self): + r = '' + for c in self.subcommands: + r += self.command + " " + c.__str__() + "\n" + return r[:-1] + def __eq__(self, other): + return self.command == other + def run(self, args, config_dir=None): + #print "SubCommand().run({0}, {1})".format(self.command, args) + if len(args) > 0: + cmd = args[0] + for c in self.subcommands: + if c == cmd: + return c.run(args[1:], config_dir=config_dir) + raise CommandException("command not found") + +############################################### +class Command(): + def __init__(self, command, args_defs={}, description=None, function=None): + self.command = command + self.args_defs = args_defs + if function is None: + raise Exception("Command must have a function") + self.function = function + if description is None: + self.description = function.__doc__.strip() + else: + self.description = description + def __str__(self): + ret = self.command+" " + for k,v in self.args_defs.iteritems(): + if v is None: + ret += "[{0}] ".format(k) + else: + ret += "[{0}={1}] ".format(k,v) + ret += "\n\t"+self.description + return ret + + def __eq__(self, other): + return self.command == other + + def run(self, args, config_dir=None): + config = MOLNSConfig(config_dir=config_dir) + self.function(args, config=config) ############################################### COMMAND_LIST = [ @@ -1019,6 +1079,10 @@ def clear_instances(cls, args, config): function=MOLNSProvider.show_provider), Command('delete',{'name':None}, function=MOLNSProvider.delete_provider), + Command('export',{'name':None}, + function=MOLNSProvider.provider_export), + Command('import',{'filename.json':None}, + function=MOLNSProvider.provider_import), ]), # Commands to interact with the instance DB SubCommand('instancedb',[ From a99e51c46d736b9dc15156e0dddaaac7abe3a79b Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 27 Jul 2015 23:00:37 -0700 Subject: [PATCH 005/100] Programatic API development --- MolnsLib/EucalyptusProvider.py | 2 +- MolnsLib/molns_datastore.py | 4 +- molns.py | 262 +++++++++++++++++++++++++-------- 3 files changed, 207 insertions(+), 61 deletions(-) diff --git a/MolnsLib/EucalyptusProvider.py b/MolnsLib/EucalyptusProvider.py index 6eb3b6a..0f96b23 100644 --- a/MolnsLib/EucalyptusProvider.py +++ b/MolnsLib/EucalyptusProvider.py @@ -77,7 +77,7 @@ class EucalyptusProvider(EucalyptusBase): ('aws_secret_key', {'q':'Eucalyptus secret key', 'default':os.environ.get('EC2_SECRET_KEY'), 'ask':True, 'obfuscate':True}), ('ec2_url', - {'q':'URL of Eucalyptus service (EC2_URL)', 'default':os.environ.get('EC2_URL'), 'ask':True, 'obfuscate':True}), + {'q':'URL of Eucalyptus service (EC2_URL)', 'default':os.environ.get('EC2_URL'), 'ask':True, 'obfuscate':False}), # ('aws_region', # {'q':'Eucalyptus AWS region', 'default':EucalyptusProvider_config_get_region(), 'ask':True}), ('key_name', diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index bdc8ead..f86451d 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -154,6 +154,7 @@ def __init__(self, db_file=None, config_dir=None): if not os.path.exists(config_dir): os.makedirs(config_dir) self.engine = create_engine('sqlite:///{0}/{1}'.format(config_dir, self.MOLNS_DATASTORE)) + self.config_dir = config_dir else: if not os.path.exists(self.MOLNS_CONFIG_DIR): os.makedirs(self.MOLNS_CONFIG_DIR) @@ -162,7 +163,6 @@ def __init__(self, db_file=None, config_dir=None): Base.metadata.create_all(self.engine) # Create all the tables Session = sessionmaker(bind=self.engine) self.session = Session() - self.config_dir = config_dir def __del__(self): """ Destructor. """ @@ -261,7 +261,7 @@ def get_object_by_id(self, id, kind): (handle, d_handle) = HANDLE_MAPPING[kind] p = self.session.query(handle).filter_by(id=id).first() if p is None: - raise DatastoreException("{0} {1} not found".format(kind, name)) + raise DatastoreException("{0} {1} not found".format(kind, id)) return self._get_object_data(d_handle, kind, p.type, p) def _get_object_data(self, d_handle, kind, ptype, p): diff --git a/molns.py b/molns.py index 3e3c7a8..29663d6 100755 --- a/molns.py +++ b/molns.py @@ -15,7 +15,8 @@ #logger.setLevel(logging.INFO) #for Debugging logger.setLevel(logging.CRITICAL) ############################################### - +class MOLNSException(Exception): + pass ############################################### class MOLNSConfig(Datastore): @@ -51,8 +52,7 @@ def _get_controllerobj(cls, args, config): if len(args) > 0: controller_name = args[0] else: - print "No controller name given" - return None + raise MOLNSException("No controller name given") # Get controller db object try: controller_obj = config.get_object(name=controller_name, kind='Controller') @@ -60,10 +60,64 @@ def _get_controllerobj(cls, args, config): controller_obj = None #logging.debug("controller_obj {0}".format(controller_obj)) if controller_obj is None: - print "controller '{0}' is not initialized, use 'molns controller setup {0}' to initialize the controller.".format(controller_name) + raise MOLNSException("controller '{0}' is not initialized, use 'molns controller setup {0}' to initialize the controller.".format(controller_name)) return controller_obj class MOLNSController(MOLNSbase): + @classmethod + def controller_export(cls, args, config): + """ Export the configuration of a controller. """ + if len(args) < 1: + raise MOLNSException("USAGE: molns controller export name [Filename]\n"\ + "\tExport the data from the controller with the given name.") + controller_name = args[0] + if len(args) > 1: + filename = args[1] + else: + filename = 'Molns-Export-Controller-' + controller_name + '.json' + # check if provider exists + try: + controller_obj = config.get_object(controller_name, kind='Controller') + except DatastoreException as e: + raise MOLNSException("provider not found") + data = {'name': controller_obj.name, + 'provider_name': controller_obj.provider.name, + 'config': controller_obj.config} + return {'data': json.dumps(data), + 'type': 'file', + 'filename': filename} + + @classmethod + def controller_import(cls, args, config, json_data=None): + """ Import the configuration of a controller. """ + if json_data is None: + if len(args) < 1: + raise MOLNSException("USAGE: molns controller import [Filename.json]\n"\ + "\Import the data from the controller with the given name.") + filename = args[0] + with open(filename) as fd: + data = json.load(fd) + else: + data = json_data + controller_name = data['name'] + msg = '' + try: + provider_obj = config.get_object(data['provider_name'], kind='Provider') + except DatastoreException as e: + raise MOLNSException("unknown provider '{0}'".format(data['provider_name'])) + try: + controller_obj = config.get_object(controller_name, kind='Controller') + msg += "Found existing controller\n" + if controller_obj.provider.name != provider_obj.name: + raise MOLNSException("Import data has provider '{0}'. Controller {1} exists with provider {2}. provider conversion is not possible.".format(data['provider_name'], controller_obj.name, controller_obj.provider.name)) + except DatastoreException as e: + controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', provider_id=provider_obj.id) + msg += "Creating new controller\n" + controller_obj.config = data['config'] + config.save_object(controller_obj, kind='Controller') + msg += "Controller data imported\n" + return {'msg':msg} + @classmethod def setup_controller(cls, args, config): """Setup a controller. Set the provider configuration for the head node. Use 'worker setup' to set the configuration for worker nodes @@ -104,29 +158,27 @@ def list_controller(cls, args, config): """ List all the currently configured controllers.""" controllers = config.list_objects(kind='Controller') if len(controllers) == 0: - print "No controllers configured" + return {'msg':"No controllers configured"} else: table_data = [] for c in controllers: provider_name = config.get_object_by_id(c.provider_id, 'Provider').name table_data.append([c.name, provider_name]) - table_print(['name', 'provider'], table_data) + return {'type':'table','column_names':['name', 'provider'], 'data':table_data} @classmethod def show_controller(cls, args, config): """ Show all the details of a controller config. """ if len(args) == 0: - print "USAGE: molns controller show name" - return - print config.get_object(name=args[0], kind='Controller') + raise MOLNSException("USAGE: molns controller show name") + return {'msg':str(config.get_object(name=args[0], kind='Controller'))} @classmethod def delete_controller(cls, args, config): """ Delete a controller config. """ #print "MOLNSProvider.delete_provider(args={0}, config={1})".format(args, config) if len(args) == 0: - print "USAGE: molns cluser delete name" - return + raise MOLNSException("USAGE: molns cluser delete name") config.delete_object(name=args[0], kind='Controller') @classmethod @@ -181,7 +233,7 @@ def upload_controller(cls, args, config): cmd = ['/usr/bin/scp','-r','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] print " ".join(cmd) subprocess.call(cmd) - print "SSH process completed" + print "SCP process completed" @classmethod def put_controller(cls, args, config): @@ -229,8 +281,7 @@ def status_controller(cls, args, config): table_data.append([controller_name, status, 'controller', provider_name, i.provider_instance_identifier, i.ip_address]) else: - print "No instance running for this controller" - return + return {'msg': "No instance running for this controller"} # Check if any worker instances are assigned to this controller instance_list = config.get_worker_instances(controller_id=controller_obj.id) if len(instance_list) > 0: @@ -244,7 +295,6 @@ def status_controller(cls, args, config): else: instance_list = config.get_all_instances() if len(instance_list) > 0: - print "Current instances:" table_data = [] for i in instance_list: provider_name = config.get_object_by_id(i.provider_id, 'Provider').name @@ -255,10 +305,11 @@ def status_controller(cls, args, config): else: table_data.append([controller_name, 'controller', provider_name, i.provider_instance_identifier]) - table_print(['name','type','provider','instance id'],table_data) - print "\n\tUse 'molns status NAME' to see current status of each instance." + r = {'type':'table', 'column_names':['name','type','provider','instance id'], 'data':table_data} + r['msg']= "\n\tUse 'molns status NAME' to see current status of each instance." + return r else: - print "No instance found" + return {'msg': "No instance found"} @classmethod @@ -391,6 +442,69 @@ def start_spark(cls, args, config): ############################################### class MOLNSWorkerGroup(MOLNSbase): + @classmethod + def worker_group_export(cls, args, config): + """ Export the configuration of a worker group. """ + if len(args) < 1: + raise MOLNSException("USAGE: molns worker export name [Filename]\n"\ + "\tExport the data from the worker group with the given name.") + worker_name = args[0] + if len(args) > 1: + filename = args[1] + else: + filename = 'Molns-Export-Worker-' + worker_name + '.json' + # check if provider exists + try: + worker_obj = config.get_object(worker_name, kind='WorkerGroup') + except DatastoreException as e: + raise MOLNSException("worker group not found") + data = {'name': worker_obj.name, + 'provider_name': worker_obj.provider.name, + 'controller_name': worker_obj.controller.name, + 'config': worker_obj.config} + return {'data': json.dumps(data), + 'type': 'file', + 'filename': filename} + + @classmethod + def worker_group_import(cls, args, config, json_data=None): + """ Import the configuration of a worker group. """ + if json_data is None: + if len(args) < 1: + raise MOLNSException("USAGE: molns worker import [Filename.json]\n"\ + "\Import the data from the worker with the given name.") + filename = args[0] + with open(filename) as fd: + data = json.load(fd) + else: + data = json_data + worker_name = data['name'] + msg = '' + try: + provider_obj = config.get_object(data['provider_name'], kind='Provider') + except DatastoreException as e: + raise MOLNSException("unknown provider '{0}'".format(data['provider_name'])) + try: + controller_obj = config.get_object(data['controller_name'], kind='Controller') + except DatastoreException as e: + raise MOLNSException("unknown controller '{0}'".format(data['provider_name'])) + try: + worker_obj = config.get_object(worker_name, kind='WorkerGroup') + msg += "Found existing worker group\n" + if worker_obj.provider.name != provider_obj.name: + raise MOLNSException("Import data has provider '{0}'. Worker group {1} exists with provider {2}. provider conversion is not possible.".format(data['provider_name'], worker_obj.name, worker_obj.provider.name)) + if worker_obj.controller.name != controller_obj.name: + raise MOLNSException("Import data has controller '{0}'. Worker group {1} exists with controller {2}. provider conversion is not possible.".format(data['controller_name'], worker_obj.name, worker_obj.controller.name)) + except DatastoreException as e: + worker_obj = config.create_object(ptype=provider_obj.type, name=worker_name, kind='WorkerGroup', provider_id=provider_obj.id, controller_id=controller_obj.id) + msg += "Creating new worker group\n" + worker_obj.config = data['config'] + config.save_object(worker_obj, kind='WorkerGroup') + msg += "Worker group data imported\n" + return {'msg':msg} + + + @classmethod def setup_worker_groups(cls, args, config): """ Configure a worker group. """ @@ -441,28 +555,28 @@ def list_worker_groups(cls, args, config): """ List all the currently configured worker groups.""" groups = config.list_objects(kind='WorkerGroup') if len(groups) == 0: - print "No worker groups configured" + raise MOLNSException("No worker groups configured") else: table_data = [] for g in groups: provider_name = config.get_object_by_id(g.provider_id, 'Provider').name controller_name = config.get_object_by_id(g.controller_id, 'Controller').name table_data.append([g.name, provider_name, controller_name]) - table_print(['name', 'provider', 'controller'], table_data) + return {'type':'table','column_names':['name', 'provider', 'controller'], 'data':table_data} @classmethod def show_worker_groups(cls, args, config): """ Show all the details of a worker group config. """ if len(args) == 0: - print "USAGE: molns worker show name" + raise MOLNSException("USAGE: molns worker show name") return - print config.get_object(name=args[0], kind='WorkerGroup') + return {'msg': str(config.get_object(name=args[0], kind='WorkerGroup'))} @classmethod def delete_worker_groups(cls, args, config): """ Delete a worker group config. """ if len(args) == 0: - print "USAGE: molns worker delete name" + raise MOLNSException("USAGE: molns worker delete name") return config.delete_object(name=args[0], kind='WorkerGroup') @@ -485,11 +599,11 @@ def status_worker_groups(cls, args, config): provider_name = config.get_object_by_id(i.provider_id, 'Provider').name status = worker_obj.get_instance_status(i) table_data.append([worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) - table_print(['name','status','type','provider','instance id', 'IP address'],table_data) + return {'type':'table','column_names':['name','status','type','provider','instance id', 'IP address'],'data':table_data} else: - print "No worker instances running for this cluster" + return {'msg': "No worker instances running for this cluster"} else: - print "USAGE: molns worker status NAME" + raise MOLNSException("USAGE: molns worker status NAME") @classmethod def start_worker_groups(cls, args, config): @@ -669,53 +783,53 @@ class MOLNSProvider(): def provider_export(cls, args, config): """ Export the configuration of a provider. """ if len(args) < 1: - print "USAGE: molns provider export name [Filename]" - print "\Export the data from the provider with the given name." - return + raise MOLNSException("USAGE: molns provider export name [Filename]\n"\ + "\tExport the data from the provider with the given name.") provider_name = args[0] + if len(args) > 1: + filename = args[1] + else: + filename = 'Molns-Export-Provider-' + provider_name + '.json' # check if provider exists try: provider_obj = config.get_object(args[0], kind='Provider') except DatastoreException as e: - print "provider not found" - return + raise MOLNSException("provider not found") data = {'name': provider_obj.name, 'type': provider_obj.type, 'config': provider_obj.config} - if len(args) > 1 and args[1] is not None: - with open(args[1],'w+') as fd: - json.dump(data, fd) - else: - print json.dumps(data) + return {'data': json.dumps(data), + 'type': 'file', + 'filename': filename} @classmethod - def provider_import(cls, args, config): + def provider_import(cls, args, config, json_data=None): """ Import the configuration of a provider. """ - if len(args) < 1: - print "USAGE: molns provider import [Filename.json]" - print "\Import the data from the provider with the given name." - return - filename = args[0] - with open(filename) as fd: - data = json.load(fd) + if json_data is None: + if len(args) < 1: + raise MOLNSException("USAGE: molns provider import [Filename.json]\n"\ + "\Import the data from the provider with the given name.") + filename = args[0] + with open(filename) as fd: + data = json.load(fd) + else: + data = json_data provider_name = data['name'] + msg = '' if data['type'] not in VALID_PROVIDER_TYPES: - print "Error: unknown provider type '{0}'".format(data['type']) - return + raise MOLNSException("unknown provider type '{0}'".format(data['type'])) try: provider_obj = config.get_object(provider_name, kind='Provider') - print "Found existing provider" + msg += "Found existing provider\n" if provider_obj.type != data['type']: - print "Import data has provider type '{0}'. Provier {1} exists with type {2}. Type conversion is not possible.".format(data['type'], provider_obj.name, provider_obj.type) - return + raise MOLNSException("Import data has provider type '{0}'. Provier {1} exists with type {2}. Type conversion is not possible.".format(data['type'], provider_obj.name, provider_obj.type)) except DatastoreException as e: provider_obj = config.create_object(name=provider_name, ptype=data['type'], kind='Provider') - print "Creating new provider" + msg += "Creating new provider\n" provider_obj.config = data['config'] config.save_object(provider_obj, kind='Provider') - print "Provider data imported" - - + msg += "Provider data imported\n" + return {'msg':msg} @classmethod def provider_setup(cls, args, config): @@ -906,14 +1020,34 @@ def clear_instances(cls, args, config): class CommandException(Exception): pass -############################################### + +def process_output_exception(e): + logging.exception(e) + sys.stderr.write("Error: {0}\n".format(e)) + +def process_output(result): + if result is not None: + if type(result)==dict and 'type' in result: + if result['type'] == 'table' and 'column_names' in result and 'data' in result: + table_print(result['column_names'],result['data']) + if result['type'] == 'file' and 'filename' in result and 'data' in result: + output_to_file(result['filename'],result['data']) + elif type(result)==dict and 'msg' in result: + print result['msg'] + else: + print result + +def output_to_file(filename, data): + with open(filename,'w+') as fd: + fd.write(data) + def table_print(column_names, data): column_width = [0]*len(column_names) for i,n in enumerate(column_names): column_width[i] = len(str(n)) for row in data: if len(row) != len(column_names): - print "len(row) != len(column_names): {0} vs {1}".format(len(row), len(column_names)) + raise Exception("len(row) != len(column_names): {0} vs {1}".format(len(row), len(column_names))) for i,n in enumerate(row): if len(str(n)) > column_width[i]: column_width[i] = len(str(n)) @@ -1014,7 +1148,7 @@ def __eq__(self, other): def run(self, args, config_dir=None): config = MOLNSConfig(config_dir=config_dir) - self.function(args, config=config) + return self.function(args, config=config) ############################################### COMMAND_LIST = [ @@ -1045,6 +1179,10 @@ def run(self, args, config_dir=None): function=MOLNSController.show_controller), Command('delete', {'name':None}, function=MOLNSController.delete_controller), + Command('export',{'name':None}, + function=MOLNSController.controller_export), + Command('import',{'filename.json':None}, + function=MOLNSController.controller_import), ]), # Commands to interact with Worker-Groups SubCommand('worker',[ @@ -1066,6 +1204,10 @@ def run(self, args, config_dir=None): # function=MOLNSWorkerGroup.stop_worker_groups), Command('terminate', {'name':None}, function=MOLNSWorkerGroup.terminate_worker_groups), + Command('export',{'name':None}, + function=MOLNSWorkerGroup.worker_group_export), + Command('import',{'filename.json':None}, + function=MOLNSWorkerGroup.worker_group_import), ]), # Commands to interact with Infrastructure-Providers SubCommand('provider',[ @@ -1117,7 +1259,6 @@ def parseArgs(): if arg_list[0].startswith('--debug'): print "Turning on Debugging output" logger.setLevel(logging.DEBUG) #for Debugging - #logger.setLevel(logging.INFO) #for Debugging arg_list = arg_list[1:] if len(arg_list) == 0 or arg_list[0] =='help' or arg_list[0] == '-h': @@ -1128,10 +1269,15 @@ def parseArgs(): for cmd in COMMAND_LIST: if cmd == arg_list[0]: try: - cmd.run(arg_list[1:], config_dir=config_dir) + output = cmd.run(arg_list[1:], config_dir=config_dir) + process_output(output) return except CommandException: pass + except Exception as e: + process_output_exception(e) + return + print "unknown command: " + " ".join(arg_list) #printHelp() print "use 'molns help' to see all possible commands" From eefdd0c6d197cf16517f23558b610ea560498db6 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 28 Jul 2015 17:53:48 -0700 Subject: [PATCH 006/100] Programming API complete --- MolnsLib/EucalyptusProvider.py | 2 - MolnsLib/molns_datastore.py | 4 +- molns.py | 175 +++++++++++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 9 deletions(-) diff --git a/MolnsLib/EucalyptusProvider.py b/MolnsLib/EucalyptusProvider.py index 0f96b23..f652acf 100644 --- a/MolnsLib/EucalyptusProvider.py +++ b/MolnsLib/EucalyptusProvider.py @@ -78,8 +78,6 @@ class EucalyptusProvider(EucalyptusBase): {'q':'Eucalyptus secret key', 'default':os.environ.get('EC2_SECRET_KEY'), 'ask':True, 'obfuscate':True}), ('ec2_url', {'q':'URL of Eucalyptus service (EC2_URL)', 'default':os.environ.get('EC2_URL'), 'ask':True, 'obfuscate':False}), -# ('aws_region', -# {'q':'Eucalyptus AWS region', 'default':EucalyptusProvider_config_get_region(), 'ask':True}), ('key_name', {'q':'Eucalyptus Key Pair name', 'default':EucalyptusProvider_default_key_name(), 'ask':True}), ('group_name', diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index f86451d..8984732 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -149,7 +149,7 @@ def __init__(self, db_file=None, config_dir=None): if db_file is not None: self.engine = create_engine('sqlite:///{0}'.format(db_file)) if config_dir is None: - self.config_dir = os.path.basename(db_file) + self.config_dir = os.path.abspath(os.path.dirname(db_file)) elif config_dir is not None: if not os.path.exists(config_dir): os.makedirs(config_dir) @@ -282,6 +282,8 @@ def _get_object_data(self, d_handle, kind, ptype, p): #logging.debug("_get_object_data(): controller_id={0}".format(p.controller_id)) ret.controller = self.get_object_by_id(id=p.controller_id, kind='Controller') return ret + + def save_object(self, config, kind): """ Save the configuration of a provider object. diff --git a/molns.py b/molns.py index 29663d6..ebdde37 100755 --- a/molns.py +++ b/molns.py @@ -2,7 +2,7 @@ import os import re import sys -from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES +from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES, get_provider_handle from MolnsLib.molns_provider import ProviderException from collections import OrderedDict import subprocess @@ -118,6 +118,59 @@ def controller_import(cls, args, config, json_data=None): msg += "Controller data imported\n" return {'msg':msg} + @classmethod + def controller_get_config(cls, name=None, provider_type=None, config=None): + """ Return a list of dict of config var for the controller config. + Each dict in the list has the keys: 'key', 'value', 'type' + + Either 'name' or 'provider_type' must be specified. + If 'name' is specified, then it will retreive the value from that + config and return it in 'value' (or return the string '********' + if that config is obfuscated, such passwords). + + """ + if config is None: + raise MOLNSException("no config specified") + if name is None and provider_type is None: + raise MOLNSException("Controller name or provider type must be specified") + if name is None: + if provider_type not in VALID_PROVIDER_TYPES: + raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) + p_hand = get_provider_handle('Controller',provider_type) + obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + else: + try: + obj = config.get_object(name, kind='Controller') + except DatastoreException as e: + raise MOLNSException("Controller {0} not found".format(name)) + + ret = [] + for key, conf, value in obj.get_config_vars(): + if 'ask' in conf and not conf['ask']: + continue + if value is not None: + myval = value + else: + if 'default' in conf and conf['default']: + if callable(conf['default']): + f1 = conf['default'] + try: + myval = f1() + except TypeError: + pass + else: + myval = conf['default'] + else: + myval = None + if myval is not None and 'obfuscate' in conf and conf['obfuscate']: + myval = '********' + ret.append({ + 'key':key, + 'value': myval, + 'type':'string' + }) + return ret + @classmethod def setup_controller(cls, args, config): """Setup a controller. Set the provider configuration for the head node. Use 'worker setup' to set the configuration for worker nodes @@ -435,9 +488,6 @@ def connect_controller_to_local(cls, args, config): fd.write(client_file_data) print "Success" - @classmethod - def start_spark(cls, args, config): - """ Start Apache Spark on the cluster. """ ############################################### @@ -503,8 +553,58 @@ def worker_group_import(cls, args, config, json_data=None): msg += "Worker group data imported\n" return {'msg':msg} - - + @classmethod + def worker_group_get_config(cls, name=None, provider_type=None, config=None): + """ Return a list of dict of config var for the worker group config. + Each dict in the list has the keys: 'key', 'value', 'type' + + Either 'name' or 'provider_type' must be specified. + If 'name' is specified, then it will retreive the value from that + config and return it in 'value' (or return the string '********' + if that config is obfuscated, such passwords). + + """ + if config is None: + raise MOLNSException("no config specified") + if name is None and provider_type is None: + raise MOLNSException("'name' or 'provider_type' must be specified.") + if name is None: + if provider_type not in VALID_PROVIDER_TYPES: + raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) + p_hand = get_provider_handle('WorkerGroup',provider_type) + obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + else: + try: + obj = config.get_object(name, kind='WorkerGroup') + except DatastoreException as e: + raise MOLNSException("Worker group {0} not found".format(name)) + ret = [] + for key, conf, value in obj.get_config_vars(): + if 'ask' in conf and not conf['ask']: + continue + if value is not None: + myval = value + else: + if 'default' in conf and conf['default']: + if callable(conf['default']): + f1 = conf['default'] + try: + myval = f1() + except TypeError: + pass + else: + myval = conf['default'] + else: + myval = None + if myval is not None and 'obfuscate' in conf and conf['obfuscate']: + myval = '********' + ret.append({ + 'key':key, + 'value': myval, + 'type':'string' + }) + return ret + @classmethod def setup_worker_groups(cls, args, config): """ Configure a worker group. """ @@ -830,6 +930,58 @@ def provider_import(cls, args, config, json_data=None): config.save_object(provider_obj, kind='Provider') msg += "Provider data imported\n" return {'msg':msg} + + @classmethod + def provider_get_config(cls, name=None, provider_type=None, config=None): + """ Return a list of dict of config var for the provider config. + Each dict in the list has the keys: 'key', 'value', 'type' + + Either 'name' or 'provider_type' must be specified. + If 'name' is specified, then it will retreive the value from that + config and return it in 'value' (or return the string '********' + if that config is obfuscated, such passwords). + + """ + if config is None: + raise MOLNSException("no config specified") + if name is None and provider_type is None: + raise MOLNSException("provider name or type must be specified") + if name is None: + if provider_type not in VALID_PROVIDER_TYPES: + raise MOLNSException("unknown provider type '{0}'".format(provider_type)) + p_hand = get_provider_handle('Provider',provider_type) + obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + else: + try: + obj = config.get_object(name, kind='Provider') + except DatastoreException as e: + raise MOLNSException("provider {0} not found".format(name)) + ret = [] + for key, conf, value in obj.get_config_vars(): + if 'ask' in conf and not conf['ask']: + continue + if value is not None: + myval = value + else: + if 'default' in conf and conf['default']: + if callable(conf['default']): + f1 = conf['default'] + try: + myval = f1() + except TypeError: + pass + else: + myval = conf['default'] + else: + myval = None + if myval is not None and 'obfuscate' in conf and conf['obfuscate']: + myval = '********' + ret.append({ + 'key':key, + 'value': myval, + 'type':'string' + }) + return ret @classmethod def provider_setup(cls, args, config): @@ -869,6 +1021,17 @@ def provider_setup(cls, args, config): setup_object(provider_obj) config.save_object(provider_obj, kind='Provider') # + cls.provider_initialize(args[0], config) + + + @classmethod + def provider_initialize(cls, provider_name, config): + """ Create the MOLNS image and SSH key if necessary.""" + try: + provider_obj = config.get_object(provider_name, kind='Provider') + except DatastoreException as e: + raise MOLNSException("provider not found") + # print "Checking all config artifacts." # check for ssh key if provider_obj['key_name'] is None or provider_obj['key_name'] == '': From 176d1a4b03dfad4985bbd062367dac4f39fbd0a6 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 28 Jul 2015 17:56:41 -0700 Subject: [PATCH 007/100] disable Euca --- MolnsLib/molns_datastore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 8984732..e3588c4 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -9,7 +9,8 @@ import sys ############################################################# #VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] -VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] +#VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] +VALID_PROVIDER_TYPES = ['OpenStack', 'EC2'] ############################################################# #### SCHEMA ################################################# ############################################################# From 2b8e8ef2715c29f4fc0f3079cf02323657e9a013 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 3 Aug 2015 16:03:19 -0700 Subject: [PATCH 008/100] bug fix --- molns.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/molns.py b/molns.py index ebdde37..9191ced 100755 --- a/molns.py +++ b/molns.py @@ -28,6 +28,28 @@ def __str__(self): ############################################### class MOLNSbase(): + @classmethod + def merge_config(self, obj, config): + for key, conf, value in obj.get_config_vars(): + if key not in config: + if value is not None: + myval = value + else: + if 'default' in conf and conf['default']: + if callable(conf['default']): + f1 = conf['default'] + try: + myval = f1() + except TypeError: + myval = None + else: + myval = conf['default'] + else: + myval = None + obj.confg[key] = myval + else: + obj.confg[key] = config[key] + @classmethod def _get_workerobj(cls, args, config): # Name @@ -113,7 +135,7 @@ def controller_import(cls, args, config, json_data=None): except DatastoreException as e: controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', provider_id=provider_obj.id) msg += "Creating new controller\n" - controller_obj.config = data['config'] + cls.merge_config(controller_obj, data['config']) config.save_object(controller_obj, kind='Controller') msg += "Controller data imported\n" return {'msg':msg} @@ -548,7 +570,7 @@ def worker_group_import(cls, args, config, json_data=None): except DatastoreException as e: worker_obj = config.create_object(ptype=provider_obj.type, name=worker_name, kind='WorkerGroup', provider_id=provider_obj.id, controller_id=controller_obj.id) msg += "Creating new worker group\n" - worker_obj.config = data['config'] + cls.merge_config(worker_obj, data['config']) config.save_object(worker_obj, kind='WorkerGroup') msg += "Worker group data imported\n" return {'msg':msg} @@ -878,7 +900,7 @@ def terminate_worker_groups(cls, args, config): ############################################### -class MOLNSProvider(): +class MOLNSProvider(MOLNSbase): @classmethod def provider_export(cls, args, config): """ Export the configuration of a provider. """ @@ -926,11 +948,12 @@ def provider_import(cls, args, config, json_data=None): except DatastoreException as e: provider_obj = config.create_object(name=provider_name, ptype=data['type'], kind='Provider') msg += "Creating new provider\n" - provider_obj.config = data['config'] + cls.merge_config(provider_obj, data['config']) config.save_object(provider_obj, kind='Provider') msg += "Provider data imported\n" return {'msg':msg} - + + @classmethod def provider_get_config(cls, name=None, provider_type=None, config=None): """ Return a list of dict of config var for the provider config. @@ -1121,7 +1144,7 @@ def delete_provider(cls, args, config): config.delete_object(name=args[0], kind='Provider') ############################################### -class MOLNSInstances(): +class MOLNSInstances(MOLNSbase): @classmethod def show_instances(cls, args, config): """ List all instances in the db """ From f0d381d3aac64f95645a236277ea80b455b6b835 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 3 Aug 2015 17:22:38 -0700 Subject: [PATCH 009/100] bug fix --- molns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/molns.py b/molns.py index 9191ced..a114f51 100755 --- a/molns.py +++ b/molns.py @@ -46,9 +46,9 @@ def merge_config(self, obj, config): myval = conf['default'] else: myval = None - obj.confg[key] = myval + obj.config[key] = myval else: - obj.confg[key] = config[key] + obj.config[key] = config[key] @classmethod def _get_workerobj(cls, args, config): From 2a9339615b3d5d506ac62f6829b1eabe92e74aa8 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 3 Aug 2015 23:43:29 -0700 Subject: [PATCH 010/100] bug fix --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index a114f51..2f0b949 100755 --- a/molns.py +++ b/molns.py @@ -39,7 +39,7 @@ def merge_config(self, obj, config): if callable(conf['default']): f1 = conf['default'] try: - myval = f1() + myval = f1(obj) except TypeError: myval = None else: From b102e4c7eb706f58d4f0be4c976b7373b261860a Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 6 Aug 2015 09:00:07 -0700 Subject: [PATCH 011/100] API can set password for controller start --- molns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/molns.py b/molns.py index 2f0b949..3819b38 100755 --- a/molns.py +++ b/molns.py @@ -388,7 +388,7 @@ def status_controller(cls, args, config): @classmethod - def start_controller(cls, args, config): + def start_controller(cls, args, config, password=None): """ Start the MOLNs controller. """ logging.debug("MOLNSController.start_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) @@ -414,7 +414,7 @@ def start_controller(cls, args, config): inst = controller_obj.start_instance() # deploying sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) - sshdeploy.deploy_ipython_controller(inst.ip_address) + sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password) sshdeploy.deploy_molns_webserver(inst.ip_address) #sshdeploy.deploy_stochss(inst.ip_address, port=443) From 6298eef16f57370b6e0eed5567356dddf07210e0 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 6 Aug 2015 15:39:10 -0700 Subject: [PATCH 012/100] adding questions to the config output --- MolnsLib/EucalyptusProvider.py | 6 +++--- MolnsLib/molns_datastore.py | 3 +-- molns.py | 2 ++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/MolnsLib/EucalyptusProvider.py b/MolnsLib/EucalyptusProvider.py index f652acf..3f9b38f 100644 --- a/MolnsLib/EucalyptusProvider.py +++ b/MolnsLib/EucalyptusProvider.py @@ -35,9 +35,9 @@ def EucalyptusProvider_config_get_ubuntu_images_by_region(conf=None): secret_key = conf['aws_secret_key'] ec2_url = conf['ec2_url'] else: - access_key = os.environ['EC2_ACCESS_KEY'] - secret_key = os.environ['EC2_SECRET_KEY'] - ec2_url = os.environ['EC2_URL'] + access_key = os.environ.get('EC2_ACCESS_KEY') + secret_key = os.environ.get('EC2_SECRET_KEY') + ec2_url = os.environ.get('EC2_URL') try: o = urlparse(ec2_url) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index e3588c4..8984732 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -9,8 +9,7 @@ import sys ############################################################# #VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] -#VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] -VALID_PROVIDER_TYPES = ['OpenStack', 'EC2'] +VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] ############################################################# #### SCHEMA ################################################# ############################################################# diff --git a/molns.py b/molns.py index 3819b38..d318513 100755 --- a/molns.py +++ b/molns.py @@ -983,6 +983,7 @@ def provider_get_config(cls, name=None, provider_type=None, config=None): for key, conf, value in obj.get_config_vars(): if 'ask' in conf and not conf['ask']: continue + question = conf['q'] if value is not None: myval = value else: @@ -1000,6 +1001,7 @@ def provider_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ + 'question':question, 'key':key, 'value': myval, 'type':'string' From 146b7cfabe9c3825e037b4292f2b1445600b2205 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 10 Aug 2015 08:27:16 -0700 Subject: [PATCH 013/100] add question to controller and worker get_config --- MolnsLib/EucalyptusProvider.py | 33 +++++++++++++++++++-------------- molns.py | 4 ++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/MolnsLib/EucalyptusProvider.py b/MolnsLib/EucalyptusProvider.py index 3f9b38f..b79e9f1 100644 --- a/MolnsLib/EucalyptusProvider.py +++ b/MolnsLib/EucalyptusProvider.py @@ -144,6 +144,11 @@ def check_molns_image(self): def create_molns_image(self): """ Create the molns image is created. """ self._connect() + # clear the network-related persisent udev rules: + #echo "" > /etc/udev/rules.d/70-persistent-net.rules + #echo "" > /lib/udev/rules.d/75-persistent-net-generator.rules + # + # start vm instances = self.eucalyptus.start_eucalyptus_instances(image_id=self.config["ubuntu_image_name"]) instance = instances[0] @@ -157,21 +162,21 @@ def create_molns_image(self): # create image logging.debug("Shutting down instance") self.eucalyptus.stop_eucalyptus_instances([instance]) - #logging.debug("Creating image") - #image_id = instance.create_image(name=self._get_image_name()) - logging.debug("Finding volume of instance") - vol = None - for v in self.eucalyptus.conn.get_all_volumes(): - if v.attach_data is not None and v.attach_data.instance_id == instance.id: - vol = v - break - if vol is None: - raise Exception("Can not find volume associated with instance. Base image must be an EBS backed image.") - snap = vol.create_snapshot() - logging.debug('Snapshot {0} of volume {1}'.format(snap.id, vol.id)) + logging.debug("Creating image") + image_id = instance.create_image(name=self._get_image_name()) + #logging.debug("Finding volume of instance") + #vol = None + #for v in self.eucalyptus.conn.get_all_volumes(): + # if v.attach_data is not None and v.attach_data.instance_id == instance.id: + # vol = v + # break + #if vol is None: + # raise Exception("Can not find volume associated with instance. Base image must be an EBS backed image.") + #snap = vol.create_snapshot() + #logging.debug('Snapshot {0} of volume {1}'.format(snap.id, vol.id)) #image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id, delete_root_volume_on_termination=True) - #deleteOnTermination - image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id) + ##deleteOnTermination + #image_id = self.eucalyptus.conn.register_image(name=self._get_image_name(), snapshot_id=snap.id) logging.debug("Image created: {0}".format(image_id)) except Exception as e: logging.exception(e) diff --git a/molns.py b/molns.py index d318513..934c588 100755 --- a/molns.py +++ b/molns.py @@ -170,6 +170,7 @@ def controller_get_config(cls, name=None, provider_type=None, config=None): for key, conf, value in obj.get_config_vars(): if 'ask' in conf and not conf['ask']: continue + question = conf['q'] if value is not None: myval = value else: @@ -187,6 +188,7 @@ def controller_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ + 'question':question, 'key':key, 'value': myval, 'type':'string' @@ -604,6 +606,7 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): for key, conf, value in obj.get_config_vars(): if 'ask' in conf and not conf['ask']: continue + question = conf['q'] if value is not None: myval = value else: @@ -621,6 +624,7 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ + 'question':question, 'key':key, 'value': myval, 'type':'string' From 330459e1c86779ba6ad984a4e51862ccca999aee Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 10 Aug 2015 09:05:58 -0700 Subject: [PATCH 014/100] Fixed get_config to use both name and provider_type --- molns.py | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/molns.py b/molns.py index 934c588..05f25e0 100755 --- a/molns.py +++ b/molns.py @@ -155,16 +155,19 @@ def controller_get_config(cls, name=None, provider_type=None, config=None): raise MOLNSException("no config specified") if name is None and provider_type is None: raise MOLNSException("Controller name or provider type must be specified") - if name is None: + obj = None + if obj is None and name is not None: + try: + obj = config.get_object(name, kind='Controller') + except DatastoreException as e: + pass + if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) p_hand = get_provider_handle('Controller',provider_type) obj = p_hand('__tmp__',data={},config_dir=config.config_dir) - else: - try: - obj = config.get_object(name, kind='Controller') - except DatastoreException as e: - raise MOLNSException("Controller {0} not found".format(name)) + if obj is None: + raise MOLNSException("Controller {0} not found".format(name)) ret = [] for key, conf, value in obj.get_config_vars(): @@ -592,16 +595,19 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): raise MOLNSException("no config specified") if name is None and provider_type is None: raise MOLNSException("'name' or 'provider_type' must be specified.") - if name is None: + obj = None + if obj is None and name is not None: + try: + obj = config.get_object(name, kind='WorkerGroup') + except DatastoreException as e: + pass + if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) p_hand = get_provider_handle('WorkerGroup',provider_type) obj = p_hand('__tmp__',data={},config_dir=config.config_dir) - else: - try: - obj = config.get_object(name, kind='WorkerGroup') - except DatastoreException as e: - raise MOLNSException("Worker group {0} not found".format(name)) + if obj is None: + raise MOLNSException("Worker group {0} not found".format(name)) ret = [] for key, conf, value in obj.get_config_vars(): if 'ask' in conf and not conf['ask']: @@ -973,16 +979,19 @@ def provider_get_config(cls, name=None, provider_type=None, config=None): raise MOLNSException("no config specified") if name is None and provider_type is None: raise MOLNSException("provider name or type must be specified") - if name is None: + obj = None + if obj is None and name is not None: + try: + obj = config.get_object(name, kind='Provider') + except DatastoreException as e: + pass + if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("unknown provider type '{0}'".format(provider_type)) p_hand = get_provider_handle('Provider',provider_type) obj = p_hand('__tmp__',data={},config_dir=config.config_dir) - else: - try: - obj = config.get_object(name, kind='Provider') - except DatastoreException as e: - raise MOLNSException("provider {0} not found".format(name)) + if obj is None: + raise MOLNSException("provider {0} not found".format(name)) ret = [] for key, conf, value in obj.get_config_vars(): if 'ask' in conf and not conf['ask']: From b0630c13926cbe65c215f9fc8f0265a25a147ec3 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 7 Sep 2015 07:41:56 -0700 Subject: [PATCH 015/100] Fixed controller status --- molns.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/molns.py b/molns.py index 05f25e0..e3a1caf 100755 --- a/molns.py +++ b/molns.py @@ -371,7 +371,9 @@ def status_controller(cls, args, config): provider_name = config.get_object_by_id(i.provider_id, 'Provider').name status = worker_obj.get_instance_status(i) table_data.append([worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) - table_print(['name','status','type','provider','instance id', 'IP address'],table_data) + #table_print(['name','status','type','provider','instance id', 'IP address'],table_data) + r = {'type':'table', 'column_names':['name','status','type','provider','instance id', 'IP address'], 'data':table_data} + return r else: instance_list = config.get_all_instances() if len(instance_list) > 0: @@ -1138,7 +1140,9 @@ def provider_list(cls, args, config): table_data = [] for p in providers: table_data.append([p.name, p.type]) - table_print(['name', 'type'], table_data) + #table_print(['name', 'type'], table_data) + r = {{'type':'table', 'column_names':['name', 'type'],'data':table_data} + return r @classmethod def show_provider(cls, args, config): From 2f876a44339c98a890f9a32115dacd7959c2b01b Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 8 Sep 2015 05:14:40 -0700 Subject: [PATCH 016/100] typo --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index e3a1caf..4ed9f35 100755 --- a/molns.py +++ b/molns.py @@ -1141,7 +1141,7 @@ def provider_list(cls, args, config): for p in providers: table_data.append([p.name, p.type]) #table_print(['name', 'type'], table_data) - r = {{'type':'table', 'column_names':['name', 'type'],'data':table_data} + r = {'type':'table', 'column_names':['name', 'type'],'data':table_data} return r @classmethod From 57babc0396abc457f15d61849f5f49017e6f6c9f Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 8 Sep 2015 09:14:26 -0700 Subject: [PATCH 017/100] typo --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index e3a1caf..4ed9f35 100755 --- a/molns.py +++ b/molns.py @@ -1141,7 +1141,7 @@ def provider_list(cls, args, config): for p in providers: table_data.append([p.name, p.type]) #table_print(['name', 'type'], table_data) - r = {{'type':'table', 'column_names':['name', 'type'],'data':table_data} + r = {'type':'table', 'column_names':['name', 'type'],'data':table_data} return r @classmethod From b53bf1e2fb4cad48e2784adceba9f45d34b5d467 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Wed, 9 Sep 2015 12:38:57 -0700 Subject: [PATCH 018/100] initial function prototypes --- molns.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/molns.py b/molns.py index 4ed9f35..d6d6912 100755 --- a/molns.py +++ b/molns.py @@ -317,7 +317,7 @@ def upload_controller(cls, args, config): @classmethod def put_controller(cls, args, config): - """ Copy a local file to the controller's shared area. """ + """ Copy a local file to the controller's and workers' shared area. """ logging.debug("MOLNSController.put_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) if controller_obj is None: return @@ -1215,6 +1215,41 @@ def clear_instances(cls, args, config): print "No instance found" +############################################### + +class MOLNSExec(MOLNSbase): + @classmethod + def start_job(cls, args, config): + ''' Execute a process on the controller.''' + raise Execption('TODO') + + @classmethod + def job_status(cls, args, config): + ''' Check if a process is still running on the controller.''' + raise Execption('TODO') + + @classmethod + def job_logs(cls, args, config): + ''' Return the output (stdout/stderr) of the process.''' + raise Execption('TODO') + + @classmethod + def fetch_job_results(cls, args, config): + ''' Transfer files created by the process from the controller to local file system.''' + raise Execption('TODO') + + + @classmethod + def cleanup_job(cls, args, config): + ''' Remove process files from the controller (will kill active processes if running).''' + raise Execption('TODO') + + @classmethod + def list_jobs(cls, args, config): + ''' List all jobs. ''' + raise Execption('TODO') + + ############################################################################################## ############################################################################################## ############################################################################################## From 421d992f4547d196034c53080b4fbf6238b3cbe8 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Wed, 9 Sep 2015 20:36:07 -0700 Subject: [PATCH 019/100] Fix for #41 --- MolnsLib/molns_datastore.py | 14 +++++++++++--- molns.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 8984732..00093c5 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -277,12 +277,20 @@ def _get_object_data(self, d_handle, kind, ptype, p): ret.datastore = self if 'provider_id' in p.__dict__: #logging.debug("_get_object_data(): provider_id={0}".format(p.provider_id)) - ret.provider = self.get_object_by_id(id=p.provider_id, kind='Provider') + try: + ret.provider = self.get_object_by_id(id=p.provider_id, kind='Provider') + except DatastoreException as e: + logging.debug('Error: provider {0} not found'.format(p.provider_id)) + ret.provider = None if 'controller_id' in p.__dict__: #logging.debug("_get_object_data(): controller_id={0}".format(p.controller_id)) - ret.controller = self.get_object_by_id(id=p.controller_id, kind='Controller') + try: + ret.controller = self.get_object_by_id(id=p.controller_id, kind='Controller') + except DatastoreException as e: + logging.debug('Error: controller {0} not found'.format(p.controller_id)) + ret.controller = None return ret - + def save_object(self, config, kind): diff --git a/molns.py b/molns.py index 4ed9f35..d7f9acd 100755 --- a/molns.py +++ b/molns.py @@ -242,7 +242,11 @@ def list_controller(cls, args, config): else: table_data = [] for c in controllers: - provider_name = config.get_object_by_id(c.provider_id, 'Provider').name + try: + p = config.get_object_by_id(c.provider_id, 'Provider') + provider_name = p.name + except DatastoreException as e: + provider_name = 'ERROR: {0}'.format(e) table_data.append([c.name, provider_name]) return {'type':'table','column_names':['name', 'provider'], 'data':table_data} @@ -355,7 +359,12 @@ def status_controller(cls, args, config): table_data = [] if len(instance_list) > 0: for i in instance_list: - provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + #provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + try: + p = config.get_object_by_id(i.provider_id, 'Provider') + provider_name = p.name + except DatastoreException as e: + provider_name = 'ERROR: {0}'.format(e) controller_name = config.get_object_by_id(i.controller_id, 'Controller').name status = controller_obj.get_instance_status(i) table_data.append([controller_name, status, 'controller', provider_name, i.provider_instance_identifier, i.ip_address]) @@ -368,7 +377,12 @@ def status_controller(cls, args, config): for i in instance_list: worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name worker_obj = cls._get_workerobj([worker_name], config) - provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + #provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + try: + p = config.get_object_by_id(i.provider_id, 'Provider') + provider_name = p.name + except DatastoreException as e: + provider_name = 'ERROR: {0}'.format(e) status = worker_obj.get_instance_status(i) table_data.append([worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) #table_print(['name','status','type','provider','instance id', 'IP address'],table_data) @@ -693,8 +707,17 @@ def list_worker_groups(cls, args, config): else: table_data = [] for g in groups: - provider_name = config.get_object_by_id(g.provider_id, 'Provider').name - controller_name = config.get_object_by_id(g.controller_id, 'Controller').name + #provider_name = config.get_object_by_id(g.provider_id, 'Provider').name + try: + p = config.get_object_by_id(g.provider_id, 'Provider') + provider_name = p.name + except DatastoreException as e: + provider_name = 'ERROR: {0}'.format(e) + try: + c = config.get_object_by_id(g.controller_id, 'Controller') + controller_name = c.name + except DatastoreException as e: + controller_name = 'ERROR: {0}'.format(e) table_data.append([g.name, provider_name, controller_name]) return {'type':'table','column_names':['name', 'provider', 'controller'], 'data':table_data} From e0f2e48aa0c0cce114954ab37fef567d3e5272f2 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 13 Sep 2015 09:53:36 -0700 Subject: [PATCH 020/100] Exec command data store --- MolnsLib/molns_datastore.py | 48 +++++++++++++++++++++ molns.py | 86 +++++++++++++++++++++++++++++++++---- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 00093c5..175203f 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -7,6 +7,8 @@ import os import logging import sys +import uuid +import datetime ############################################################# #VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] @@ -95,6 +97,18 @@ class Instance(Base): def __str__(self): return "Instance({0}): provider_instance_identifier={1} provider_id={2} controller_id={3} worker_group_id={4}".format(self.id, self.provider_instance_identifier, self.provider_id, self.controller_id, self.worker_group_id) +class ExecJob(Base): + """ DB object for MOLNS exec jobs. """ + __tablename__ = 'jobs' + id = Column(Integer, Sequence('instance_id_seq'), primary_key=True) + controller_id = Column(Integer) + exec_str = Column(String) + jobID = Column(String) + date = Column(String) + + def __str__(self): + return "ExecJob({0}): jobID={1} controller_id={2}, exec_str={3}".format(self.id, self.jobID, self.controller_id, self.exec_str) + class DatastoreException(Exception): pass @@ -392,5 +406,39 @@ def delete_instance(self, instance): self.session.delete(instance) self.session.commit() + def get_all_jobs(self, controller_id=None): + if controller_id is not None: + #logging.debug("get_all_instances by controller_id={0}".format(controller_id)) + ret = self.session.query(ExecJob).filter_by(controller_id=controller_id).all() + else: + ret = self.session.query(ExecJob).all() + if ret is None: + return [] + else: + return ret + + def get_job(self, jobID): + """ Get the objet for a job. """ + #logging.debug("get_job(jobID={0})".format(jobID)) + j = self.session.query(ExecJob).filter_by(jobID=jobID).first() + if j is None: + raise DatastoreException("Job {0} not found".format(jobID)) + return j + + def start_job(self, controller_id=None, exec_str=None): + """ Create the objet for a job. """ + date_str = str(datetime.datetime.now()) + jobID = str(uuid.uuid4()) + j = ExecJob(jobID=jobID, controller_id=controller_id, exec_str=exec_str, date=date_str) + self.session.add(j) + self.session.commit() + logging.debug("Creating ExecJob: {0}".format(j)) + return j + + def delete_job(self, job): + self.session.delete(job) + self.session.commit() + + diff --git a/molns.py b/molns.py index db12731..cfa1ff0 100755 --- a/molns.py +++ b/molns.py @@ -9,6 +9,7 @@ from MolnsLib.ssh_deploy import SSHDeploy import multiprocessing import json +from collections import OrderedDict import logging logger = logging.getLogger() @@ -1244,33 +1245,86 @@ class MOLNSExec(MOLNSbase): @classmethod def start_job(cls, args, config): ''' Execute a process on the controller.''' - raise Execption('TODO') + # Get Controller + if len(args) < 2: + raise MOLNSException("USAGE: molns exec start name [Command]\n"\ + "\tExecute 'Command' on the controller with the given name.") + + else: + controller_obj = cls._get_controllerobj(args, config) + if controller_obj is None: + raise Exception("Countroller {0} not found".format(args[0])) + # Check if controller is running + instance_list = config.get_all_instances(controller_id=controller_obj.id) + is_running = False + if len(instance_list) > 0: + for i in instance_list: + status = controller_obj.get_instance_status(i) + if status == controller_obj.STATUS_RUNNING: + is_running = True + break + if not is_running: + raise MOLNSException("Controller {0} is not running.".format(args[0])) + # Create Datastore object + exec_str = args[1] + config.start_job(controller_id=controller_obj.id, exec_str=exec_str) + # parse command, retreive files to upload (iff they are in the local directory) + # create remote direct=ory + # transfer files, and helper file (to .molns subdirectory) + # execute command + # + return {'msg':"Job started."} + @classmethod def job_status(cls, args, config): ''' Check if a process is still running on the controller.''' - raise Execption('TODO') + raise Exception('TODO') @classmethod def job_logs(cls, args, config): ''' Return the output (stdout/stderr) of the process.''' - raise Execption('TODO') + raise Exception('TODO') @classmethod def fetch_job_results(cls, args, config): ''' Transfer files created by the process from the controller to local file system.''' - raise Execption('TODO') + raise Exception('TODO') @classmethod def cleanup_job(cls, args, config): ''' Remove process files from the controller (will kill active processes if running).''' - raise Execption('TODO') + if len(args) < 1: + raise MOLNSException("USAGE: molns exec cleanup [JobID]\n"\ + "\tRemove process files from the controller (will kill active processes if running).") + j = config.get_job(jobID=args[0]) + config.delete_job(j) + return {'msg':"Job {0} deleted".format(args[0])} @classmethod def list_jobs(cls, args, config): - ''' List all jobs. ''' - raise Execption('TODO') + ''' List all jobs. If 'name' is specified, list all jobs on named controller.''' + if len(args) > 0: + controller_obj = cls._get_controllerobj(args, config) + if controller_obj is None: + raise Exception("Countroller {0} not found".format(args[0])) + jobs = config.get_all_jobs(controller_id=controller_obj.id) + else: + jobs = config.get_all_jobs() + + if len(jobs) == 0: + return {'msg':"No jobs found"} + else: + table_data = [] + for j in jobs: + try: + p = config.get_object_by_id(j.controller_id, 'Controller') + controller_name = p.name + except DatastoreException as e: + controller_name = 'ERROR: {0}'.format(e) + table_data.append([j.jobID, controller_name, j.exec_str, j.date]) + return {'type':'table','column_names':['JobID', 'Controller', 'Command', 'Date'], 'data':table_data} ############################################################################################## @@ -1463,8 +1517,8 @@ def run(self, args, config_dir=None): function=MOLNSWorkerGroup.add_worker_groups), Command('status', {'name':None}, function=MOLNSWorkerGroup.status_worker_groups), - #Command('stop', {'name':None}, - # function=MOLNSWorkerGroup.stop_worker_groups), + Command('stop', {'name':None}, + function=MOLNSWorkerGroup.terminate_worker_groups), Command('terminate', {'name':None}, function=MOLNSWorkerGroup.terminate_worker_groups), Command('export',{'name':None}, @@ -1498,6 +1552,20 @@ def run(self, args, config_dir=None): Command('clear', {}, function=MOLNSInstances.clear_instances), ]), + SubCommand('exec',[ + Command('start', OrderedDict([('name',None), ('command',None)]), + function=MOLNSExec.start_job), + Command('status', {'jobID':None}, + function=MOLNSExec.job_status), + Command('logs', {'jobID':None}, + function=MOLNSExec.job_logs), + Command('fetch', OrderedDict([('jobID',None), ('filename', None)]), + function=MOLNSExec.fetch_job_results), + Command('cleanup', {'jobID':None}, + function=MOLNSExec.cleanup_job), + Command('list', {'name':None}, + function=MOLNSExec.list_jobs), + ]), ] From 9688b24bfe129f7045e96d99f0aaa455d6f8b70a Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 15 Sep 2015 09:45:29 -0700 Subject: [PATCH 021/100] Remote execution API --- MolnsLib/molns_exec_helper.py | 40 +++++++++++ MolnsLib/ssh_deploy.py | 124 ++++++++++++++++++++++++++++++++++ molns.py | 84 +++++++++++++++++++---- 3 files changed, 235 insertions(+), 13 deletions(-) create mode 100644 MolnsLib/molns_exec_helper.py diff --git a/MolnsLib/molns_exec_helper.py b/MolnsLib/molns_exec_helper.py new file mode 100644 index 0000000..6f3a5b2 --- /dev/null +++ b/MolnsLib/molns_exec_helper.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import os +import subprocess +import shlex +import json +import traceback +import sys + + +def run_job(exec_str, stdout_file): + with open(stdout_file, 'w') as stdout_fh: + try: + p = subprocess.Popen( + shlex.split(exec_str), + stdout=stdout_fh, + stderr=stdout_fh, + ) + pid = p.pid + # create pid file + pid_file = ".molns/pid" + return_code_file = ".molns/return_value" + with open(pid_file, 'w+') as fd: + fd.write(str(pid)) + # Wait on program execution... + return_code = p.wait() + print "Return code:", return_code + if return_code_file is not None: + with open(return_code_file, 'w+') as fd: + fd.write(str(return_code)) + except Exception as e: + stdout_fh.write('Error: {}'.format(str(e))) + stdout_fh.write(traceback.format_exc()) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + +if __name__ == "__main__": + with open(".molns/cmd",'r') as fd: + exec_str = fd.read() + print "exec_str", exec_str + run_job(exec_str, ".molns/stdout") diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 129b5ce..c160a25 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -30,6 +30,9 @@ class SSHDeploy: DEFAULT_IPCONTROLLER_PORT = 9000 DEFAULT_PYURDME_TEMPDIR="/mnt/pyurdme_tmp" + + REMOTE_EXEC_JOB_PATH = "/mnt/molnsexec" + def __init__(self, config=None, config_dir=None): @@ -313,6 +316,127 @@ def get_number_processors(self): except Exception as e: raise SSHDeployException("Could not determine the number of processors on the remote system: {0}".format(e)) + def deploy_remote_execution_job(self, ip_address, jobID, exec_str): + base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) + EXEC_HELPER_FILENAME = 'molns_exec_helper.py' + try: + self.connect(ip_address, self.ssh_endpoint) + # parse command, retreive files to upload (iff they are in the local directory) + # create remote direct=ory + self.exec_command("sudo mkdir -p {0}".format(base_path)) + self.exec_command("sudo chown ubuntu {0}".format(base_path)) + self.exec_command("mkdir -p {0}/.molns/".format(base_path)) + sftp = self.ssh.open_sftp() + # Parse exec_str to get job files + files_to_transfer = [] + remote_command_list = [] + for c in exec_str.split(): + if c.startswith('~'): + c = os.path.expanduser(c) + if os.path.isfile(c): + files_to_transfer.append(c) + remote_command_list.append(os.path.basename(c)) + else: + remote_command_list.append(c) + # Transfer job files + for f in files_to_transfer: + logging.debug('Uploading file {0}'.format(f)) + sftp.put(f, "{0}/{1}".format(base_path, os.path.basename(f))) + # Transfer helper file (to .molns subdirectory) + logging.debug('Uploading file {0}'.format(EXEC_HELPER_FILENAME)) + sftp.put( + os.path.join(os.path.dirname(os.path.abspath(__file__)),EXEC_HELPER_FILENAME), + "{0}/.molns/{1}".format(base_path,EXEC_HELPER_FILENAME) + ) + # Write 'cmd' file + remote_command = " ".join(remote_command_list) + logging.debug("Writing remote_command = {0}".format(remote_command)) + cmd_file = sftp.file("{0}/.molns/{1}".format(base_path,'cmd'), 'w') + cmd_file.write(remote_command) + cmd_file.close() + # execute command + logging.debug("Executing command") + self.exec_command("cd {0};python {0}/.molns/{1} &".format(base_path, EXEC_HELPER_FILENAME)) + self.ssh.close() + except Exception as e: + print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + def remote_execution_job_status(self, ip_address, jobID): + ''' Check the status of a remote process. + + Returns: Tuple with two elements: (Is_Running, Message) + Is_Running: bool True if the process is running + Message: str Description of the status + ''' + base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) + try: + self.connect(ip_address, self.ssh_endpoint) + sftp = self.ssh.open_sftp() + # Does the 'pid' file exists remotely? + try: + sftp.stat("{0}/.molns/pid".format(base_path)) + except (IOError, OSError) as e: + self.ssh.close() + raise SSHDeployException("Remote process not started (pid file not found") + # Does the 'return_value' file exist? + try: + sftp.stat("{0}/.molns/return_value".format(base_path)) + # Process is complete + return (False, "Remote process finished") + except (IOError, OSError) as e: + pass + # is the process running? + try: + self.exec_command("kill -0 `cat {0}/.molns/pid` > /dev/null 2&>1".format(base_path)) + return (True, "Remote process running") + except SSHDeployException as e: + raise SSHDeployException("Remote process not running (process not found)") + finally: + self.ssh.close() + except Exception as e: + print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + def remote_execution_get_job_logs(self, ip_address, jobID, seek): + base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) + try: + self.connect(ip_address, self.ssh_endpoint) + sftp = self.ssh.open_sftp() + log = sftp.file("{0}/.molns/stdout".format(base_path), 'r') + log.seek(seek) + output = log.read() + self.ssh.close() + return output + except Exception as e: + print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + def remote_execution_delete_job(self, ip_address, jobID): + base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) + try: + self.connect(ip_address, self.ssh_endpoint) + self.exec_command("rm -rf {0}/* {0}/.molns/".format(base_path)) + self.exec_command("sudo rmdir {0}".format(base_path)) + self.ssh.close() + except Exception as e: + print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + def remote_execution_fetch_file(self, ip_address, jobID, filename): + base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) + try: + self.connect(ip_address, self.ssh_endpoint) + sftp = self.ssh.open_sftp() + sftp.get("{0}/{1}".format(base_path, filename), filename) + self.ssh.close() + except Exception as e: + print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) + raise sys.exc_info()[1], None, sys.exc_info()[2] + + + +#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% def deploy_stochss(self, ip_address, port=1443): try: print "{0}:{1}".format(ip_address, self.ssh_endpoint) diff --git a/molns.py b/molns.py index cfa1ff0..8340f3a 100755 --- a/molns.py +++ b/molns.py @@ -1242,6 +1242,22 @@ def clear_instances(cls, args, config): ############################################### class MOLNSExec(MOLNSbase): + @classmethod + def _get_ip_for_job(cls, job, config): + instance_list = config.get_controller_instances(controller_id=job.controller_id) + controller_obj = config.get_object_by_id(job.controller_id, 'Controller') + if controller_obj is None: + raise MOLNSException("Could not find the controller for this job") + # Check if they are running + ip = None + if len(instance_list) > 0: + for i in instance_list: + status = controller_obj.get_instance_status(i) + logging.debug("instance={0} has status={1}".format(i, status)) + if status == controller_obj.STATUS_RUNNING: + ip = i.ip_address + return ip, controller_obj + @classmethod def start_job(cls, args, config): ''' Execute a process on the controller.''' @@ -1256,40 +1272,75 @@ def start_job(cls, args, config): raise Exception("Countroller {0} not found".format(args[0])) # Check if controller is running instance_list = config.get_all_instances(controller_id=controller_obj.id) - is_running = False + inst = None if len(instance_list) > 0: for i in instance_list: status = controller_obj.get_instance_status(i) if status == controller_obj.STATUS_RUNNING: - is_running = True + inst = i break - if not is_running: + if inst is None: raise MOLNSException("Controller {0} is not running.".format(args[0])) # Create Datastore object exec_str = args[1] - config.start_job(controller_id=controller_obj.id, exec_str=exec_str) - # parse command, retreive files to upload (iff they are in the local directory) - # create remote direct=ory - # transfer files, and helper file (to .molns subdirectory) + job = config.start_job(controller_id=controller_obj.id, exec_str=exec_str) # execute command + sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + sshdeploy.deploy_remote_execution_job(inst.ip_address, job.jobID, exec_str) # - return {'msg':"Job started."} - + return {'msg':"Job started, JobID={0}".format(job.jobID)} @classmethod def job_status(cls, args, config): ''' Check if a process is still running on the controller.''' - raise Exception('TODO') + if len(args) < 1: + raise MOLNSException("USAGE: molns exec status [JobID]\n"\ + "\tCheck if a process is still running on the controller.") + j = config.get_job(jobID=args[0]) + ip, controller_obj = cls._get_ip_for_job(j, config) + if ip is None: + return {'msg': "No active instance for this controller"} + sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + (running, msg) = sshdeploy.remote_execution_job_status(ip, j.jobID) + return {'running':running, 'msg':msg} @classmethod def job_logs(cls, args, config): ''' Return the output (stdout/stderr) of the process.''' - raise Exception('TODO') + if len(args) < 1: + raise MOLNSException("USAGE: molns exec logs [JobID] [seek]\n"\ + "\tReturn the output (stdout/stderr) of the process (starting from 'seek').") + j = config.get_job(jobID=args[0]) + ip, controller_obj = cls._get_ip_for_job(j, config) + seek = 0 + if len(args) > 1: + try: + seek = int(args[1]) + except Exception: + raise MOLNSException("'seek' must be an integer") + sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + logs = sshdeploy.remote_execution_get_job_logs(ip, j.jobID, seek) + return {'msg': logs} + @classmethod - def fetch_job_results(cls, args, config): + def fetch_job_results(cls, args, config, overwrite=False): ''' Transfer files created by the process from the controller to local file system.''' - raise Exception('TODO') + if len(args) < 2: + raise MOLNSException("USAGE: molns exec fetch [JobID] [filename]\n"\ + "\tRemove process files from the controller (will kill active processes if running).") + filename = args[1] + j = config.get_job(jobID=args[0]) + if j is None: + return {'msg':"Job not found"} + ip, controller_obj = cls._get_ip_for_job(j, config) + if ip is None: + return {'msg': "No active instance for this controller"} + sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + if os.path.isfile(filename) and not overwrite and (len(args) < 3 or args[2] != '--force'): + raise MOLNSException("File {0} exists, use '--force' or overwrite=True to ignore.") + sshdeploy.remote_execution_fetch_file(ip, j.jobID, filename) + return {'msg': "File transfer complete."} @classmethod @@ -1299,6 +1350,13 @@ def cleanup_job(cls, args, config): raise MOLNSException("USAGE: molns exec cleanup [JobID]\n"\ "\tRemove process files from the controller (will kill active processes if running).") j = config.get_job(jobID=args[0]) + if j is None: + return {'msg':"Job not found"} + ip, controller_obj = cls._get_ip_for_job(j, config) + if ip is None: + return {'msg': "No active instance for this controller"} + sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + sshdeploy.remote_execution_delete_job(ip, j.jobID) config.delete_job(j) return {'msg':"Job {0} deleted".format(args[0])} From a7e0b604a2c8556b8db662d65f5430238b71e00a Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 15 Sep 2015 11:11:33 -0700 Subject: [PATCH 022/100] Access jobs by ID or JobID --- MolnsLib/molns_datastore.py | 6 +++++- MolnsLib/ssh_deploy.py | 9 +++++---- molns.py | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 175203f..92dc886 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -420,7 +420,11 @@ def get_all_jobs(self, controller_id=None): def get_job(self, jobID): """ Get the objet for a job. """ #logging.debug("get_job(jobID={0})".format(jobID)) - j = self.session.query(ExecJob).filter_by(jobID=jobID).first() + try: + id = int(jobID) + j = self.session.query(ExecJob).filter_by(id=id).first() + except Exception: + j = self.session.query(ExecJob).filter_by(jobID=jobID).first() if j is None: raise DatastoreException("Job {0} not found".format(jobID)) return j diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index c160a25..5e9b8e7 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -331,11 +331,12 @@ def deploy_remote_execution_job(self, ip_address, jobID, exec_str): files_to_transfer = [] remote_command_list = [] for c in exec_str.split(): + c2 = c if c.startswith('~'): - c = os.path.expanduser(c) - if os.path.isfile(c): - files_to_transfer.append(c) - remote_command_list.append(os.path.basename(c)) + c2 = os.path.expanduser(c) + if os.path.isfile(c2): + files_to_transfer.append(c2) + remote_command_list.append(os.path.basename(c2)) else: remote_command_list.append(c) # Transfer job files diff --git a/molns.py b/molns.py index 8340f3a..8072b41 100755 --- a/molns.py +++ b/molns.py @@ -1288,7 +1288,7 @@ def start_job(cls, args, config): sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_remote_execution_job(inst.ip_address, job.jobID, exec_str) # - return {'msg':"Job started, JobID={0}".format(job.jobID)} + return {'msg':"Job started, ID={1} JobID={0}".format(job.jobID,job.id)} @classmethod def job_status(cls, args, config): @@ -1381,8 +1381,8 @@ def list_jobs(cls, args, config): controller_name = p.name except DatastoreException as e: controller_name = 'ERROR: {0}'.format(e) - table_data.append([j.jobID, controller_name, j.exec_str, j.date]) - return {'type':'table','column_names':['JobID', 'Controller', 'Command', 'Date'], 'data':table_data} + table_data.append([j.id, j.jobID, controller_name, j.exec_str, j.date]) + return {'type':'table','column_names':['ID', 'JobID', 'Controller', 'Command', 'Date'], 'data':table_data} ############################################################################################## From 6c33c657fadb9e9c6ae2e38ba3667d3d21ec9952 Mon Sep 17 00:00:00 2001 From: ahellander Date: Wed, 16 Sep 2015 14:45:25 +0200 Subject: [PATCH 023/100] added gillespy, reorganized the install_software list thematically --- MolnsLib/installSoftware.py | 98 +++++++++++++++++++++++-------------- molns.py | 23 +++++++++ 2 files changed, 83 insertions(+), 38 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 29c53c7..2ea45cd 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -16,7 +16,7 @@ class InstallSW: This class is used for installing software ''' - # Install the necessary software for IPython and PyURDME. + # Contextualization, install the software for IPython and PyURDME. # Commands can be specified in 3 ways: # 1: a string # 2: a list a strings @@ -24,6 +24,8 @@ class InstallSW: # item is a 'check' command, which should error (return code 1) if the first item(s) did not # install correctly command_list = [ + + # Basic contextualization "curl http://www.ubuntu.com", # Check to make sure networking is up. "sudo apt-get update", "sudo apt-get -y install git", @@ -32,46 +34,73 @@ class InstallSW: "sudo apt-get -y install python-matplotlib python-numpy python-scipy", "sudo apt-get -y install make", "sudo apt-get -y install python-software-properties", - "sudo add-apt-repository -y ppa:fenics-packages/fenics", - "sudo apt-get update", - "sudo apt-get -y install fenics", "sudo apt-get -y install cython python-h5py", "sudo apt-get -y install python-pip python-dev build-essential", "sudo pip install pyzmq --upgrade", "sudo pip install dill cloud pygments", "sudo pip install tornado Jinja2", + # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty + "sudo apt-get -y remove python-scipy", + "sudo pip install scipy", + + # Molnsutil + [ + "sudo pip install jsonschema jsonpointer", + # EC2/S3 and OpenStack APIs + "sudo pip install boto", + "sudo apt-get -y install pandoc", + # This set of packages is really only needed for OpenStack, but molnsutil uses them + "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", + "sudo pip install python-novaclient", + "sudo easy_install -U pip", + "sudo pip install python-keystoneclient", + "sudo pip install python-swiftclient", + ], + + [ + "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", + "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", + "cd /usr/local/molnsutil && sudo python setup.py install" + ], - - # For molnsutil - "sudo pip install jsonschema jsonpointer", - # S3 and OS APIs - "sudo pip install boto", - "sudo apt-get -y install pandoc", - # This set of packages is really only needed for OpenStack, but molnsutil uses them - "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", - "sudo pip install python-novaclient", - "sudo easy_install -U pip", - "sudo pip install python-keystoneclient", - "sudo pip install python-swiftclient", # So the workers can mount the controller via SSHfs [ "sudo apt-get -y install sshfs", "sudo gpasswd -a ubuntu fuse", "echo 'ServerAliveInterval 60' >> /home/ubuntu/.ssh/config", ], - # FOR DEVELOPMENT, NEEDS TO BE TESTED - # High-performance ssh-hpn - #[ - # "sudo add-apt-repository ppa:w-rouesnel/openssh-hpn -y", - # "sudo apt-get update -y", - #], - - # IPython install + + # IPython [ "sudo rm -rf ipython;git clone --recursive https://github.com/Molns/ipython.git", - "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", - "sudo rm -rf ipython", - "ipython profile create default", - "sudo pip install terminado", #Jupyter terminals + "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", + "sudo rm -rf ipython", + "ipython profile create default", + "sudo pip install terminado", #Jupyter terminals + "python -c \"from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')\"" ], + + + ### Simulation software related to pyurdme and StochSS + + # Gillespy + [ "sudo rm -rf /usr/local/stochkit;sudo mkdir -p /usr/local/stochkit;sudo chown ubuntu /usr/local/stochkit", + "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git", + "cd /usr/local/stochkit && ./install.sh", + + "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", + "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/stochkit/ python setup.py install" + + ], + + # FeniCS/Dolfin/pyurdme + [ "sudo add-apt-repository -y ppa:fenics-packages/fenics", + "sudo apt-get update", + "sudo apt-get -y install fenics", + # Gmsh for Finite Element meshes + "sudo apt-get install -y gmsh", + ], + + # pyurdme [ "sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme", "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", "cd /usr/local/pyurdme && git checkout develop", @@ -79,21 +108,14 @@ class InstallSW: "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* .ipython/profile_default/static/custom/", "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", ], + + # example notebooks [ "rm -rf MOLNS_notebooks;git clone https://github.com/Molns/MOLNS_notebooks.git", "cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks;", "ls *.ipynb" ], - [ - "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", - "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", - "cd /usr/local/molnsutil && sudo python setup.py install" - ], - "python -c \"from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')\"", + - # Upgrade scipy from pip to get rid of six.py bug on Trusty - "sudo apt-get -y remove python-scipy", - "sudo pip install scipy", - "sync", # This is critial for some infrastructures. ] diff --git a/molns.py b/molns.py index d7f9acd..e05aa00 100755 --- a/molns.py +++ b/molns.py @@ -1454,6 +1454,29 @@ def run(self, args, config_dir=None): Command('import',{'filename.json':None}, function=MOLNSProvider.provider_import), ]), + + SubCommand('service',[ + Command('setup',{'name':None}, + function=MOLNSProvider.provider_setup), + Command('setup',{'name':None}, + function=MOLNSProvider.provider_setup), + Command('start',{'name':None}, + function=MOLNSProvider.provider_setup), + Command('rebuild',{'name':None}, + function=MOLNSProvider.provider_rebuild), + Command('list',{'name':None}, + function=MOLNSProvider.provider_list), + Command('show',{'name':None}, + function=MOLNSProvider.show_provider), + Command('terminate',{'name':None}, + function=MOLNSProvider.delete_provider), + Command('export',{'name':None}, + function=MOLNSProvider.provider_export), + Command('import',{'filename.json':None}, + function=MOLNSProvider.provider_import), + ]), + + # Commands to interact with the instance DB SubCommand('instancedb',[ Command('list', {}, From 798395f5accf7e999d84eb937be3157e92c052ea Mon Sep 17 00:00:00 2001 From: Andreas Hellander Date: Wed, 16 Sep 2015 21:12:21 +0200 Subject: [PATCH 024/100] Update README.md --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 19b0c63..0cfee6c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # MOLNs spatial stochastic simulation appliance # -MOLNs is a cloud appliance that will set up, start and manage a virtual platform for scalable, distributed computational experiments using PyURDME (www.pyurdme.org). +MOLNs is a cloud appliance that will set up, start and manage a virtual platform for scalable, distributed computational experiments using (spatial) stochastic simulation software such as PyURDME (www.pyurdme.org) and StochKit/Gillespy (www.github.com/Gillespy/gillespy). In addition, MOLNs by default makes FEniCS/Dolfin available as-a Service. + +Since MOLNs will configure and manage a virtual IPython Cluster (with a Notebook frontend), with Numpy, SciPy and Ipython Parallel enabled, it can also be useful for general contextualization and management of dynamic, cloud-agnostic (supports EC2 and OpenStack-based clouds) virtual IPython environments, even if you are not into spatial stochstic simulations in systems biology. + +Note: MOLNs is currenly compatible only with 'EC2-Classic', we are working on supporting Amazon VPC. ### Prerequisites ### To use MOLNs, you need valid credentials to an OpenStack cloud, Amazon Elastic Compute Cloud (EC2) or HP Helion public cloud. You also need Python, and the following packages: @@ -71,9 +75,10 @@ To set up a start a MOLNs virtual platform named "molns-test" in a cloud provide $ molns start molns-test $ molns worker start molns-test-workers -You will be presented with a URL for the controller node of your platform. Navigate there using a browser (Google Chrome or Firefox are recommended). The easiest way to get started using the platform is to dive into one of the provided tutorial notebooks that are made available in every fresh MOLNs virtual platform. +You will be presented with a URL for the controller node of your platform. Navigate there using a browser (Google Chrome is strongly recommended, and Safari should be avoided). The easiest way to get started using the platform is to dive into one of the provided tutorial notebooks that are made available in every fresh MOLNs virtual platform. For a complete list of the valid subcommands for molns, type + $ molns help ### Above commands explained ### From c9dc36afe0dbe3a98b86a4916fd4e5c24189727f Mon Sep 17 00:00:00 2001 From: ahellander Date: Wed, 16 Sep 2015 21:13:26 +0200 Subject: [PATCH 025/100] bug fix, enabled gmsh --- MolnsLib/installSoftware.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 2ea45cd..50bbdf7 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -39,10 +39,7 @@ class InstallSW: "sudo pip install pyzmq --upgrade", "sudo pip install dill cloud pygments", "sudo pip install tornado Jinja2", - # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty - "sudo apt-get -y remove python-scipy", - "sudo pip install scipy", - + # Molnsutil [ "sudo pip install jsonschema jsonpointer", @@ -114,7 +111,11 @@ class InstallSW: "cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks;", "ls *.ipynb" ], - + + # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty + "sudo apt-get -y remove python-scipy", + "sudo pip install scipy", + "sync", # This is critial for some infrastructures. ] From ae02099461eb9cf0d7f9200482b8f2153b537bbd Mon Sep 17 00:00:00 2001 From: ahellander Date: Thu, 17 Sep 2015 12:22:53 +0200 Subject: [PATCH 026/100] removed the service command --- MolnsLib/installSoftware.py | 10 +++++----- molns.py | 23 +---------------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 50bbdf7..1f5f4fb 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -68,11 +68,11 @@ class InstallSW: # IPython [ "sudo rm -rf ipython;git clone --recursive https://github.com/Molns/ipython.git", - "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", - "sudo rm -rf ipython", - "ipython profile create default", - "sudo pip install terminado", #Jupyter terminals - "python -c \"from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')\"" + "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", + "sudo rm -rf ipython", + "ipython profile create default", + "sudo pip install terminado", #Jupyter terminals + "python -c \"from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')\"" ], diff --git a/molns.py b/molns.py index e05aa00..1a0cbdc 100755 --- a/molns.py +++ b/molns.py @@ -1454,28 +1454,7 @@ def run(self, args, config_dir=None): Command('import',{'filename.json':None}, function=MOLNSProvider.provider_import), ]), - - SubCommand('service',[ - Command('setup',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('setup',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('start',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('rebuild',{'name':None}, - function=MOLNSProvider.provider_rebuild), - Command('list',{'name':None}, - function=MOLNSProvider.provider_list), - Command('show',{'name':None}, - function=MOLNSProvider.show_provider), - Command('terminate',{'name':None}, - function=MOLNSProvider.delete_provider), - Command('export',{'name':None}, - function=MOLNSProvider.provider_export), - Command('import',{'filename.json':None}, - function=MOLNSProvider.provider_import), - ]), - + # Commands to interact with the instance DB SubCommand('instancedb',[ From abea7d54581eaf2227e5dc9fcc4519e7ddd3c97f Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 17 Sep 2015 11:13:11 -0700 Subject: [PATCH 027/100] Bug fixes and adding ODE solver for GillesPy --- MolnsLib/installSoftware.py | 22 +++++++++++++++++----- molns.py | 23 ----------------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 50bbdf7..f702320 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -46,7 +46,7 @@ class InstallSW: # EC2/S3 and OpenStack APIs "sudo pip install boto", "sudo apt-get -y install pandoc", - # This set of packages is really only needed for OpenStack, but molnsutil uses them + # This set of packages is needed for OpenStack, as molnsutil uses them for hybrid cloud deployment "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", "sudo pip install python-novaclient", "sudo easy_install -U pip", @@ -79,13 +79,23 @@ class InstallSW: ### Simulation software related to pyurdme and StochSS # Gillespy - [ "sudo rm -rf /usr/local/stochkit;sudo mkdir -p /usr/local/stochkit;sudo chown ubuntu /usr/local/stochkit", - "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git", - "cd /usr/local/stochkit && ./install.sh", + [ "sudo rm -rf /usr/local/StochKit;sudo mkdir -p /usr/local/StochKit;sudo chown ubuntu /usr/local/StochKit", + "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", + "cd /usr/local/StochKit && ./install.sh", + + "sudo rm -rf /usr/local/ode-1.0.2;sudo mkdir -p /usr/local/ode-1.0.2/;sudo chown ubuntu /usr/local/ode-1.0.2", + "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.2.tgz?raw=true -q -O /tmp/ode.tgz", + "cd /usr/local/ && tar -xzf /tmp/ode.tgz", + "rm /tmp/ode.tgz", + "cd /usr/local/ode-1.0.2/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", + "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.2/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.2/ make 1>stdout.log 2>stderr.log", "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", - "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/stochkit/ python setup.py install" + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.2/ python setup.py install" ], @@ -115,6 +125,8 @@ class InstallSW: # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty "sudo apt-get -y remove python-scipy", "sudo pip install scipy", + + "sudo pip install jsonschema jsonpointer", #redo this install to be sure it has not been removed. "sync", # This is critial for some infrastructures. diff --git a/molns.py b/molns.py index e05aa00..d7f9acd 100755 --- a/molns.py +++ b/molns.py @@ -1454,29 +1454,6 @@ def run(self, args, config_dir=None): Command('import',{'filename.json':None}, function=MOLNSProvider.provider_import), ]), - - SubCommand('service',[ - Command('setup',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('setup',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('start',{'name':None}, - function=MOLNSProvider.provider_setup), - Command('rebuild',{'name':None}, - function=MOLNSProvider.provider_rebuild), - Command('list',{'name':None}, - function=MOLNSProvider.provider_list), - Command('show',{'name':None}, - function=MOLNSProvider.show_provider), - Command('terminate',{'name':None}, - function=MOLNSProvider.delete_provider), - Command('export',{'name':None}, - function=MOLNSProvider.provider_export), - Command('import',{'filename.json':None}, - function=MOLNSProvider.provider_import), - ]), - - # Commands to interact with the instance DB SubCommand('instancedb',[ Command('list', {}, From 542c9e48ad46da4f6d212c00f75ad17c384a769e Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 17 Sep 2015 11:45:36 -0700 Subject: [PATCH 028/100] Use release version of PyURDME --- MolnsLib/installSoftware.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 770101f..92cb2e5 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -110,8 +110,7 @@ class InstallSW: # pyurdme [ "sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme", "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", - "cd /usr/local/pyurdme && git checkout develop", - + #"cd /usr/local/pyurdme && git checkout develop", # for development only "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* .ipython/profile_default/static/custom/", "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", ], From a356a5c6e113e8084561b47945e0dcb2ab8f0294 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 21 Sep 2015 09:17:59 -0700 Subject: [PATCH 029/100] Fixes for StochSS integration --- MolnsLib/ssh_deploy.py | 31 ++++++++++++++++--------------- molns.py | 25 ++++++++++++++++--------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 5e9b8e7..c15f95f 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -280,7 +280,7 @@ def connect(self, hostname, port): time.sleep(self.SSH_CONNECT_WAITTIME) raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(hostname,self.ssh_endpoint)) - def deploy_molns_webserver(self, ip_address): + def deploy_molns_webserver(self, ip_address, openWebBrowser=True): try: self.connect(ip_address, self.ssh_endpoint) self.exec_command("sudo rm -rf /usr/local/molns_webroot") @@ -292,18 +292,19 @@ def deploy_molns_webserver(self, ip_address): self.ssh.close() print "Deploying MOLNs webserver" url = "http://{0}/".format(ip_address) - while True: - try: - req = urllib2.urlopen(url) - sys.stdout.write("\n") - sys.stdout.flush() - break - except Exception as e: - #sys.stdout.write("{0}".format(e)) - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(1) - webbrowser.open(url) + if openWebBrowser: + while True: + try: + req = urllib2.urlopen(url) + sys.stdout.write("\n") + sys.stdout.flush() + break + except Exception as e: + #sys.stdout.write("{0}".format(e)) + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(1) + webbrowser.open(url) except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] @@ -424,12 +425,12 @@ def remote_execution_delete_job(self, ip_address, jobID): print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def remote_execution_fetch_file(self, ip_address, jobID, filename): + def remote_execution_fetch_file(self, ip_address, jobID, filename, localfilename): base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) try: self.connect(ip_address, self.ssh_endpoint) sftp = self.ssh.open_sftp() - sftp.get("{0}/{1}".format(base_path, filename), filename) + sftp.get("{0}/{1}".format(base_path, filename), localfilename) self.ssh.close() except Exception as e: print "Remote execution failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) diff --git a/molns.py b/molns.py index 8072b41..e02945d 100755 --- a/molns.py +++ b/molns.py @@ -353,8 +353,11 @@ def status_controller(cls, args, config): """ Get status of the head node of a MOLNs controller. """ logging.debug("MOLNSController.status_controller(args={0})".format(args)) if len(args) > 0: - controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + try: + controller_obj = cls._get_controllerobj(args, config) + except MOLNSException: + return {} + if controller_obj is None: return {} # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) table_data = [] @@ -410,7 +413,7 @@ def status_controller(cls, args, config): @classmethod - def start_controller(cls, args, config, password=None): + def start_controller(cls, args, config, password=None, openWebBrowser=True): """ Start the MOLNs controller. """ logging.debug("MOLNSController.start_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) @@ -437,7 +440,7 @@ def start_controller(cls, args, config, password=None): # deploying sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password) - sshdeploy.deploy_molns_webserver(inst.ip_address) + sshdeploy.deploy_molns_webserver(inst.ip_address, openWebBrowser=openWebBrowser) #sshdeploy.deploy_stochss(inst.ip_address, port=443) @classmethod @@ -1288,7 +1291,7 @@ def start_job(cls, args, config): sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_remote_execution_job(inst.ip_address, job.jobID, exec_str) # - return {'msg':"Job started, ID={1} JobID={0}".format(job.jobID,job.id)} + return {'id':job.id, 'msg':"Job started, ID={1} JobID={0}".format(job.jobID,job.id)} @classmethod def job_status(cls, args, config): @@ -1299,7 +1302,7 @@ def job_status(cls, args, config): j = config.get_job(jobID=args[0]) ip, controller_obj = cls._get_ip_for_job(j, config) if ip is None: - return {'msg': "No active instance for this controller"} + return {'running':False, 'msg': "No active instance for this controller"} sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) (running, msg) = sshdeploy.remote_execution_job_status(ip, j.jobID) return {'running':running, 'msg':msg} @@ -1327,7 +1330,7 @@ def job_logs(cls, args, config): def fetch_job_results(cls, args, config, overwrite=False): ''' Transfer files created by the process from the controller to local file system.''' if len(args) < 2: - raise MOLNSException("USAGE: molns exec fetch [JobID] [filename]\n"\ + raise MOLNSException("USAGE: molns exec fetch [JobID] [filename] (destination filename)\n"\ "\tRemove process files from the controller (will kill active processes if running).") filename = args[1] j = config.get_job(jobID=args[0]) @@ -1337,9 +1340,13 @@ def fetch_job_results(cls, args, config, overwrite=False): if ip is None: return {'msg': "No active instance for this controller"} sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) - if os.path.isfile(filename) and not overwrite and (len(args) < 3 or args[2] != '--force'): + if os.path.isfile(filename) and not overwrite and (len(args) < 3 or args[-1] != '--force'): raise MOLNSException("File {0} exists, use '--force' or overwrite=True to ignore.") - sshdeploy.remote_execution_fetch_file(ip, j.jobID, filename) + if len(args) >= 3 and not args[2].startswith('--'): + localfile = args[2] + else: + localfile = filename + sshdeploy.remote_execution_fetch_file(ip, j.jobID, filename, localfile) return {'msg': "File transfer complete."} From dc39579d6023819a5b7a76e23d4da334dac6cf1c Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 22 Sep 2015 16:04:20 -0700 Subject: [PATCH 030/100] moved logging --- molns.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/molns.py b/molns.py index d7f9acd..80b49a6 100755 --- a/molns.py +++ b/molns.py @@ -11,9 +11,6 @@ import json import logging -logger = logging.getLogger() -#logger.setLevel(logging.INFO) #for Debugging -logger.setLevel(logging.CRITICAL) ############################################### class MOLNSException(Exception): pass @@ -1512,4 +1509,7 @@ def parseArgs(): if __name__ == "__main__": + logger = logging.getLogger() + #logger.setLevel(logging.INFO) #for Debugging + logger.setLevel(logging.CRITICAL) parseArgs() From 9e6a5de9a383eef8fbbecbf135d8c6a8a4aa9f99 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 22 Sep 2015 18:25:28 -0700 Subject: [PATCH 031/100] Addresses #27 --- MolnsLib/ssh_deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 129b5ce..ac55632 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -43,6 +43,7 @@ def __init__(self, config=None, config_dir=None): self.endpoint = self.DEFAULT_PRIVATE_NOTEBOOK_PORT self.ssh_endpoint = self.DEFAULT_SSH_PORT self.keyfile = config.sshkeyfilename() + self.provider_name = config.name self.ssh = paramiko.SSHClient() self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.profile = 'default' @@ -147,7 +148,7 @@ def create_s3_config(self): s3_config_file = sftp.file(remote_file_name, 'w') config = {} config["provider_type"] = self.config.type - config["bucket_name"] = "molns_storage_{0}".format(self.get_cluster_id()) + config["bucket_name"] = "molns_storage_{1}_{0}".format(self.get_cluster_id(), self.provider_name) config["credentials"] = self.config.get_config_credentials() s3_config_file.write(json.dumps(config)) s3_config_file.close() From 7a42699fe298ebeecb06288c57c068d7f2bbba3e Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 28 Sep 2015 14:48:58 -0700 Subject: [PATCH 032/100] update to molns for stability in job execution --- MolnsLib/ssh_deploy.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index ac55632..21d254e 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -123,23 +123,24 @@ def create_ipython_config(self, hostname, notebook_password=None): "c.IPControllerApp.log_level=20", "c.HeartMonitor.period=10000", "c.HeartMonitor.max_heartmonitor_misses=10", + "c.HubFactory.db_class = \"SQLiteDB\"", ])) notebook_config_file.close() - # IPython startup code - remote_file_name='{0}startup/molns_dill_startup.py'.format(self.profile_dir_server) - dill_init_file = sftp.file(remote_file_name, 'w+') - dill_init_file.write('\n'.join([ - 'import dill', - 'from IPython.utils import pickleutil', - 'pickleutil.use_dill()', - 'import logging', - "logging.getLogger('UFL').setLevel(logging.ERROR)", - "logging.getLogger('FFC').setLevel(logging.ERROR)" - "import cloud", - "logging.getLogger('Cloud').setLevel(logging.ERROR)" - ])) - dill_init_file.close() +# # IPython startup code +# remote_file_name='{0}startup/molns_dill_startup.py'.format(self.profile_dir_server) +# dill_init_file = sftp.file(remote_file_name, 'w+') +# dill_init_file.write('\n'.join([ +# 'import dill', +# 'from IPython.utils import pickleutil', +# 'pickleutil.use_dill()', +# 'import logging', +# "logging.getLogger('UFL').setLevel(logging.ERROR)", +# "logging.getLogger('FFC').setLevel(logging.ERROR)" +# "import cloud", +# "logging.getLogger('Cloud').setLevel(logging.ERROR)" +# ])) +# dill_init_file.close() sftp.close() def create_s3_config(self): From e1bcb7333cd3be9d16d2aaaf682719191ff055a3 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Wed, 30 Sep 2015 10:49:06 -0700 Subject: [PATCH 033/100] return JobID --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index 3c164ce..7dafb3a 100755 --- a/molns.py +++ b/molns.py @@ -1288,7 +1288,7 @@ def start_job(cls, args, config): sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_remote_execution_job(inst.ip_address, job.jobID, exec_str) # - return {'id':job.id, 'msg':"Job started, ID={1} JobID={0}".format(job.jobID,job.id)} + return {'JobID':job.jobID, 'id':job.id, 'msg':"Job started, ID={1} JobID={0}".format(job.jobID,job.id)} @classmethod def job_status(cls, args, config): From fd29d43eb55713e84304bab8f39c5fdeade46fb0 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sat, 3 Oct 2015 17:47:34 -0700 Subject: [PATCH 034/100] update for the new molnsutil --- MolnsLib/OpenStackProvider.py | 1 + MolnsLib/ssh_deploy.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MolnsLib/OpenStackProvider.py b/MolnsLib/OpenStackProvider.py index 26c8779..f98c191 100644 --- a/MolnsLib/OpenStackProvider.py +++ b/MolnsLib/OpenStackProvider.py @@ -192,6 +192,7 @@ def _connect(self): creds['api_key'] = self.config['nova_password'] creds['auth_url'] = self.config['nova_auth_url'] creds['project_id'] = self.config['nova_project_id'] + #creds['region_name'] = "regionOne" self.nova = novaclient.Client(self.config['nova_version'], **creds) self.connected = True diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index c15f95f..fc968da 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -418,7 +418,7 @@ def remote_execution_delete_job(self, ip_address, jobID): base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) try: self.connect(ip_address, self.ssh_endpoint) - self.exec_command("rm -rf {0}/* {0}/.molns/".format(base_path)) + self.exec_command("rm -rf {0}/* {0}/.molns*".format(base_path)) self.exec_command("sudo rmdir {0}".format(base_path)) self.ssh.close() except Exception as e: From 14b064902c7549d586d54fd38fa704572f055282 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sat, 3 Oct 2015 17:57:46 -0700 Subject: [PATCH 035/100] use the new molnsutil --- MolnsLib/installSoftware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 92cb2e5..a523498 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -56,7 +56,9 @@ class InstallSW: [ "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", - "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", + #"cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", + "cd /usr/local/ && git clone https://github.com/briandrawert/molnsutil.git", + "cd /usr/local/molnsutil && git checkout molnsutil_state", "cd /usr/local/molnsutil && sudo python setup.py install" ], From 53c53f979b618c3ab11c9d9ca2fe7b3cb89e608c Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 11 Oct 2015 13:47:55 -0700 Subject: [PATCH 036/100] specify the number of CPUs to reserve --- MolnsLib/ssh_deploy.py | 4 ++-- molns.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index fc968da..32926b5 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -489,7 +489,7 @@ def deploy_stochss(self, ip_address, port=1443): print "StochSS launch failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def deploy_ipython_controller(self, ip_address, notebook_password=None): + def deploy_ipython_controller(self, ip_address, notebook_password=None, reserved_cpus=2): controller_hostname = '' engine_file_data = '' try: @@ -522,7 +522,7 @@ def deploy_ipython_controller(self, ip_address, notebook_password=None): self.exec_command("source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2} --log-to-file".format(ip_address, self.profile, self.ipython_port), '\n') # Start one ipengine per processor num_procs = self.get_number_processors() - num_engines = num_procs - 2 + num_engines = num_procs - reserved_cpus for _ in range(num_engines): self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format(self.profile, self.ipengine_env)) diff --git a/molns.py b/molns.py index 7dafb3a..c1a82cf 100755 --- a/molns.py +++ b/molns.py @@ -410,7 +410,7 @@ def status_controller(cls, args, config): @classmethod - def start_controller(cls, args, config, password=None, openWebBrowser=True): + def start_controller(cls, args, config, password=None, openWebBrowser=True, reserved_cpus=2): """ Start the MOLNs controller. """ logging.debug("MOLNSController.start_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) @@ -436,7 +436,7 @@ def start_controller(cls, args, config, password=None, openWebBrowser=True): inst = controller_obj.start_instance() # deploying sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) - sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password) + sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password, reserved_cpus=reserved_cpus) sshdeploy.deploy_molns_webserver(inst.ip_address, openWebBrowser=openWebBrowser) #sshdeploy.deploy_stochss(inst.ip_address, port=443) From b4fa0ccbd84e0179f5592c62420e3c9f10c34e93 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 11 Oct 2015 17:39:08 -0700 Subject: [PATCH 037/100] More Debug statements --- MolnsLib/ssh_deploy.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 32926b5..7e47ebd 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -281,6 +281,7 @@ def connect(self, hostname, port): raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(hostname,self.ssh_endpoint)) def deploy_molns_webserver(self, ip_address, openWebBrowser=True): + logging.debug('deploy_molns_webserver(ip_address={0}, openWebBrowser={1})'.format(ip_address, openWebBrowser)) try: self.connect(ip_address, self.ssh_endpoint) self.exec_command("sudo rm -rf /usr/local/molns_webroot") @@ -490,6 +491,7 @@ def deploy_stochss(self, ip_address, port=1443): raise sys.exc_info()[1], None, sys.exc_info()[2] def deploy_ipython_controller(self, ip_address, notebook_password=None, reserved_cpus=2): + logging.debug('deploy_ipython_controller(ip_address={0}, reserved_cpus={1})'.format(ip_address, reserved_cpus)) controller_hostname = '' engine_file_data = '' try: @@ -520,9 +522,14 @@ def deploy_ipython_controller(self, ip_address, notebook_password=None, reserved self.create_ipython_config(ip_address, notebook_password) self.create_engine_config() self.exec_command("source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2} --log-to-file".format(ip_address, self.profile, self.ipython_port), '\n') + # Give the controller time to startup + import time + logging.debug('Waiting 5 seconds for the IPython controller to start.') + time.sleep(5) # Start one ipengine per processor num_procs = self.get_number_processors() num_engines = num_procs - reserved_cpus + logging.debug('Starting {0} engines (#cpu={1}, reserved_cpus={2})'.format(num_engines, num_procs, reserved_cpus)) for _ in range(num_engines): self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format(self.profile, self.ipengine_env)) From 346aee92cc219c6ca8d1378d4b20c4cea8b50643 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Mon, 12 Oct 2015 19:22:06 -0700 Subject: [PATCH 038/100] Controller check on exec logs --- molns.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/molns.py b/molns.py index c1a82cf..21661f7 100755 --- a/molns.py +++ b/molns.py @@ -345,6 +345,26 @@ def put_controller(cls, args, config): print "SSH process completed" + @classmethod + def is_controller_running(cls, args, config): + logging.debug("MOLNSController.is_controller_running(args={0})".format(args)) + if len(args) > 0: + try: + controller_obj = cls._get_controllerobj(args, config) + except MOLNSException: + return {} + if controller_obj is None: return False + # Check if any instances are assigned to this controller + instance_list = config.get_controller_instances(controller_id=controller_obj.id) + if len(instance_list) > 0: + for i in instance_list: + status = controller_obj.get_instance_status(i) + if status == controller_obj.get_instance_status.STATUS_RUNNING: + return True + + return False + + @classmethod def status_controller(cls, args, config): """ Get status of the head node of a MOLNs controller. """ @@ -1312,6 +1332,8 @@ def job_logs(cls, args, config): "\tReturn the output (stdout/stderr) of the process (starting from 'seek').") j = config.get_job(jobID=args[0]) ip, controller_obj = cls._get_ip_for_job(j, config) + if ip is None: + return {'msg': 'WARNING: controller is not running'} seek = 0 if len(args) > 1: try: From 58aeb32c67cf2068f1261bce073c1517b772b9b7 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 15 Oct 2015 19:04:33 -0700 Subject: [PATCH 039/100] job cleanup now will terminate running jobs --- MolnsLib/ssh_deploy.py | 6 ++++++ molns.py | 17 +++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 7e47ebd..05fd05a 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -419,6 +419,12 @@ def remote_execution_delete_job(self, ip_address, jobID): base_path = "{0}/{1}".format(self.REMOTE_EXEC_JOB_PATH,jobID) try: self.connect(ip_address, self.ssh_endpoint) + ### If process is still running, terminate it + try: + self.exec_command("kill -TERM `cat {0}/.molns/pid` > /dev/null 2&>1".format(base_path)) + except Exception as e: + pass + ### Remove the filess on the remote server self.exec_command("rm -rf {0}/* {0}/.molns*".format(base_path)) self.exec_command("sudo rmdir {0}".format(base_path)) self.ssh.close() diff --git a/molns.py b/molns.py index 21661f7..6227b3c 100755 --- a/molns.py +++ b/molns.py @@ -281,8 +281,7 @@ def ssh_controller(cls, args, config): if status == controller_obj.STATUS_RUNNING: ip = i.ip_address if ip is None: - print "No active instance for this controller" - return + raise MOLNSException("No active instance for this controller") #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) cmd = ['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)] @@ -308,8 +307,7 @@ def upload_controller(cls, args, config): if status == controller_obj.STATUS_RUNNING: ip = i.ip_address if ip is None: - print "No active instance for this controller" - return + raise MOLNSException("No active instance for this controller") #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) cmd = ['/usr/bin/scp','-r','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] @@ -335,8 +333,7 @@ def put_controller(cls, args, config): if status == controller_obj.STATUS_RUNNING: ip = i.ip_address if ip is None: - print "No active instance for this controller" - return + raise MOLNSException("No active instance for this controller") #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/shared'.format(ip)] @@ -1333,7 +1330,7 @@ def job_logs(cls, args, config): j = config.get_job(jobID=args[0]) ip, controller_obj = cls._get_ip_for_job(j, config) if ip is None: - return {'msg': 'WARNING: controller is not running'} + raise MOLNSException("No active instance for this controller") seek = 0 if len(args) > 1: try: @@ -1354,10 +1351,10 @@ def fetch_job_results(cls, args, config, overwrite=False): filename = args[1] j = config.get_job(jobID=args[0]) if j is None: - return {'msg':"Job not found"} + raise MOLNSException("Job not found") ip, controller_obj = cls._get_ip_for_job(j, config) if ip is None: - return {'msg': "No active instance for this controller"} + raise MOLNSException("No active instance for this controller") sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) if os.path.isfile(filename) and not overwrite and (len(args) < 3 or args[-1] != '--force'): raise MOLNSException("File {0} exists, use '--force' or overwrite=True to ignore.") @@ -1380,7 +1377,7 @@ def cleanup_job(cls, args, config): return {'msg':"Job not found"} ip, controller_obj = cls._get_ip_for_job(j, config) if ip is None: - return {'msg': "No active instance for this controller"} + raise MOLNSException("No active instance for this controller") sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.remote_execution_delete_job(ip, j.jobID) config.delete_job(j) From 2cf8bc2d4784e2c925e2105c76bd6307d11f5342 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Wed, 11 Nov 2015 12:24:04 -0800 Subject: [PATCH 040/100] Add the ability to set regions --- MolnsLib/OpenStackProvider.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MolnsLib/OpenStackProvider.py b/MolnsLib/OpenStackProvider.py index 26c8779..2393065 100644 --- a/MolnsLib/OpenStackProvider.py +++ b/MolnsLib/OpenStackProvider.py @@ -43,6 +43,8 @@ class OpenStackProvider(OpenStackBase): {'q':'OpenStack project_name', 'default':os.environ.get('OS_TENANT_NAME'), 'ask':True}), ('neutron_nic', {'q':'Network ID (leave empty if only one possible network)', 'default':None, 'ask':True}), + ('region_name', + {'q':'Specify the region to use', 'default':os.environ.get('OS_REGION_NAME'), 'ask':True}), ('floating_ip_pool', {'q':'Name of Floating IP Pool (leave empty if only one possible pool)', 'default':None, 'ask':True}), ('nova_version', @@ -192,6 +194,8 @@ def _connect(self): creds['api_key'] = self.config['nova_password'] creds['auth_url'] = self.config['nova_auth_url'] creds['project_id'] = self.config['nova_project_id'] + if 'region_name' in self.config and self.config['region_name'] is not None: + creds['region_name'] = self.config['region_name'] self.nova = novaclient.Client(self.config['nova_version'], **creds) self.connected = True From bedea9f02f4910933e44e678b8be86d146d1f16c Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Wed, 11 Nov 2015 12:28:41 -0800 Subject: [PATCH 041/100] updated message --- MolnsLib/OpenStackProvider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/OpenStackProvider.py b/MolnsLib/OpenStackProvider.py index 2393065..5a35c98 100644 --- a/MolnsLib/OpenStackProvider.py +++ b/MolnsLib/OpenStackProvider.py @@ -44,7 +44,7 @@ class OpenStackProvider(OpenStackBase): ('neutron_nic', {'q':'Network ID (leave empty if only one possible network)', 'default':None, 'ask':True}), ('region_name', - {'q':'Specify the region to use', 'default':os.environ.get('OS_REGION_NAME'), 'ask':True}), + {'q':'Specify the region (leave empty if only one region)', 'default':os.environ.get('OS_REGION_NAME'), 'ask':True}), ('floating_ip_pool', {'q':'Name of Floating IP Pool (leave empty if only one possible pool)', 'default':None, 'ask':True}), ('nova_version', From 0d44fc329d65315e9828ae8c58f0d931450d8be4 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Fri, 22 Apr 2016 01:58:59 -0700 Subject: [PATCH 042/100] Fixing installation of ode solver after StocSS 1.7 --- MolnsLib/installSoftware.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 92cb2e5..196cd60 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -83,19 +83,21 @@ class InstallSW: "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", "cd /usr/local/StochKit && ./install.sh", - "sudo rm -rf /usr/local/ode-1.0.2;sudo mkdir -p /usr/local/ode-1.0.2/;sudo chown ubuntu /usr/local/ode-1.0.2", - "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.2.tgz?raw=true -q -O /tmp/ode.tgz", - "cd /usr/local/ && tar -xzf /tmp/ode.tgz", + #"sudo rm -rf /usr/local/ode-1.0.3;sudo mkdir -p /usr/local/ode-1.0.3/;sudo chown ubuntu /usr/local/ode-1.0.3", + "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.3.tgz?raw=true -q -O /tmp/ode.tgz", + #"cd /usr/local/ && tar -xzf /tmp/ode.tgz", + "cd /tmp && tar -xzf /tmp/ode.tgz", + "sudo mv /tmp/ode-1.0.3 /usr/local/", "rm /tmp/ode.tgz", - "cd /usr/local/ode-1.0.2/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.2/ make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.3/ make 1>stdout.log 2>stderr.log", "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", - "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.2/ python setup.py install" + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.3/ python setup.py install" ], From c07fefd45b1e32d62c19921dffe0cdcc42e3664e Mon Sep 17 00:00:00 2001 From: aviral26 Date: Fri, 22 Apr 2016 16:36:32 -0700 Subject: [PATCH 043/100] add docker provider class skeleton --- .gitignore | 2 + MolnsLib/DockerProvider.py | 207 ++++++++++++++++++++++++++++++++++++ MolnsLib/EC2Provider.py | 4 +- MolnsLib/installSoftware.py | 16 +-- MolnsLib/molns_datastore.py | 9 +- MolnsLib/molns_provider.py | 3 + molns.py | 53 ++++----- 7 files changed, 254 insertions(+), 40 deletions(-) create mode 100644 MolnsLib/DockerProvider.py diff --git a/.gitignore b/.gitignore index bf0e1ac..24daa93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.pyc .molns/ molns_install.log +.ec2_creds_molns +.idea/ \ No newline at end of file diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py new file mode 100644 index 0000000..e17b548 --- /dev/null +++ b/MolnsLib/DockerProvider.py @@ -0,0 +1,207 @@ +import logging +import time +import os + +from collections import OrderedDict +from molns_provider import ProviderBase, ProviderException + + +class DockerBase(ProviderBase): + """ Abstract class for Docker. """ + + SSH_KEY_EXTENSION = ".pem" + PROVIDER_TYPE = 'Docker' + + +class DockerProvider(DockerBase): + """ Provider handle for local Docker service. """ + + OBJ_NAME = 'DockerProvider' + + CONFIG_VARS = OrderedDict([ + ('key_name', + {'q': 'Docker Key Pair name', 'default': docker_provider_default_key_name(), 'ask': True}), + ('group_name', + {'q': 'Docker Security Group name', 'default': 'molns', 'ask': False}), + ('ubuntu_image_name', + {'q': 'Base Ubuntu image to use (leave blank to use ubuntu-14.04', 'default': 'ubuntu:14.04', + 'ask': True}), + ('molns_image_name', + {'q': 'ID of the MOLNs image (leave empty for none)', 'default': None, 'ask': True}) + ]) + + def check_ssh_key(self): + """ Check that the SSH key is found locally and in the container. + Returns: + True if the key is valid, otherwise False. + """ + ssh_key_dir = os.path.join(self.config_dir, self.name) + logging.debug('ssh_key_dir={0}'.format(ssh_key_dir)) + if not os.path.isdir(ssh_key_dir): + logging.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) + os.makedirs(ssh_key_dir) + ssh_key_file = os.path.join(ssh_key_dir, self.config['key_name'] + self.SSH_KEY_EXTENSION) + if not os.path.isfile(ssh_key_file): + logging.debug("ssh_key_file '{0}' not found".format(ssh_key_file)) + return False + self._connect() + return self.docker.keypair_exists(self.config['key_name']) + + def create_ssh_key(self): + """ Create the ssh key and write the file locally. """ + self._connect() + ssh_key_dir = os.path.join(self.config_dir, self.name) + logging.debug('creating ssh key {0} in dir{1}'.format(self.config['key_name'], ssh_key_dir)) + self.docker.create_keypair(self.config['key_name'], ssh_key_dir) + + def check_security_group(self): + """ Check if the security group is created. """ + self._connect() + return self.docker.security_group_exists(self.config['group_name']) + + def create_seurity_group(self): + """ Create the security group. """ + self._connect() + return self.docker.create_security_group(self.config['group_name']) + + def create_molns_image(self): + """ Create the molns image. """ + self._connect() + + # start container + instances = self.docker.start_container(image=self.config["ubuntu_image_name"]) + instance = instances[0] + + # get login ip + ip = instance.public_dns_name + + # install software + try: + logging.debug("installing software on container (ip={0}). (I am not implemented yet)".format(ip)) + # TODO Where do we store the created image? + except Exception as e: + logging.exception(e) + raise ProviderException("Failed to create molns image: {0}".format(e)) + finally: + logging.debug("stopping container {0}".format(instance)) + self.docker.stop_containers([instance]) + return None + + def check_molns_image(self): + """ Check if the molns image exists. """ + if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config[ + 'molns_image_name'] != '': + self._connect() + return self.docker.image_exists(self.config['molns_image_name']) + return False + + def _connect(self): + if self.connected: + return + self.docker = Docker(config=self) + self.connected = True + + +def docker_provider_default_key_name(): + user = os.environ.get('USER') or 'USER' + return "{0}_molns_docker_sshkey_{1}".format(user, hex(int(time.time())).replace('0x','')) + + +class DockerController(DockerBase): + """ Provider handle for a Docker controller. """ + OBJ_NAME = 'DockerController' + + CONFIG_VARS = ([ + ]) + + +class Docker: + """ + Create containers with the provided configuration. + """ + + def __init__(self, config=None): + # TODO + logging.debug("I am not implemented yet. The config is {0}.".format(config)) + + def create_keypair(self, key_name, conf_dir): + """ + Creates a key pair and writes it locally and on the container. + :return: void + """ + # TODO + logging.debug("I am not implemented yet. The key name is {0} and conf_dir is {1}.".format(key_name, conf_dir)) + + def security_group_exists(self, group_name): + # TODO + logging.debug("I am not implemented yet. The group name is {0}. Returning true.".format(group_name)) + return True + + def keypair_exists(self, key_name): + # TODO + logging.debug("I am not implemented yet. The key name is {0}. Returning true.".format(key_name)) + return True + + def create_security_group(selfself, group_name): + # TODO + logging.debug("I am not implemented yet. The group name is {0}.".format(group_name)) + + def start_container(self, image): + # TODO + logging.debug("I am not implemented yet. The image is {0}.".format(image)) + return None + + def stop_containers(self, containers): + # TODO + logging.debug("I am not implemented yet. The containers are {0}.".format(containers)) + + def image_exists(self, image): + # TODO + logging.debug("I am not implemented yet. The image is {0}. Returning false.".format(image)) + return False + + def get_container_status(self, container): + # TODO + logging.debug("I am not implemented yet. The instance id is {0}. Returning None.".format(container)) + return None + + +class DockerController(DockerBase): + """ Provider handle for a Docker controller. """ + + OBJ_NAME = 'DockerController' + + CONFIG_VARS = OrderedDict([ + ]) + + def _connect(self): + if self.connected: + return + self.docker = Docker(config=self.provider) + self.connected = True + + def get_instance_status(self, instance): + self._connect() + try: + status = self.docker.get_container_status(instance.provider_instance_identifier) + except Exception as e: + # logging.exception(e) + return self.STATUS_TERMINATED + if status == 'running' or status == 'pending': + return self.STATUS_RUNNING + if status == 'stopped' or status == 'stopping': + return self.STATUS_STOPPED + if status == 'terminated' or status == 'shutting-down': + return self.STATUS_TERMINATED + raise ProviderException("DockerController.get_instance_status() got unknown status '{0}'".format(status)) + + +class DockerWorkerGroup(DockerController): + """ Provider handle for an open stack controller. """ + + OBJ_NAME = 'DockerWorkerGroup' + + CONFIG_VARS = OrderedDict([ + ('num_vms', + {'q': 'Number of containers in group', 'default': '1', 'ask': True}), + ]) diff --git a/MolnsLib/EC2Provider.py b/MolnsLib/EC2Provider.py index 3451570..c26887e 100644 --- a/MolnsLib/EC2Provider.py +++ b/MolnsLib/EC2Provider.py @@ -169,7 +169,7 @@ def _get_image_name(self): ########################################## class EC2Controller(EC2Base): - """ Provider handle for an open stack controller. """ + """ Provider handle for an EC2 controller. """ OBJ_NAME = 'EC2Controller' @@ -464,7 +464,7 @@ def start_ec2_instances(self, image_id=None, key_name=None, group_name=None, num raise ProviderException("Could not find image_id={0}".format(image_id)) if img.state != "available": if img.state != "pending": - raise ProviderException("Image {0} is not available, it has state is {1}.".format(image_id, img.state)) + raise ProviderException("Image {0} is not available, it's state is {1}.".format(image_id, img.state)) while img.state == "pending": print "Image {0} has state {1}, waiting {2} seconds for it to become available.".format(image_id, img.state, self.PENDING_IMAGE_WAITTIME) time.sleep(self.PENDING_IMAGE_WAITTIME) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 92cb2e5..fbdfcc0 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -83,19 +83,19 @@ class InstallSW: "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", "cd /usr/local/StochKit && ./install.sh", - "sudo rm -rf /usr/local/ode-1.0.2;sudo mkdir -p /usr/local/ode-1.0.2/;sudo chown ubuntu /usr/local/ode-1.0.2", - "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.2.tgz?raw=true -q -O /tmp/ode.tgz", + "sudo rm -rf /usr/local/ode-1.0.3;sudo mkdir -p /usr/local/ode-1.0.3/;sudo chown ubuntu /usr/local/ode-1.0.3", + "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.3.tgz?raw=true -q -O /tmp/ode.tgz", "cd /usr/local/ && tar -xzf /tmp/ode.tgz", "rm /tmp/ode.tgz", - "cd /usr/local/ode-1.0.2/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.2/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.2/ make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.3/ make 1>stdout.log 2>stderr.log", "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", - "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.2/ python setup.py install" + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.3/ python setup.py install" ], diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 00093c5..e901743 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -9,7 +9,7 @@ import sys ############################################################# #VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] -VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus'] +VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus', 'Docker'] ############################################################# #### SCHEMA ################################################# ############################################################# @@ -101,9 +101,9 @@ class DatastoreException(Exception): ############################################################# HANDLE_MAPPING = { - 'Provider':(Provider,ProviderData), - 'Controller':(Controller,ControllerData), - 'WorkerGroup':(WorkerGroup,WorkerGroupData), + 'Provider': (Provider, ProviderData), + 'Controller': (Controller, ControllerData), + 'WorkerGroup': (WorkerGroup, WorkerGroupData), } #from OpenStackProvider import OpenStackProvider, OpenStackController, OpenStackWorkerGroup @@ -129,6 +129,7 @@ def get_provider_handle(kind, ptype): if pkg_name not in sys.modules: logging.debug("loading {0} from {1}".format(cls_name, pkg_name)) pkg = dynamic_module_import(pkg_name) + pkg = dynamic_module_import(pkg_name) try: #logging.debug("dir(pkg={0})={1}".format(pkg, dir(pkg))) mod = getattr(pkg, cls_name) diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index 587c3eb..a985d28 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -1,8 +1,11 @@ import os import collections + + class ProviderException(Exception): pass + class ProviderBase(): """ Abstract class. """ diff --git a/molns.py b/molns.py index d7f9acd..d9ca02d 100755 --- a/molns.py +++ b/molns.py @@ -22,7 +22,7 @@ class MOLNSException(Exception): class MOLNSConfig(Datastore): def __init__(self, config_dir=None, db_file=None): Datastore.__init__(self,config_dir=config_dir, db_file=db_file) - + def __str__(self): return "MOLNSConfig(config_dir={0})".format(self.config_dir) @@ -144,12 +144,12 @@ def controller_import(cls, args, config, json_data=None): def controller_get_config(cls, name=None, provider_type=None, config=None): """ Return a list of dict of config var for the controller config. Each dict in the list has the keys: 'key', 'value', 'type' - + Either 'name' or 'provider_type' must be specified. If 'name' is specified, then it will retreive the value from that config and return it in 'value' (or return the string '********' if that config is obfuscated, such passwords). - + """ if config is None: raise MOLNSException("no config specified") @@ -214,7 +214,7 @@ def setup_controller(cls, args, config): except DatastoreException as e: # provider providers = config.list_objects(kind='Provider') - if len(providers)==0: + if len(providers) == 0: print "No providers configured, please configure one ('molns provider setup') before initializing controller." return print "Select a provider:" @@ -249,7 +249,7 @@ def list_controller(cls, args, config): provider_name = 'ERROR: {0}'.format(e) table_data.append([c.name, provider_name]) return {'type':'table','column_names':['name', 'provider'], 'data':table_data} - + @classmethod def show_controller(cls, args, config): """ Show all the details of a controller config. """ @@ -345,7 +345,7 @@ def put_controller(cls, args, config): print " ".join(cmd) subprocess.call(cmd) print "SSH process completed" - + @classmethod def status_controller(cls, args, config): @@ -462,7 +462,7 @@ def stop_controller(cls, args, config): if status == worker_obj.STATUS_RUNNING or status == worker_obj.STATUS_STOPPED: print "Terminating worker '{1}' running at {0}".format(i.ip_address, worker_name) worker_obj.terminate_instance(i) - + else: print "No instance running for this controller" @@ -475,7 +475,7 @@ def terminate_controller(cls, args, config): if controller_obj is None: return instance_list = config.get_all_instances(controller_id=controller_obj.id) logging.debug("\tinstance_list={0}".format([str(i) for i in instance_list])) - # Check if they are running or stopped + # Check if they are running or stopped if len(instance_list) > 0: for i in instance_list: if i.worker_group_id is None: @@ -600,12 +600,12 @@ def worker_group_import(cls, args, config, json_data=None): def worker_group_get_config(cls, name=None, provider_type=None, config=None): """ Return a list of dict of config var for the worker group config. Each dict in the list has the keys: 'key', 'value', 'type' - + Either 'name' or 'provider_type' must be specified. If 'name' is specified, then it will retreive the value from that config and return it in 'value' (or return the string '********' if that config is obfuscated, such passwords). - + """ if config is None: raise MOLNSException("no config specified") @@ -652,7 +652,7 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): 'type':'string' }) return ret - + @classmethod def setup_worker_groups(cls, args, config): """ Configure a worker group. """ @@ -746,7 +746,7 @@ def status_worker_groups(cls, args, config): if worker_obj is None: return # Check if any instances are assigned to this worker instance_list = config.get_all_instances(worker_group_id=worker_obj.id) - # Check if they are running or stopped + # Check if they are running or stopped if len(instance_list) > 0: table_data = [] for i in instance_list: @@ -780,7 +780,7 @@ def start_worker_groups(cls, args, config): except ProviderException as e: print "Could not start workers: {0}".format(e) - + @classmethod def add_worker_groups(cls, args, config): """ Add workers of a MOLNs cluster. """ @@ -822,7 +822,7 @@ def __launch_workers__get_controller(cls, worker_obj, config): print "No controller running for this worker group." return return controller_ip - + @classmethod def __launch_worker__start_or_resume_vms(cls, worker_obj, config, num_vms_to_start=0): @@ -993,12 +993,12 @@ def provider_import(cls, args, config, json_data=None): def provider_get_config(cls, name=None, provider_type=None, config=None): """ Return a list of dict of config var for the provider config. Each dict in the list has the keys: 'key', 'value', 'type' - + Either 'name' or 'provider_type' must be specified. If 'name' is specified, then it will retreive the value from that config and return it in 'value' (or return the string '********' if that config is obfuscated, such passwords). - + """ if config is None: raise MOLNSException("no config specified") @@ -1115,7 +1115,7 @@ def provider_initialize(cls, provider_name, config): provider_obj.create_seurity_group() else: print "security group={0} is valid.".format(provider_obj['group_name']) - + # check for MOLNS image if provider_obj['molns_image_name'] is None or provider_obj['molns_image_name'] == '': if provider_obj['ubuntu_image_name'] is None or provider_obj['ubuntu_image_name'] == '': @@ -1124,13 +1124,13 @@ def provider_initialize(cls, provider_name, config): print "Creating new image, this process can take a long time (10-30 minutes)." provider_obj['molns_image_name'] = provider_obj.create_molns_image() elif not provider_obj.check_molns_image(): - print "Error: an molns image was provided, but it is not available in cloud." + print "Error: a molns image ID was provided, but it does not exist." return print "Success." config.save_object(provider_obj, kind='Provider') - - + + @classmethod def provider_rebuild(cls, args, config): """ Rebuild the MOLNS image.""" @@ -1325,6 +1325,7 @@ def raw_input_default_config(q, default=None, obj=None): def setup_object(obj): """ Setup a molns_datastore object using raw_input_default function. """ for key, conf, value in obj.get_config_vars(): + #print "key: {0}\nconf: {1}\nvalue: {2}".format(key, conf, value) obj[key] = raw_input_default_config(conf, default=value, obj=obj) ############################################### @@ -1370,7 +1371,7 @@ def __str__(self): ret += "[{0}={1}] ".format(k,v) ret += "\n\t"+self.description return ret - + def __eq__(self, other): return self.command == other @@ -1463,7 +1464,7 @@ def run(self, args, config_dir=None): Command('clear', {}, function=MOLNSInstances.clear_instances), ]), - + ] def printHelp(): @@ -1478,7 +1479,7 @@ def parseArgs(): if len(sys.argv) < 2 or sys.argv[1] == '-h': printHelp() return - + arg_list = sys.argv[1:] config_dir = './.molns/' while len(arg_list) > 0 and arg_list[0].startswith('--'): @@ -1488,11 +1489,11 @@ def parseArgs(): print "Turning on Debugging output" logger.setLevel(logging.DEBUG) #for Debugging arg_list = arg_list[1:] - + if len(arg_list) == 0 or arg_list[0] =='help' or arg_list[0] == '-h': printHelp() return - + if arg_list[0] in COMMAND_LIST: for cmd in COMMAND_LIST: if cmd == arg_list[0]: @@ -1506,7 +1507,7 @@ def parseArgs(): process_output_exception(e) return - print "unknown command: " + " ".join(arg_list) + print "unknown command: " + " ".join(arg_list) #printHelp() print "use 'molns help' to see all possible commands" From 972b25d1b3db0ea03f19f994d96d42e8215f1f5c Mon Sep 17 00:00:00 2001 From: aviral26 Date: Tue, 26 Apr 2016 19:00:32 -0700 Subject: [PATCH 044/100] add Docker Provider skeleton --- MolnsLib/DockerProvider.py | 119 +++++++++++++++++++++++-------------- MolnsLib/EC2Provider.py | 2 +- MolnsLib/constants.py | 2 + MolnsLib/utils.py | 23 +++++++ NOTES | 1 + 5 files changed, 101 insertions(+), 46 deletions(-) create mode 100644 MolnsLib/constants.py create mode 100644 MolnsLib/utils.py create mode 100644 NOTES diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index e17b548..5ad1012 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -5,6 +5,14 @@ from collections import OrderedDict from molns_provider import ProviderBase, ProviderException +logging.basicConfig() +logger = logging.getLogger("Docker") +logger.setLevel(logging.DEBUG) + +def docker_provider_default_key_name(): + user = os.environ.get('USER') or 'USER' + return "{0}_molns_docker_sshkey_{1}".format(user, hex(int(time.time())).replace('0x','')) + class DockerBase(ProviderBase): """ Abstract class for Docker. """ @@ -24,7 +32,7 @@ class DockerProvider(DockerBase): ('group_name', {'q': 'Docker Security Group name', 'default': 'molns', 'ask': False}), ('ubuntu_image_name', - {'q': 'Base Ubuntu image to use (leave blank to use ubuntu-14.04', 'default': 'ubuntu:14.04', + {'q': 'Base Ubuntu image to use (leave blank to use ubuntu-14.04)', 'default': 'ubuntu:14.04', 'ask': True}), ('molns_image_name', {'q': 'ID of the MOLNs image (leave empty for none)', 'default': None, 'ask': True}) @@ -36,13 +44,13 @@ def check_ssh_key(self): True if the key is valid, otherwise False. """ ssh_key_dir = os.path.join(self.config_dir, self.name) - logging.debug('ssh_key_dir={0}'.format(ssh_key_dir)) + logger.debug('ssh_key_dir={0}'.format(ssh_key_dir)) if not os.path.isdir(ssh_key_dir): - logging.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) + logger.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) os.makedirs(ssh_key_dir) ssh_key_file = os.path.join(ssh_key_dir, self.config['key_name'] + self.SSH_KEY_EXTENSION) if not os.path.isfile(ssh_key_file): - logging.debug("ssh_key_file '{0}' not found".format(ssh_key_file)) + logger.debug("ssh_key_file '{0}' not found".format(ssh_key_file)) return False self._connect() return self.docker.keypair_exists(self.config['key_name']) @@ -51,7 +59,7 @@ def create_ssh_key(self): """ Create the ssh key and write the file locally. """ self._connect() ssh_key_dir = os.path.join(self.config_dir, self.name) - logging.debug('creating ssh key {0} in dir{1}'.format(self.config['key_name'], ssh_key_dir)) + logger.debug('creating ssh key {0} in dir{1}'.format(self.config['key_name'], ssh_key_dir)) self.docker.create_keypair(self.config['key_name'], ssh_key_dir) def check_security_group(self): @@ -77,13 +85,13 @@ def create_molns_image(self): # install software try: - logging.debug("installing software on container (ip={0}). (I am not implemented yet)".format(ip)) + logger.debug("installing software on container (ip={0}). (I am not implemented yet)".format(ip)) # TODO Where do we store the created image? except Exception as e: - logging.exception(e) + logger.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) finally: - logging.debug("stopping container {0}".format(instance)) + logger.debug("stopping container {0}".format(instance)) self.docker.stop_containers([instance]) return None @@ -102,19 +110,6 @@ def _connect(self): self.connected = True -def docker_provider_default_key_name(): - user = os.environ.get('USER') or 'USER' - return "{0}_molns_docker_sshkey_{1}".format(user, hex(int(time.time())).replace('0x','')) - - -class DockerController(DockerBase): - """ Provider handle for a Docker controller. """ - OBJ_NAME = 'DockerController' - - CONFIG_VARS = ([ - ]) - - class Docker: """ Create containers with the provided configuration. @@ -122,7 +117,7 @@ class Docker: def __init__(self, config=None): # TODO - logging.debug("I am not implemented yet. The config is {0}.".format(config)) + logger.debug("I am not implemented yet. The config is {0}.".format(config)) def create_keypair(self, key_name, conf_dir): """ @@ -130,41 +125,56 @@ def create_keypair(self, key_name, conf_dir): :return: void """ # TODO - logging.debug("I am not implemented yet. The key name is {0} and conf_dir is {1}.".format(key_name, conf_dir)) + logger.debug("I am not implemented yet. The key name is {0} and conf_dir is {1}.".format(key_name, conf_dir)) def security_group_exists(self, group_name): # TODO - logging.debug("I am not implemented yet. The group name is {0}. Returning true.".format(group_name)) + logger.debug("I am not implemented yet. The group name is {0}. Returning true.".format(group_name)) return True def keypair_exists(self, key_name): # TODO - logging.debug("I am not implemented yet. The key name is {0}. Returning true.".format(key_name)) + logger.debug("I am not implemented yet. The key name is {0}. Returning true.".format(key_name)) return True def create_security_group(selfself, group_name): # TODO - logging.debug("I am not implemented yet. The group name is {0}.".format(group_name)) + logger.debug("I am not implemented yet. The group name is {0}.".format(group_name)) def start_container(self, image): # TODO - logging.debug("I am not implemented yet. The image is {0}.".format(image)) + logger.debug("I am not implemented yet. The image is {0}.".format(image)) return None def stop_containers(self, containers): # TODO - logging.debug("I am not implemented yet. The containers are {0}.".format(containers)) + logger.debug("I am not implemented yet. The containers are {0}.".format(containers)) def image_exists(self, image): # TODO - logging.debug("I am not implemented yet. The image is {0}. Returning false.".format(image)) + logger.debug("I am not implemented yet. The image is {0}. Returning false.".format(image)) return False def get_container_status(self, container): # TODO - logging.debug("I am not implemented yet. The instance id is {0}. Returning None.".format(container)) + print "hi" + logger.debug("I am not implemented yet. The instance id is {0}. Returning None.".format(container)) return None + def start_docker_containers(self, image_id=None, key_name=None, group_name=None, num=1, instance_type=None): + # TODO Create and start containers here. + + # What should be returned here? Look at EC2Provider.py, line number 485. + logger.debug("I am not implemented yet. Returning None.") + return None + + def resume_docker_containers(self, containers): + num_container = len(containers) + print "Resuming Docker container(s). This will take a minute..." + # TODO resume containers here. + + # What should be returned? + class DockerController(DockerBase): """ Provider handle for a Docker controller. """ @@ -180,24 +190,30 @@ def _connect(self): self.docker = Docker(config=self.provider) self.connected = True - def get_instance_status(self, instance): - self._connect() - try: - status = self.docker.get_container_status(instance.provider_instance_identifier) - except Exception as e: - # logging.exception(e) - return self.STATUS_TERMINATED - if status == 'running' or status == 'pending': - return self.STATUS_RUNNING - if status == 'stopped' or status == 'stopping': - return self.STATUS_STOPPED - if status == 'terminated' or status == 'shutting-down': - return self.STATUS_TERMINATED - raise ProviderException("DockerController.get_instance_status() got unknown status '{0}'".format(status)) + def get_container_status(self, container): + # TODO + logger.debug("I am not implemented yet.") + + def start_instance(self, num=1): + """ Start or resume the controller. """ + # TODO + logger.debug("I am not implemented yet") + + def resume_instance(self, instances): + # TODO + logger.debug("I am not implemented yet") + + def stop_instance(self, instances): + # TODO + logger.debug("I am not implemented yet") + + def terminate_instance(self, instances): + # TODO + logger.debug("I am not implemented yet.") class DockerWorkerGroup(DockerController): - """ Provider handle for an open stack controller. """ + """ Provider handle for Docker worker group. """ OBJ_NAME = 'DockerWorkerGroup' @@ -205,3 +221,16 @@ class DockerWorkerGroup(DockerController): ('num_vms', {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) + + def start_container_group(self, num=1): + """ Start worker group containers. """ + + # TODO start given number of containers. + logger.debug("I am not implemented yet.") + # Look at EC2Provider, line 287. + # How to store container references in the datastore? + # What should be returned? + + def terminate_container_group(selfself, containers): + # TODO remove given containers. + logger.debug("I am not implemented yet.") diff --git a/MolnsLib/EC2Provider.py b/MolnsLib/EC2Provider.py index c26887e..42f706f 100644 --- a/MolnsLib/EC2Provider.py +++ b/MolnsLib/EC2Provider.py @@ -272,7 +272,7 @@ def get_instance_status(self, instance): ########################################## class EC2WorkerGroup(EC2Controller): - """ Provider handle for an open stack controller. """ + """ Provider handle for EC2 worker group. """ OBJ_NAME = 'EC2WorkerGroup' diff --git a/MolnsLib/constants.py b/MolnsLib/constants.py new file mode 100644 index 0000000..e46fb7e --- /dev/null +++ b/MolnsLib/constants.py @@ -0,0 +1,2 @@ +class Constants: + LOGGING_DIRECTORY = "~/MOLNS_LOG"; \ No newline at end of file diff --git a/MolnsLib/utils.py b/MolnsLib/utils.py new file mode 100644 index 0000000..02b685f --- /dev/null +++ b/MolnsLib/utils.py @@ -0,0 +1,23 @@ +import os +import logging + +from MolnsLib.constants import Constants + + +class Logger(object): + + def __init__(self, name): + name = name.replace('.log','') + logger = logging.getLogger('MOLNS.%s' % name) + logger.setLevel(logging.DEBUG) + if not logger.handlers: + file_name = os.path.join(Constants.LOGGING_DIRECTORY, '%s.log' % name) + handler = logging.FileHandler(file_name) + formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s %(message)s') + handler.setFormatter(formatter) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + self._logger = logger + + def get(self): + return self._logger \ No newline at end of file diff --git a/NOTES b/NOTES new file mode 100644 index 0000000..250e2ca --- /dev/null +++ b/NOTES @@ -0,0 +1 @@ +1. DockerProvider - line 167, 176, 222 \ No newline at end of file From 5eaff85575d8ab47ceb3d9e1c9df8df380c9ad1e Mon Sep 17 00:00:00 2001 From: aviral26 Date: Fri, 13 May 2016 12:39:08 -0700 Subject: [PATCH 045/100] add Docker wrapper. work in progress. --- .gitignore | 4 +- MolnsLib/Constants.py | 7 ++ MolnsLib/Docker.py | 64 ++++++++++++++ MolnsLib/DockerProvider.py | 145 ++++++++------------------------ MolnsLib/{utils.py => Utils.py} | 2 +- MolnsLib/constants.py | 2 - MolnsLib/docker_test.py | 136 ++++++++++++++++++++++++++++++ MolnsLib/installSoftware.py | 6 +- MolnsLib/molns_provider.py | 2 +- MolnsLib/test.py | 5 ++ 10 files changed, 259 insertions(+), 114 deletions(-) create mode 100644 MolnsLib/Constants.py create mode 100644 MolnsLib/Docker.py rename MolnsLib/{utils.py => Utils.py} (94%) delete mode 100644 MolnsLib/constants.py create mode 100644 MolnsLib/docker_test.py create mode 100644 MolnsLib/test.py diff --git a/.gitignore b/.gitignore index 24daa93..de50c22 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .molns/ molns_install.log .ec2_creds_molns -.idea/ \ No newline at end of file +.idea/ +*.tar.gz +NOTES diff --git a/MolnsLib/Constants.py b/MolnsLib/Constants.py new file mode 100644 index 0000000..92333be --- /dev/null +++ b/MolnsLib/Constants.py @@ -0,0 +1,7 @@ +class Constants: + LOGGING_DIRECTORY = "~/MOLNS_LOG"; + DOCKER_BASE_URL = "unix://var/run/docker.sock" + DOCKER_DEFAULT_IMAGE = "aviralcse/docker-provider" + DOCKER_DEFAULT_PORT = '9000' + DOCKER_CONTAINER_RUNNING = "running" + DOCKER_CONTAINER_EXITED = "exited" \ No newline at end of file diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py new file mode 100644 index 0000000..0500c80 --- /dev/null +++ b/MolnsLib/Docker.py @@ -0,0 +1,64 @@ +import logging + +from Constants import Constants +from docker import Client +from docker.errors import NotFound, NullResource, APIError + + +class Docker: + + """ A wrapper over docker-py.""" + + LOG_TAG = "Docker" + + def __init__(self): + self.client = Client(base_url=Constants.DOCKER_BASE_URL) + logging.basicConfig(level=logging.DEBUG) + + def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): + """Create a new container.""" + logging.debug(Docker.LOG_TAG + " Using image {0}".format(image_id)) + container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True) + # TODO self.execute_command(container, "su - ubuntu") + return container + + def stop_containers(self, containers): + """Stop given containers.""" + for container in containers: + self.stop_container(container) + + def stop_container(self, container): + self.client.stop(container) + + def is_container_running(self, container): + """Check if container of given name is running or not.""" + return self.client.inspect_container(container.get('Id')).get('State').get('Status') + + def start_containers(self, containers): + """Start each container object in given list.""" + for container in containers: + self.start_container(container) + + def start_container(self, container): + logging.debug(Docker.LOG_TAG + " Starting container " + container.get('Id')) + try: + self.client.start(container.get('Id')) + except (NotFound, NullResource) as e: + logging.error(Docker.LOG_TAG + " Something went wrong while starting container.", e) + return False + return True + + def execute_command(self, container, command): + logging.debug(Docker.LOG_TAG + " Container ID: {0} Command: {1}".format(container.get('Id'), command)) + + if self.start_container(container) is False: + logging.error("Docker", " Could not start container.") + return None + + try: + exec_instance = self.client.exec_create(container.get('Id'), "/bin/bash -c \"" + command + "\"") + response = self.client.exec_start(exec_instance) + return [self.client.exec_inspect(exec_instance), response] + except (NotFound, APIError) as e: + logging.error(Docker.LOG_TAG + " Could not execute command.", e) + return None diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 5ad1012..ae3b29d 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -1,17 +1,21 @@ +import Constants import logging import time import os +import Docker +import installSoftware from collections import OrderedDict from molns_provider import ProviderBase, ProviderException logging.basicConfig() -logger = logging.getLogger("Docker") +logger = logging.getLogger("DockerProvider") logger.setLevel(logging.DEBUG) + def docker_provider_default_key_name(): user = os.environ.get('USER') or 'USER' - return "{0}_molns_docker_sshkey_{1}".format(user, hex(int(time.time())).replace('0x','')) + return "{0}_molns_docker_sshkey_{1}".format(user, hex(int(time.time())).replace('0x', '')) class DockerBase(ProviderBase): @@ -27,73 +31,46 @@ class DockerProvider(DockerBase): OBJ_NAME = 'DockerProvider' CONFIG_VARS = OrderedDict([ - ('key_name', - {'q': 'Docker Key Pair name', 'default': docker_provider_default_key_name(), 'ask': True}), - ('group_name', - {'q': 'Docker Security Group name', 'default': 'molns', 'ask': False}), ('ubuntu_image_name', - {'q': 'Base Ubuntu image to use (leave blank to use ubuntu-14.04)', 'default': 'ubuntu:14.04', + {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'ID of the MOLNs image (leave empty for none)', 'default': None, 'ask': True}) + {'q': 'Local MOLNs image name to use', 'default': None, 'ask': True}) ]) def check_ssh_key(self): - """ Check that the SSH key is found locally and in the container. - Returns: - True if the key is valid, otherwise False. + """ Returns true, because Docker does not use SSH. """ - ssh_key_dir = os.path.join(self.config_dir, self.name) - logger.debug('ssh_key_dir={0}'.format(ssh_key_dir)) - if not os.path.isdir(ssh_key_dir): - logger.debug('making ssh_key_dir={0}'.format(ssh_key_dir)) - os.makedirs(ssh_key_dir) - ssh_key_file = os.path.join(ssh_key_dir, self.config['key_name'] + self.SSH_KEY_EXTENSION) - if not os.path.isfile(ssh_key_file): - logger.debug("ssh_key_file '{0}' not found".format(ssh_key_file)) - return False - self._connect() - return self.docker.keypair_exists(self.config['key_name']) + return True def create_ssh_key(self): - """ Create the ssh key and write the file locally. """ - self._connect() - ssh_key_dir = os.path.join(self.config_dir, self.name) - logger.debug('creating ssh key {0} in dir{1}'.format(self.config['key_name'], ssh_key_dir)) - self.docker.create_keypair(self.config['key_name'], ssh_key_dir) + """ Does nothing. """ + return True def check_security_group(self): - """ Check if the security group is created. """ - self._connect() - return self.docker.security_group_exists(self.config['group_name']) + """ Does nothing.""" + return True def create_seurity_group(self): - """ Create the security group. """ - self._connect() - return self.docker.create_security_group(self.config['group_name']) + """ Does nothing. """ def create_molns_image(self): - """ Create the molns image. """ + """ Create the molns image and save it locally. """ self._connect() - # start container - instances = self.docker.start_container(image=self.config["ubuntu_image_name"]) - instance = instances[0] - - # get login ip - ip = instance.public_dns_name + # create container + container = self.docker.create_container() # install software try: - logger.debug("installing software on container (ip={0}). (I am not implemented yet)".format(ip)) - # TODO Where do we store the created image? + logger.debug("Installing software on container ID: {0}".format(container.get('Id'))) + self._run_commands(container, installSoftware.InstallSW.get_command_list()) except Exception as e: logger.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) finally: - logger.debug("stopping container {0}".format(instance)) - self.docker.stop_containers([instance]) - return None + logger.debug("Stopping container {0}".format(container)) + self.docker.stop_containers([container]) def check_molns_image(self): """ Check if the molns image exists. """ @@ -106,74 +83,26 @@ def check_molns_image(self): def _connect(self): if self.connected: return - self.docker = Docker(config=self) + self.docker = Docker.Docker() self.connected = True + def _run_commands(self, container, commands): + """Run given commands in the given container. Fails if even a single command returns non-zero error code. """ -class Docker: - """ - Create containers with the provided configuration. - """ + # Start container and exec given commands in it. - def __init__(self, config=None): - # TODO - logger.debug("I am not implemented yet. The config is {0}.".format(config)) - - def create_keypair(self, key_name, conf_dir): - """ - Creates a key pair and writes it locally and on the container. - :return: void - """ - # TODO - logger.debug("I am not implemented yet. The key name is {0} and conf_dir is {1}.".format(key_name, conf_dir)) - - def security_group_exists(self, group_name): - # TODO - logger.debug("I am not implemented yet. The group name is {0}. Returning true.".format(group_name)) - return True - - def keypair_exists(self, key_name): - # TODO - logger.debug("I am not implemented yet. The key name is {0}. Returning true.".format(key_name)) - return True - - def create_security_group(selfself, group_name): - # TODO - logger.debug("I am not implemented yet. The group name is {0}.".format(group_name)) - - def start_container(self, image): - # TODO - logger.debug("I am not implemented yet. The image is {0}.".format(image)) - return None - - def stop_containers(self, containers): - # TODO - logger.debug("I am not implemented yet. The containers are {0}.".format(containers)) - - def image_exists(self, image): - # TODO - logger.debug("I am not implemented yet. The image is {0}. Returning false.".format(image)) - return False - - def get_container_status(self, container): - # TODO - print "hi" - logger.debug("I am not implemented yet. The instance id is {0}. Returning None.".format(container)) - return None - - def start_docker_containers(self, image_id=None, key_name=None, group_name=None, num=1, instance_type=None): - # TODO Create and start containers here. - - # What should be returned here? Look at EC2Provider.py, line number 485. - logger.debug("I am not implemented yet. Returning None.") - return None - - def resume_docker_containers(self, containers): - num_container = len(containers) - print "Resuming Docker container(s). This will take a minute..." - # TODO resume containers here. + self._connect() - # What should be returned? + for entry in commands: + if isinstance(entry, list): + for sub_entry in entry: + ret_val, response = self.docker.execute_command(container, sub_entry) + if ret_val is None or ret_val.get('ExitCode') != 0: + raise installSoftware.InstallSWException() + else: + ret_val, response = self.docker.execute_command(container, entry) + if ret_val is None or ret_val.get('ExitCode') != 0: + raise installSoftware.InstallSWException() class DockerController(DockerBase): diff --git a/MolnsLib/utils.py b/MolnsLib/Utils.py similarity index 94% rename from MolnsLib/utils.py rename to MolnsLib/Utils.py index 02b685f..956751f 100644 --- a/MolnsLib/utils.py +++ b/MolnsLib/Utils.py @@ -1,7 +1,7 @@ import os import logging -from MolnsLib.constants import Constants +from MolnsLib.Constants import Constants class Logger(object): diff --git a/MolnsLib/constants.py b/MolnsLib/constants.py deleted file mode 100644 index e46fb7e..0000000 --- a/MolnsLib/constants.py +++ /dev/null @@ -1,2 +0,0 @@ -class Constants: - LOGGING_DIRECTORY = "~/MOLNS_LOG"; \ No newline at end of file diff --git a/MolnsLib/docker_test.py b/MolnsLib/docker_test.py new file mode 100644 index 0000000..3386eee --- /dev/null +++ b/MolnsLib/docker_test.py @@ -0,0 +1,136 @@ +from Docker import Docker + +command_list = [ + + # Basic contextualization + "sudo apt-get update", + "sudo apt-get -y install git", + "sudo apt-get -y install build-essential python-dev", + "sudo apt-get -y install python-setuptools", + "sudo apt-get -y install python-matplotlib python-numpy python-scipy", + "sudo apt-get -y install make", + "sudo apt-get -y install python-software-properties", + "sudo apt-get -y install cython python-h5py", + "sudo apt-get -y install python-pip python-dev build-essential", + "sudo pip install pyzmq --upgrade", + "sudo pip install dill cloud pygments", + "sudo pip install tornado Jinja2", + + # Molnsutil + [ + "sudo pip install jsonschema jsonpointer", + # EC2/S3 and OpenStack APIs + "sudo pip install boto", + "sudo apt-get -y install pandoc", + # This set of packages is needed for OpenStack, as molnsutil uses them for hybrid cloud deployment + "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", + "sudo pip install python-novaclient", + "sudo easy_install -U pip", + "sudo pip install python-keystoneclient", + "sudo pip install python-swiftclient", + ], + + [ + "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", + "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", + "cd /usr/local/molnsutil && sudo python setup.py install" + ], + + # So the workers can mount the controller via SSHfs + ["sudo apt-get -y install sshfs", + "sudo gpasswd -a ubuntu fuse", + "echo \'ServerAliveInterval 60\' >> /home/ubuntu/.ssh/config", + ], + + # IPython + ["sudo rm -rf ipython;git clone --recursive https://github.com/Molns/ipython.git", + "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", + "sudo rm -rf ipython", + "ipython profile create default", + "sudo pip install terminado", # Jupyter terminals + "python -c 'from IPython.external import mathjax; mathjax.install_mathjax(tag=\\\"2.2.0\\\")'" + ], + + ### Simulation software related to pyurdme and StochSS + + # Gillespy + ["sudo rm -rf /usr/local/StochKit;sudo mkdir -p /usr/local/StochKit;sudo chown ubuntu /usr/local/StochKit", + "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", + "cd /usr/local/StochKit && ./install.sh", + + "sudo rm -rf /usr/local/ode-1.0.3;sudo mkdir -p /usr/local/ode-1.0.3/;sudo chown ubuntu /usr/local/ode-1.0.3", + "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.3.tgz?raw=true -q -O /tmp/ode.tgz", + "cd /usr/local/ && tar -xzf /tmp/ode.tgz", + "rm /tmp/ode.tgz", + "cd /usr/local/ode-1.0.3/cvodes/ && tar -xzf 'cvodes-2.7.0.tar.gz'", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && ./configure --prefix='/usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/cvodes' 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", + "cd /usr/local/ode-1.0.3/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.3/ make 1>stdout.log 2>stderr.log", + + "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", + "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.3/ python setup.py install" + + ], + + # FeniCS/Dolfin/pyurdme + ["sudo add-apt-repository -y ppa:fenics-packages/fenics", + "sudo apt-get update", + "sudo apt-get -y install fenics", + # Gmsh for Finite Element meshes + "sudo apt-get install -y gmsh", + ], + + # pyurdme + ["sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme", + "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", + # "cd /usr/local/pyurdme && git checkout develop", # for development only + "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* .ipython/profile_default/static/custom/", + "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", + ], + + # example notebooks + ["rm -rf MOLNS_notebooks;git clone https://github.com/Molns/MOLNS_notebooks.git", + "cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks;", + "ls *.ipynb" + ], + + # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty + "sudo apt-get -y remove python-scipy", + "sudo pip install scipy", + + "sudo pip install jsonschema jsonpointer", # redo this install to be sure it has not been removed. + + + "sync", # This is critical for some infrastructures. +] + +docker = Docker() + +container = docker.create_container() + +docker.start_container(container) + +print "is container running: {0}".format(docker.is_container_running(container)) + + +for entry in command_list: + if isinstance(entry, list): + for sub_entry in entry: + ret_val, response = docker.execute_command(container, sub_entry) + print "RETURN VALUE: {0}".format(ret_val) + if ret_val is None or ret_val.get('ExitCode') != 0: + print "ERROR" + print "RESPONSE: {0}".format(response) + print "__" + print "__" + + else: + ret_val, response = docker.execute_command(container, entry) + print "RETURN VALUE: {0}".format(ret_val) + if ret_val is None or ret_val.get('ExitCode') != 0: + print "ERROR" + print "RESPONSE: {0}".format(response) + print "__" + print "__" diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index fbdfcc0..6990378 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -268,7 +268,6 @@ def exec_command_list_switch(self, command_list): raise SystemExit("CRITICAL ERROR: could not complete command '{0}'. Exiting.".format(command)) print "Installation complete in {0}s".format(time.time() - tic) - def log_exec(self, msg): if self.log_file is not None: self.log_file.write(msg) @@ -332,6 +331,11 @@ def exec_multi_command(self, command, next_command): print "FAILED......\t{0}:{1}\t{2}\t{3}".format(self.hostname, self.ssh_endpoint, command, e) raise InstallSWException() + @staticmethod + def get_command_list(): + """Returns the whole list of dependency installation commands. """ + return InstallSW.command_list + if __name__ == "__main__": print "{0}".format(InstallSW.command_list) print "len={0}".format(len(InstallSW.command_list)) diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index a985d28..f02285b 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -25,7 +25,7 @@ class ProviderBase(): SecurityGroupRule("tcp", "9000", "65535", "0.0.0.0/0", None), ] - def __init__(self, name, config=None, config_dir=None,**kwargs): + def __init__(self, name, config=None, config_dir=None, **kwargs): self.config = {} self.name = name self.type = self.PROVIDER_TYPE diff --git a/MolnsLib/test.py b/MolnsLib/test.py new file mode 100644 index 0000000..2bf4c00 --- /dev/null +++ b/MolnsLib/test.py @@ -0,0 +1,5 @@ +import pdb + +for i in range(10): + pdb.set_trace() + print "hi" From 444e5e3292b553f08c0a46bd86f14c9e279310f2 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Thu, 19 May 2016 15:56:29 -0700 Subject: [PATCH 046/100] successfully build container --- .gitignore | 2 + MolnsLib/Docker.py | 12 +++ MolnsLib/DockerProvider.py | 67 ++++++++---- MolnsLib/docker_test.py | 209 ++++++++++++++---------------------- MolnsLib/installSoftware.py | 9 +- NOTES | 55 +++++++++- docker_file/Dockerfile | 17 +++ 7 files changed, 217 insertions(+), 154 deletions(-) create mode 100644 docker_file/Dockerfile diff --git a/.gitignore b/.gitignore index de50c22..f7cc555 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ molns_install.log .idea/ *.tar.gz NOTES +qsubscript +\#qsubscript\# \ No newline at end of file diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 0500c80..a6a2adc 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -11,8 +11,11 @@ class Docker: LOG_TAG = "Docker" + shell_commands = ["source"] + def __init__(self): self.client = Client(base_url=Constants.DOCKER_BASE_URL) + self.build_count = 0 logging.basicConfig(level=logging.DEBUG) def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): @@ -62,3 +65,12 @@ def execute_command(self, container, command): except (NotFound, APIError) as e: logging.error(Docker.LOG_TAG + " Could not execute command.", e) return None + + def build_image(self, Dockerfile): + """ Build image from given Dockerfile object and return build output. """ + self.build_count += 1 + logging.debug("Dockerfile name: " + Dockerfile.name) + image_tag = "aviralcse/docker-provider-{0}".format(self.build_count) + for line in self.client.build(fileobj=Dockerfile, rm=True, tag=image_tag): + print(line) + return image_tag diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index ae3b29d..c1c364a 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -4,7 +4,9 @@ import os import Docker import installSoftware +import tempfile +from io import BytesIO from collections import OrderedDict from molns_provider import ProviderBase, ProviderException @@ -58,19 +60,24 @@ def create_molns_image(self): """ Create the molns image and save it locally. """ self._connect() - # create container - container = self.docker.create_container() - - # install software + # create Dockerfile and build container. try: - logger.debug("Installing software on container ID: {0}".format(container.get('Id'))) - self._run_commands(container, installSoftware.InstallSW.get_command_list()) + logging.debug("Creating Dockerfile...") + dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) + logging.debug("---------------Dockerfile----------------") + logging.debug(dockerfile) + logging.debug("-----------------------------------------") + logging.debug("Building image...") + tmpf = tempfile.NamedTemporaryFile() + tmpf.write(dockerfile) + logging.debug("tmph name: " + tmpf.name) + tmpf.seek(0) + image_tag = self.docker.build_image(tmpf) + logging.debug("Image created.") + return image_tag except Exception as e: logger.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) - finally: - logger.debug("Stopping container {0}".format(container)) - self.docker.stop_containers([container]) def check_molns_image(self): """ Check if the molns image exists. """ @@ -86,23 +93,43 @@ def _connect(self): self.docker = Docker.Docker() self.connected = True - def _run_commands(self, container, commands): - """Run given commands in the given container. Fails if even a single command returns non-zero error code. """ - - # Start container and exec given commands in it. + def _create_dockerfile(self, commands): + """ Create Dockerfile from given commands. """ + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n git \ \n ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' - self._connect() + flag = False for entry in commands: if isinstance(entry, list): + dockerfile += '''\n\nRUN ''' + first = True + flag = False for sub_entry in entry: - ret_val, response = self.docker.execute_command(container, sub_entry) - if ret_val is None or ret_val.get('ExitCode') != 0: - raise installSoftware.InstallSWException() + if first is True: + dockerfile += self._preprocess(sub_entry) + first = False + else: + dockerfile += ''' && \ \n ''' + self._preprocess(sub_entry) else: - ret_val, response = self.docker.execute_command(container, entry) - if ret_val is None or ret_val.get('ExitCode') != 0: - raise installSoftware.InstallSWException() + if flag is False: + dockerfile += '''\n\nRUN ''' + flag = True + dockerfile += self._preprocess(entry) + else: + dockerfile += ''' && \ \n ''' + self._preprocess(entry) + + dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu''' + + return dockerfile + + def _preprocess(self, command): + """ Filters out any sudos in the command, prepends shell only commands with '/bin/bash -c'. """ + for shell_command in Docker.Docker.shell_commands: + if shell_command in command: + replace_string = "/bin/bash -c \"" + shell_command + command = command.replace(shell_command, replace_string) + command += "\"" + return command.replace("sudo", "") class DockerController(DockerBase): diff --git a/MolnsLib/docker_test.py b/MolnsLib/docker_test.py index 3386eee..e14f55a 100644 --- a/MolnsLib/docker_test.py +++ b/MolnsLib/docker_test.py @@ -1,136 +1,87 @@ from Docker import Docker +import tempfile +import installSoftware +from DockerProvider import DockerProvider -command_list = [ - - # Basic contextualization - "sudo apt-get update", - "sudo apt-get -y install git", - "sudo apt-get -y install build-essential python-dev", - "sudo apt-get -y install python-setuptools", - "sudo apt-get -y install python-matplotlib python-numpy python-scipy", - "sudo apt-get -y install make", - "sudo apt-get -y install python-software-properties", - "sudo apt-get -y install cython python-h5py", - "sudo apt-get -y install python-pip python-dev build-essential", - "sudo pip install pyzmq --upgrade", - "sudo pip install dill cloud pygments", - "sudo pip install tornado Jinja2", - - # Molnsutil - [ - "sudo pip install jsonschema jsonpointer", - # EC2/S3 and OpenStack APIs - "sudo pip install boto", - "sudo apt-get -y install pandoc", - # This set of packages is needed for OpenStack, as molnsutil uses them for hybrid cloud deployment - "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", - "sudo pip install python-novaclient", - "sudo easy_install -U pip", - "sudo pip install python-keystoneclient", - "sudo pip install python-swiftclient", - ], - - [ - "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", - "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", - "cd /usr/local/molnsutil && sudo python setup.py install" - ], - - # So the workers can mount the controller via SSHfs - ["sudo apt-get -y install sshfs", - "sudo gpasswd -a ubuntu fuse", - "echo \'ServerAliveInterval 60\' >> /home/ubuntu/.ssh/config", - ], - - # IPython - ["sudo rm -rf ipython;git clone --recursive https://github.com/Molns/ipython.git", - "cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install", - "sudo rm -rf ipython", - "ipython profile create default", - "sudo pip install terminado", # Jupyter terminals - "python -c 'from IPython.external import mathjax; mathjax.install_mathjax(tag=\\\"2.2.0\\\")'" - ], - - ### Simulation software related to pyurdme and StochSS - - # Gillespy - ["sudo rm -rf /usr/local/StochKit;sudo mkdir -p /usr/local/StochKit;sudo chown ubuntu /usr/local/StochKit", - "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", - "cd /usr/local/StochKit && ./install.sh", - - "sudo rm -rf /usr/local/ode-1.0.3;sudo mkdir -p /usr/local/ode-1.0.3/;sudo chown ubuntu /usr/local/ode-1.0.3", - "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.3.tgz?raw=true -q -O /tmp/ode.tgz", - "cd /usr/local/ && tar -xzf /tmp/ode.tgz", - "rm /tmp/ode.tgz", - "cd /usr/local/ode-1.0.3/cvodes/ && tar -xzf 'cvodes-2.7.0.tar.gz'", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && ./configure --prefix='/usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/cvodes' 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.3/ make 1>stdout.log 2>stderr.log", - - "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", - "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", - "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.3/ python setup.py install" - - ], - - # FeniCS/Dolfin/pyurdme - ["sudo add-apt-repository -y ppa:fenics-packages/fenics", - "sudo apt-get update", - "sudo apt-get -y install fenics", - # Gmsh for Finite Element meshes - "sudo apt-get install -y gmsh", - ], - - # pyurdme - ["sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme", - "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", - # "cd /usr/local/pyurdme && git checkout develop", # for development only - "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* .ipython/profile_default/static/custom/", - "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", - ], - - # example notebooks - ["rm -rf MOLNS_notebooks;git clone https://github.com/Molns/MOLNS_notebooks.git", - "cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks;", - "ls *.ipynb" - ], - - # Upgrade scipy from pip to get rid of super-annoying six.py bug on Trusty - "sudo apt-get -y remove python-scipy", - "sudo pip install scipy", - - "sudo pip install jsonschema jsonpointer", # redo this install to be sure it has not been removed. - - - "sync", # This is critical for some infrastructures. -] +commands = installSoftware.InstallSW.command_list docker = Docker() - -container = docker.create_container() - -docker.start_container(container) - -print "is container running: {0}".format(docker.is_container_running(container)) - - -for entry in command_list: +# +# container = docker.create_container() +# +# docker.start_container(container) +# +# print "is container running: {0}".format(docker.is_container_running(container)) +# +# +# for entry in command_list: +# if isinstance(entry, list): +# for sub_entry in entry: +# ret_val, response = docker.execute_command(container, sub_entry) +# print "RETURN VALUE: {0}".format(ret_val) +# if ret_val is None or ret_val.get('ExitCode') != 0: +# print "ERROR" +# print "RESPONSE: {0}".format(response) +# print "__" +# print "__" +# +# else: +# ret_val, response = docker.execute_command(container, entry) +# print "RETURN VALUE: {0}".format(ret_val) +# if ret_val is None or ret_val.get('ExitCode') != 0: +# print "ERROR" +# print "RESPONSE: {0}".format(response) +# print "__" +# print "__" +def create_dockerfile(commands): + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu''' + + flag = False + + for entry in commands: if isinstance(entry, list): + dockerfile += '''\n\nRUN ''' + first = True + flag = False for sub_entry in entry: - ret_val, response = docker.execute_command(container, sub_entry) - print "RETURN VALUE: {0}".format(ret_val) - if ret_val is None or ret_val.get('ExitCode') != 0: - print "ERROR" - print "RESPONSE: {0}".format(response) - print "__" - print "__" - + if first is True: + dockerfile += _preprocess(sub_entry) + first = False + else: + dockerfile += ''' && \ \n ''' + _preprocess(sub_entry) else: - ret_val, response = docker.execute_command(container, entry) - print "RETURN VALUE: {0}".format(ret_val) - if ret_val is None or ret_val.get('ExitCode') != 0: - print "ERROR" - print "RESPONSE: {0}".format(response) - print "__" - print "__" + if flag is False: + dockerfile += '''\n\nRUN ''' + flag = True + dockerfile += _preprocess(entry) + else: + dockerfile += ''' && \ \n ''' + _preprocess(entry) + + dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu''' + + return dockerfile + + +def _preprocess(command): + """ Filters out any sudos in the command, prepends shell only commands with '/bin/bash -c'. """ + for shell_command in Docker.shell_commands: + if shell_command in command: + replace_string = "/bin/bash -c \"" + shell_command + command = command.replace(shell_command, replace_string) + command += "\"" + return command.replace("sudo", "") + +print("Creating Dockerfile...") +dockerfile = create_dockerfile(commands) +print("---------------Dockerfile----------------") +print(dockerfile) +print("-----------------------------------------") +print("Building image...") +tmpfh = tempfile.NamedTemporaryFile() +tmpfh.write(dockerfile) +print("tmph name: " + tmpfh.name) +tmpfh.seek(0) +image_tag = docker.build_image(tmpfh) + +print("Image created.") + diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 6990378..c6f619e 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -63,6 +63,7 @@ class InstallSW: # So the workers can mount the controller via SSHfs [ "sudo apt-get -y install sshfs", "sudo gpasswd -a ubuntu fuse", + "mkdir -p /home/ubuntu/.ssh/", "echo 'ServerAliveInterval 60' >> /home/ubuntu/.ssh/config", ], @@ -108,16 +109,16 @@ class InstallSW: ], # pyurdme - [ "sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme", + [ "sudo rm -rf /usr/local/pyurdme && sudo mkdir -p /usr/local/pyurdme && sudo chown ubuntu /usr/local/pyurdme", "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", #"cd /usr/local/pyurdme && git checkout develop", # for development only - "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* .ipython/profile_default/static/custom/", + "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* $HOME/.ipython/profile_default/static/custom/", # TODO added $HOME here. Is it okay? "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", ], # example notebooks - [ "rm -rf MOLNS_notebooks;git clone https://github.com/Molns/MOLNS_notebooks.git", - "cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks;", + [ "rm -rf MOLNS_notebooks && git clone https://github.com/Molns/MOLNS_notebooks.git", + "cp MOLNS_notebooks/*.ipynb . && rm -rf MOLNS_notebooks", "ls *.ipynb" ], diff --git a/NOTES b/NOTES index 250e2ca..04516c1 100644 --- a/NOTES +++ b/NOTES @@ -1 +1,54 @@ -1. DockerProvider - line 167, 176, 222 \ No newline at end of file +1. DockerProvider - line 167, 176, 222 + + +pbs qsub - scheduler +comet, htcondor + +knot csc ucsb - stochss backend + +molns azure + +openstack installation on campus cluster + +gillespy + + + +1. sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme + rm: invalid option -- 'p' +------no user ubuntu in container + +2. cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks; + cp: target 'MOLNS_notebooks;' is not a directory + + + 3. sudo add-apt-repository -y ppa:fenics-packages/fenics + sudo: add-apt-repository: command not found + + +4. cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install + rpc error: code = 2 desc = "oci runtime error: exec failed: exec: \"cd\": executable file not found in $PATH" +--------cd is a built in shell command + +5. Command: sudo gpasswd -a ubuntu fuse + gpasswd: user 'ubuntu' does not exist +----------- + + 6. ipython executable not found in path + + 7. wget not found in path + + + +DEBUG:root:Docker Container ID: 1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e Command: python -c "from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')" +DEBUG:requests.packages.urllib3.connectionpool:"POST /v1.22/containers/1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e/exec HTTP/1.1" 201 74 +DEBUG:requests.packages.urllib3.connectionpool:"POST /v1.22/exec/e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978/start HTTP/1.1" 200 None +DEBUG:requests.packages.urllib3.connectionpool:"GET /v1.22/exec/e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978/json HTTP/1.1" 200 458 +RETURN VALUE: {u'OpenStderr': True, u'OpenStdout': True, u'ContainerID': u'1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e', u'DetachKeys': u'', u'CanRemove': False, u'Running': False, u'ProcessConfig': {u'tty': False, u'entrypoint': u'/bin/bash', u'arguments': [u'-c', u'python -c from', u'IPython.external', u'import', u'mathjax;', u'mathjax.install_mathjax(tag=2.2.0)'], u'privileged': False}, u'ExitCode': 1, u'ID': u'e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978', u'OpenStdin': False} +ERROR +RESPONSE: File "", line 1 + from + ^ +SyntaxError: invalid syntax + + diff --git a/docker_file/Dockerfile b/docker_file/Dockerfile new file mode 100644 index 0000000..27a284c --- /dev/null +++ b/docker_file/Dockerfile @@ -0,0 +1,17 @@ +FROM ubuntu:14.04 + +RUN apt-get update + +# Set up base environment. +RUN apt-get install -yy \ + software-properties-common \ + python-software-properties \ + wget \ + git \ + ipython + +# Create and switch to new user. +RUN useradd -ms /bin/bash ubuntu +USER ubuntu +ENV HOME /home/ubuntu +WORKDIR /home/ubuntu \ No newline at end of file From 1b737f55c19b8d473bf70419114801fe0b1f3b7e Mon Sep 17 00:00:00 2001 From: aviral26 Date: Fri, 20 May 2016 13:02:34 -0700 Subject: [PATCH 047/100] bug fixes, complete Docker provider --- MolnsLib/Constants.py | 6 ++- MolnsLib/Docker.py | 86 +++++++++++++++++++++++++++----------- MolnsLib/DockerProvider.py | 60 ++++++++++++++------------ molns.py | 15 ++++--- 4 files changed, 106 insertions(+), 61 deletions(-) diff --git a/MolnsLib/Constants.py b/MolnsLib/Constants.py index 92333be..90bf290 100644 --- a/MolnsLib/Constants.py +++ b/MolnsLib/Constants.py @@ -4,4 +4,8 @@ class Constants: DOCKER_DEFAULT_IMAGE = "aviralcse/docker-provider" DOCKER_DEFAULT_PORT = '9000' DOCKER_CONTAINER_RUNNING = "running" - DOCKER_CONTAINER_EXITED = "exited" \ No newline at end of file + DOCKER_CONTAINER_EXITED = "exited" + DOCKERFILE_NAME = "dockerfile_" + DOKCER_IMAGE_ID_LENGTH = 12 + DOCKER_IMAGE_PREFIX = "aviralcse/docker-provider-" + DOCKER_PY_IMAGE_ID_PREFIX_LENGTH = 7 \ No newline at end of file diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index a6a2adc..410df11 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -13,6 +13,10 @@ class Docker: shell_commands = ["source"] + class ImageBuildException(Exception): + def __init__(self): + super("Something went wrong while building docker container image.") + def __init__(self): self.client = Client(base_url=Constants.DOCKER_BASE_URL) self.build_count = 0 @@ -20,39 +24,48 @@ def __init__(self): def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): """Create a new container.""" + logging.debug(Docker.LOG_TAG + " Using image {0}".format(image_id)) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True) - # TODO self.execute_command(container, "su - ubuntu") return container - def stop_containers(self, containers): + def stop_containers(self, container_ids): """Stop given containers.""" - for container in containers: - self.stop_container(container) - def stop_container(self, container): - self.client.stop(container) + for container_id in container_ids: + self.stop_container(container_id) + + def stop_container(self, container_id): + """Stop the container with given ID.""" + + self.client.stop(container_id) + + def container_status(self, container_id): + """Is the container with given ID running?""" + + return self.client.inspect_container(container_id).get('State').get('Status') + + def start_containers(self, container_ids): + """Start each container in given list of container IDs.""" - def is_container_running(self, container): - """Check if container of given name is running or not.""" - return self.client.inspect_container(container.get('Id')).get('State').get('Status') + for container_id in container_ids: + self.start_container(container_id) - def start_containers(self, containers): - """Start each container object in given list.""" - for container in containers: - self.start_container(container) + def start_container(self, container_id): + """ Start the container with given ID.""" - def start_container(self, container): - logging.debug(Docker.LOG_TAG + " Starting container " + container.get('Id')) + logging.debug(Docker.LOG_TAG + " Starting container " + container_id) try: - self.client.start(container.get('Id')) + self.client.start(container=container_id) except (NotFound, NullResource) as e: logging.error(Docker.LOG_TAG + " Something went wrong while starting container.", e) return False return True def execute_command(self, container, command): - logging.debug(Docker.LOG_TAG + " Container ID: {0} Command: {1}".format(container.get('Id'), command)) + """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" + + logging.debug(Docker.LOG_TAG + " CONTAINER: {0} COMMAND: {1}".format(container.get('Id'), command)) if self.start_container(container) is False: logging.error("Docker", " Could not start container.") @@ -66,11 +79,34 @@ def execute_command(self, container, command): logging.error(Docker.LOG_TAG + " Could not execute command.", e) return None - def build_image(self, Dockerfile): - """ Build image from given Dockerfile object and return build output. """ - self.build_count += 1 - logging.debug("Dockerfile name: " + Dockerfile.name) - image_tag = "aviralcse/docker-provider-{0}".format(self.build_count) - for line in self.client.build(fileobj=Dockerfile, rm=True, tag=image_tag): - print(line) - return image_tag + def build_image(self, dockerfile): + """ Build image from given Dockerfile object and return ID of the image created. """ + + print("Building image...") + image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(self.build_count) + last_line = "" + try: + for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag): + print(line) + if "errorDetail" in line: raise Docker.ImageBuildException() + last_line = line + + # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image + # id. + tokens = last_line.split(" ") + image_id = tokens[3][:Constants.DOKCER_IMAGE_ID_LENGTH] + print("Image ID: {0}".format(image_id)) + return image_id + except (Docker.ImageBuildException, IndexError) as e: + print(e) + return None + + def image_exists(self, image_id): + """Check if image with given ID exists locally.""" + + for image in self.client.images(): + some_id = image["Id"] + if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: + logging.debug("Image exists: " + image) + return True + return False diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index c1c364a..d8cae34 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -6,14 +6,9 @@ import installSoftware import tempfile -from io import BytesIO from collections import OrderedDict from molns_provider import ProviderBase, ProviderException -logging.basicConfig() -logger = logging.getLogger("DockerProvider") -logger.setLevel(logging.DEBUG) - def docker_provider_default_key_name(): user = os.environ.get('USER') or 'USER' @@ -28,7 +23,7 @@ class DockerBase(ProviderBase): class DockerProvider(DockerBase): - """ Provider handle for local Docker service. """ + """ Provider handle for local Docker based service. """ OBJ_NAME = 'DockerProvider' @@ -40,9 +35,22 @@ class DockerProvider(DockerBase): {'q': 'Local MOLNs image name to use', 'default': None, 'ask': True}) ]) + counter = 0 + + @staticmethod + def __get_new_dockerfile_name(): + DockerProvider.counter += 1 + filename = Constants.Constants.DOCKERFILE_NAME + str(DockerProvider.counter) + return filename + + def _connect(self): + if self.connected: + return + self.docker = Docker.Docker() + self.connected = True + def check_ssh_key(self): - """ Returns true, because Docker does not use SSH. - """ + """ Returns true. (Docker does not use SSH.)""" return True def create_ssh_key(self): @@ -55,46 +63,35 @@ def check_security_group(self): def create_seurity_group(self): """ Does nothing. """ + return True def create_molns_image(self): - """ Create the molns image and save it locally. """ - self._connect() + """ Create the molns image, save it on localhost and return ID of created image. """ + self._connect() # create Dockerfile and build container. try: logging.debug("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) - logging.debug("---------------Dockerfile----------------") - logging.debug(dockerfile) - logging.debug("-----------------------------------------") - logging.debug("Building image...") - tmpf = tempfile.NamedTemporaryFile() - tmpf.write(dockerfile) - logging.debug("tmph name: " + tmpf.name) - tmpf.seek(0) - image_tag = self.docker.build_image(tmpf) + image_tag = self.docker.build_image(dockerfile) logging.debug("Image created.") return image_tag except Exception as e: - logger.exception(e) + logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) def check_molns_image(self): """ Check if the molns image exists. """ + if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config[ 'molns_image_name'] != '': self._connect() return self.docker.image_exists(self.config['molns_image_name']) return False - def _connect(self): - if self.connected: - return - self.docker = Docker.Docker() - self.connected = True - def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n git \ \n ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' flag = False @@ -120,10 +117,19 @@ def _create_dockerfile(self, commands): dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu''' - return dockerfile + dockerfile_file = DockerProvider.__get_new_dockerfile_name() + with open(dockerfile_file, 'w') as Dockerfile: + Dockerfile.write(dockerfile) + print("Using as dockerfile : " + dockerfile_file) + named_dockerfile = tempfile.NamedTemporaryFile() + named_dockerfile.write(dockerfile) + named_dockerfile.seek(0) + + return named_dockerfile def _preprocess(self, command): """ Filters out any sudos in the command, prepends shell only commands with '/bin/bash -c'. """ + for shell_command in Docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command diff --git a/molns.py b/molns.py index d9ca02d..1cb91ae 100755 --- a/molns.py +++ b/molns.py @@ -1,35 +1,34 @@ #!/usr/bin/env python import os -import re import sys from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES, get_provider_handle from MolnsLib.molns_provider import ProviderException -from collections import OrderedDict import subprocess from MolnsLib.ssh_deploy import SSHDeploy import multiprocessing import json - import logging -logger = logging.getLogger() -#logger.setLevel(logging.INFO) #for Debugging -logger.setLevel(logging.CRITICAL) + + ############################################### class MOLNSException(Exception): pass + ############################################### class MOLNSConfig(Datastore): def __init__(self, config_dir=None, db_file=None): - Datastore.__init__(self,config_dir=config_dir, db_file=db_file) + Datastore.__init__(self, config_dir=config_dir, db_file=db_file) def __str__(self): return "MOLNSConfig(config_dir={0})".format(self.config_dir) + ############################################### class MOLNSbase(): + @classmethod - def merge_config(self, obj, config): + def merge_config(cls, obj, config): for key, conf, value in obj.get_config_vars(): if key not in config: if value is not None: From cc434db91d4d41760eb92d687be747496b5d0539 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Mon, 23 May 2016 16:41:02 -0700 Subject: [PATCH 048/100] tidy up docker provider --- MolnsLib/Docker.py | 22 ++++++---------------- MolnsLib/DockerProvider.py | 30 +++++++++++++----------------- 2 files changed, 19 insertions(+), 33 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 410df11..6fb8476 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -23,37 +23,31 @@ def __init__(self): logging.basicConfig(level=logging.DEBUG) def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): - """Create a new container.""" - + """Creates a new container.""" logging.debug(Docker.LOG_TAG + " Using image {0}".format(image_id)) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True) return container def stop_containers(self, container_ids): - """Stop given containers.""" - + """Stops given containers.""" for container_id in container_ids: self.stop_container(container_id) def stop_container(self, container_id): - """Stop the container with given ID.""" - + """Stops the container with given ID.""" self.client.stop(container_id) def container_status(self, container_id): - """Is the container with given ID running?""" - + """Checks if container with given ID running.""" return self.client.inspect_container(container_id).get('State').get('Status') def start_containers(self, container_ids): - """Start each container in given list of container IDs.""" - + """Starts each container in given list of container IDs.""" for container_id in container_ids: self.start_container(container_id) def start_container(self, container_id): """ Start the container with given ID.""" - logging.debug(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) @@ -64,13 +58,10 @@ def start_container(self, container_id): def execute_command(self, container, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" - logging.debug(Docker.LOG_TAG + " CONTAINER: {0} COMMAND: {1}".format(container.get('Id'), command)) - if self.start_container(container) is False: logging.error("Docker", " Could not start container.") return None - try: exec_instance = self.client.exec_create(container.get('Id'), "/bin/bash -c \"" + command + "\"") response = self.client.exec_start(exec_instance) @@ -81,7 +72,6 @@ def execute_command(self, container, command): def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ - print("Building image...") image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(self.build_count) last_line = "" @@ -93,6 +83,7 @@ def build_image(self, dockerfile): # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image # id. + # TODO add test for this. tokens = last_line.split(" ") image_id = tokens[3][:Constants.DOKCER_IMAGE_ID_LENGTH] print("Image ID: {0}".format(image_id)) @@ -103,7 +94,6 @@ def build_image(self, dockerfile): def image_exists(self, image_id): """Check if image with given ID exists locally.""" - for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index d8cae34..0233de2 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -35,12 +35,10 @@ class DockerProvider(DockerBase): {'q': 'Local MOLNs image name to use', 'default': None, 'ask': True}) ]) - counter = 0 - @staticmethod def __get_new_dockerfile_name(): - DockerProvider.counter += 1 - filename = Constants.Constants.DOCKERFILE_NAME + str(DockerProvider.counter) + import uuid + filename = Constants.Constants.DOCKERFILE_NAME + str(uuid.uuid4()) return filename def _connect(self): @@ -50,39 +48,37 @@ def _connect(self): self.connected = True def check_ssh_key(self): - """ Returns true. (Docker does not use SSH.)""" + """ Returns true. (Implementation does not use SSH.) """ return True def create_ssh_key(self): - """ Does nothing. """ + """ Returns true. (Implementation does not use SSH.) """ return True def check_security_group(self): - """ Does nothing.""" + """ Returns true. (Implementation does not use SSH.) """ return True def create_seurity_group(self): - """ Does nothing. """ + """ Returns true. (Implementation does not use SSH.) """ return True def create_molns_image(self): - """ Create the molns image, save it on localhost and return ID of created image. """ - + """ Create a molns image, save it on localhost and return ID of created image. """ self._connect() # create Dockerfile and build container. try: logging.debug("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) - image_tag = self.docker.build_image(dockerfile) + image_id = self.docker.build_image(dockerfile) logging.debug("Image created.") - return image_tag + return image_id except Exception as e: logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) def check_molns_image(self): """ Check if the molns image exists. """ - if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config[ 'molns_image_name'] != '': self._connect() @@ -91,8 +87,9 @@ def check_molns_image(self): def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ - - dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n git \ \n ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n + software-properties-common \ \n python-software-properties \ \n wget \ \n git \ \n ipython \n + # Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' flag = False @@ -128,8 +125,7 @@ def _create_dockerfile(self, commands): return named_dockerfile def _preprocess(self, command): - """ Filters out any sudos in the command, prepends shell only commands with '/bin/bash -c'. """ - + """ Filters out any sudos in the command, prepends "shell only" commands with '/bin/bash -c'. """ for shell_command in Docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command From 42de4b513bc6a5a0e21258c55ec832f0f2ef8b71 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Mon, 23 May 2016 18:42:20 -0700 Subject: [PATCH 049/100] wip --- .gitignore | 1 + MolnsLib/Docker.py | 24 ++++++++++----- MolnsLib/DockerProvider.py | 62 +++++++++++++++++++------------------- molns.py | 7 ++++- 4 files changed, 55 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index f7cc555..5f0e906 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ molns_install.log *.tar.gz NOTES qsubscript +/dockerfile_* \#qsubscript\# \ No newline at end of file diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 6fb8476..007b792 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -1,5 +1,5 @@ import logging - +import re from Constants import Constants from docker import Client from docker.errors import NotFound, NullResource, APIError @@ -78,18 +78,18 @@ def build_image(self, dockerfile): try: for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag): print(line) - if "errorDetail" in line: raise Docker.ImageBuildException() + if "errorDetail" in line: + raise Docker.ImageBuildException() last_line = line # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image # id. - # TODO add test for this. - tokens = last_line.split(" ") - image_id = tokens[3][:Constants.DOKCER_IMAGE_ID_LENGTH] + exp = r'[a-z0-9]{12}' + image_id = re.findall(exp, str(last_line))[0] print("Image ID: {0}".format(image_id)) return image_id except (Docker.ImageBuildException, IndexError) as e: - print(e) + print("ERROR {0}".format(e)) return None def image_exists(self, image_id): @@ -97,6 +97,16 @@ def image_exists(self, image_id): for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: - logging.debug("Image exists: " + image) + print("Image exists: " + str(image)) return True return False + + def terminate_containers(self, container_ids): + """ Terminates containers with given container ids.""" + # TODO catch errors. + self.stop_containers(container_ids) + for container_id in container_ids: + terminate_container(container_id) + + def terminate_container(self, container_id): + self.client.remove_container(container_id) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 0233de2..c15c7c6 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -32,7 +32,11 @@ class DockerProvider(DockerBase): {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'Local MOLNs image name to use', 'default': None, 'ask': True}) + {'q': 'Local MOLNs image name to use', 'default': '', 'ask': True}), + ('key_name', + {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. + ('group_name', + {'q': 'Docker Security Group name', 'default': 'molns', 'ask': True}) # Unused. ]) @staticmethod @@ -68,10 +72,10 @@ def create_molns_image(self): self._connect() # create Dockerfile and build container. try: - logging.debug("Creating Dockerfile...") + print("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) - logging.debug("Image created.") + print("Image created.") return image_id except Exception as e: logging.exception(e) @@ -88,8 +92,8 @@ def check_molns_image(self): def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n - software-properties-common \ \n python-software-properties \ \n wget \ \n git \ \n ipython \n - # Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' + software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n + ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' flag = False @@ -145,29 +149,30 @@ class DockerController(DockerBase): def _connect(self): if self.connected: return - self.docker = Docker(config=self.provider) + self.docker = Docker.Docker() self.connected = True - def get_container_status(self, container): - # TODO - logger.debug("I am not implemented yet.") + def get_container_status(self, container_id): + self._connect() + self.docker.container_status(container_id) def start_instance(self, num=1): """ Start or resume the controller. """ - # TODO - logger.debug("I am not implemented yet") + self._connect() + for i in range(num): + self.docker.create_container(Constants.Constants.DOCKER_DEFAULT_IMAGE) # TODO this needs to be MOLNs image. - def resume_instance(self, instances): - # TODO - logger.debug("I am not implemented yet") + def resume_instance(self, instance_ids): + self._connect() + self.docker.start_containers(instance_ids) - def stop_instance(self, instances): - # TODO - logger.debug("I am not implemented yet") + def stop_instance(self, instance_ids): + self._connect() + self.docker.stop_containers(instance_ids) - def terminate_instance(self, instances): - # TODO - logger.debug("I am not implemented yet.") + def terminate_instance(self, instance_ids): + self._connect() + self.docker.terminate_containers(instance_ids) class DockerWorkerGroup(DockerController): @@ -177,18 +182,13 @@ class DockerWorkerGroup(DockerController): CONFIG_VARS = OrderedDict([ ('num_vms', - {'q': 'Number of containers in group', 'default': '1', 'ask': True}), + {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) def start_container_group(self, num=1): - """ Start worker group containers. """ - - # TODO start given number of containers. - logger.debug("I am not implemented yet.") - # Look at EC2Provider, line 287. - # How to store container references in the datastore? - # What should be returned? + """ Starts worker group containers. """ + # TODO - def terminate_container_group(selfself, containers): - # TODO remove given containers. - logger.debug("I am not implemented yet.") + def terminate_container_group(self, containers): + """ Terminates container group. """ + # TODO diff --git a/molns.py b/molns.py index 1cb91ae..7fd347b 100755 --- a/molns.py +++ b/molns.py @@ -9,6 +9,10 @@ import json import logging +logger = logging.getLogger() +#logger.setLevel(logging.INFO) #for Debugging +logger.setLevel(logging.CRITICAL) + ############################################### class MOLNSException(Exception): @@ -1082,7 +1086,7 @@ def provider_setup(cls, args, config): print "Enter configuration for provider {0}:".format(args[0]) setup_object(provider_obj) config.save_object(provider_obj, kind='Provider') - # + cls.provider_initialize(args[0], config) @@ -1091,6 +1095,7 @@ def provider_initialize(cls, provider_name, config): """ Create the MOLNS image and SSH key if necessary.""" try: provider_obj = config.get_object(provider_name, kind='Provider') + print "Provider object {0}".format(provider_obj) except DatastoreException as e: raise MOLNSException("provider not found") # From e0282cf9d652f63503dc58eb63d6f6027ef7b94e Mon Sep 17 00:00:00 2001 From: aviral26 Date: Thu, 2 Jun 2016 14:56:27 -0700 Subject: [PATCH 050/100] fix return status of DockerProvider and some other minor changes --- MolnsLib/Docker.py | 29 +++++++++---- MolnsLib/DockerProvider.py | 88 +++++++++++++++++++------------------- MolnsLib/molns_provider.py | 2 +- molns.py | 3 +- 4 files changed, 67 insertions(+), 55 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 007b792..67ecdfc 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -1,5 +1,6 @@ import logging import re +from molns_provider import ProviderBase from Constants import Constants from docker import Client from docker.errors import NotFound, NullResource, APIError @@ -23,10 +24,10 @@ def __init__(self): logging.basicConfig(level=logging.DEBUG) def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): - """Creates a new container.""" - logging.debug(Docker.LOG_TAG + " Using image {0}".format(image_id)) + """Creates a new container. Returns the container ID. """ + print " Using image {0}".format(image_id) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True) - return container + return container.get("Id") def stop_containers(self, container_ids): """Stops given containers.""" @@ -39,7 +40,16 @@ def stop_container(self, container_id): def container_status(self, container_id): """Checks if container with given ID running.""" - return self.client.inspect_container(container_id).get('State').get('Status') + status = ProviderBase.STATUS_TERMINATED + try: + ret_val = str(self.client.inspect_container(container_id).get('State').get('Status')) + if ret_val.startswith("running"): + status = ProviderBase.STATUS_RUNNING + else: + status = ProviderBase.STATUS_STOPPED + except NotFound: + pass + return status def start_containers(self, container_ids): """Starts each container in given list of container IDs.""" @@ -93,7 +103,7 @@ def build_image(self, dockerfile): return None def image_exists(self, image_id): - """Check if image with given ID exists locally.""" + """Checks if an image with the given ID exists locally.""" for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: @@ -103,10 +113,13 @@ def image_exists(self, image_id): def terminate_containers(self, container_ids): """ Terminates containers with given container ids.""" - # TODO catch errors. - self.stop_containers(container_ids) for container_id in container_ids: - terminate_container(container_id) + try: + if self.container_status(container_id) == ProviderBase.STATUS_RUNNING: + self.stop_container(container_id) + self.terminate_container(container_id) + except NotFound: + pass def terminate_container(self, container_id): self.client.remove_container(container_id) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index c15c7c6..37a0685 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -16,11 +16,41 @@ def docker_provider_default_key_name(): class DockerBase(ProviderBase): - """ Abstract class for Docker. """ + """ Base class for Docker. """ SSH_KEY_EXTENSION = ".pem" PROVIDER_TYPE = 'Docker' + def _connect(self): + if self.connected: + return + self.docker = Docker.Docker() + self.connected = True + + def get_container_status(self, container_id): + self._connect() + self.docker.container_status(container_id) + + def start_instance(self, num=1): + """ Start given number (or 1) containers. """ + self._connect() + started_container_ids = [] + for i in range(num): + started_container_ids.append(self.docker.create_container(self.provider.config["molns_image_name"])) + return started_container_ids + + def resume_instance(self, instance_ids): + self._connect() + self.docker.start_containers(instance_ids) + + def stop_instance(self, instance_ids): + self._connect() + self.docker.stop_containers(instance_ids) + + def terminate_instance(self, instance_ids): + self._connect() + self.docker.terminate_containers(instance_ids) + class DockerProvider(DockerBase): """ Provider handle for local Docker based service. """ @@ -32,7 +62,7 @@ class DockerProvider(DockerBase): {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'Local MOLNs image name to use', 'default': '', 'ask': True}), + {'q': 'Local MOLNs image ID to use', 'default': '', 'ask': True}), ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', @@ -45,19 +75,20 @@ def __get_new_dockerfile_name(): filename = Constants.Constants.DOCKERFILE_NAME + str(uuid.uuid4()) return filename - def _connect(self): - if self.connected: - return - self.docker = Docker.Docker() - self.connected = True - def check_ssh_key(self): """ Returns true. (Implementation does not use SSH.) """ + print "reached_check_ssh_key" return True def create_ssh_key(self): - """ Returns true. (Implementation does not use SSH.) """ - return True + """ Returns true. """ + # TODO + print "reached create_ssh_key" + ssh_key_dir = os.path.join(self.config_dir, self.name) + fp = open(ssh_key_dir, 'w') + fp.write("This is a dummy key.") + fp.close() + os.chmod(ssh_key_dir, 0o600) def check_security_group(self): """ Returns true. (Implementation does not use SSH.) """ @@ -146,34 +177,9 @@ class DockerController(DockerBase): CONFIG_VARS = OrderedDict([ ]) - def _connect(self): - if self.connected: - return - self.docker = Docker.Docker() - self.connected = True - - def get_container_status(self, container_id): - self._connect() - self.docker.container_status(container_id) - - def start_instance(self, num=1): - """ Start or resume the controller. """ - self._connect() - for i in range(num): - self.docker.create_container(Constants.Constants.DOCKER_DEFAULT_IMAGE) # TODO this needs to be MOLNs image. - - def resume_instance(self, instance_ids): - self._connect() - self.docker.start_containers(instance_ids) - - def stop_instance(self, instance_ids): - self._connect() - self.docker.stop_containers(instance_ids) - - def terminate_instance(self, instance_ids): + def get_instance_status(self, instance): self._connect() - self.docker.terminate_containers(instance_ids) - + return self.docker.container_status(instance) class DockerWorkerGroup(DockerController): """ Provider handle for Docker worker group. """ @@ -184,11 +190,3 @@ class DockerWorkerGroup(DockerController): ('num_vms', {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) - - def start_container_group(self, num=1): - """ Starts worker group containers. """ - # TODO - - def terminate_container_group(self, containers): - """ Terminates container group. """ - # TODO diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index f02285b..9216fea 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -6,7 +6,7 @@ class ProviderException(Exception): pass -class ProviderBase(): +class ProviderBase: """ Abstract class. """ STATUS_RUNNING = 'running' diff --git a/molns.py b/molns.py index 7fd347b..9dc2b95 100755 --- a/molns.py +++ b/molns.py @@ -416,7 +416,8 @@ def start_controller(cls, args, config, password=None): """ Start the MOLNs controller. """ logging.debug("MOLNSController.start_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return # Check if any instances are assigned to this controller instance_list = config.get_all_instances(controller_id=controller_obj.id) # Check if they are running or stopped (if so, resume them) From 44f452a5d05ce41caaac3633ef06031e6a7e1a00 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Fri, 3 Jun 2016 12:42:41 -0700 Subject: [PATCH 051/100] add ssh object to providers --- MolnsLib/Docker.py | 10 +-- MolnsLib/DockerProvider.py | 33 +++++--- MolnsLib/DockerSSH.py | 20 +++++ MolnsLib/SSH.py | 55 +++++++++++++ MolnsLib/molns_provider.py | 2 + MolnsLib/ssh_deploy.py | 157 +++++++++++++++---------------------- molns.py | 31 ++++++-- 7 files changed, 189 insertions(+), 119 deletions(-) create mode 100644 MolnsLib/DockerSSH.py create mode 100644 MolnsLib/SSH.py diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 67ecdfc..ef92d50 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -66,14 +66,14 @@ def start_container(self, container_id): return False return True - def execute_command(self, container, command): + def execute_command(self, container_id, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" - logging.debug(Docker.LOG_TAG + " CONTAINER: {0} COMMAND: {1}".format(container.get('Id'), command)) - if self.start_container(container) is False: - logging.error("Docker", " Could not start container.") + print("CONTAINER: {0} COMMAND: {1}".format(container_id, command)) + if self.start_container(container_id) is False: + print(" Could not start container.") return None try: - exec_instance = self.client.exec_create(container.get('Id'), "/bin/bash -c \"" + command + "\"") + exec_instance = self.client.exec_create(container_id, "/bin/bash -c \"" + command + "\"") response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 37a0685..31c52bb 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -5,6 +5,7 @@ import Docker import installSoftware import tempfile +from DockerSSH import DockerSSH from collections import OrderedDict from molns_provider import ProviderBase, ProviderException @@ -21,36 +22,34 @@ class DockerBase(ProviderBase): SSH_KEY_EXTENSION = ".pem" PROVIDER_TYPE = 'Docker' - def _connect(self): - if self.connected: - return + def __init__(self, name, config=None, config_dir=None, **kwargs): + ProviderBase.__init__(self, name, config, config_dir, **kwargs) self.docker = Docker.Docker() - self.connected = True + self.ssh = DockerSSH(self.docker) def get_container_status(self, container_id): - self._connect() self.docker.container_status(container_id) def start_instance(self, num=1): """ Start given number (or 1) containers. """ - self._connect() started_container_ids = [] for i in range(num): started_container_ids.append(self.docker.create_container(self.provider.config["molns_image_name"])) + # TODO Store these IDs somewhere. return started_container_ids def resume_instance(self, instance_ids): - self._connect() self.docker.start_containers(instance_ids) def stop_instance(self, instance_ids): - self._connect() self.docker.stop_containers(instance_ids) def terminate_instance(self, instance_ids): - self._connect() self.docker.terminate_containers(instance_ids) + def exec_command(self, container_id, command): + self.docker.execute_command(container_id, command) + class DockerProvider(DockerBase): """ Provider handle for local Docker based service. """ @@ -100,7 +99,6 @@ def create_seurity_group(self): def create_molns_image(self): """ Create a molns image, save it on localhost and return ID of created image. """ - self._connect() # create Dockerfile and build container. try: print("Creating Dockerfile...") @@ -116,7 +114,6 @@ def check_molns_image(self): """ Check if the molns image exists. """ if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config[ 'molns_image_name'] != '': - self._connect() return self.docker.image_exists(self.config['molns_image_name']) return False @@ -178,9 +175,9 @@ class DockerController(DockerBase): ]) def get_instance_status(self, instance): - self._connect() return self.docker.container_status(instance) + class DockerWorkerGroup(DockerController): """ Provider handle for Docker worker group. """ @@ -190,3 +187,15 @@ class DockerWorkerGroup(DockerController): ('num_vms', {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) + + +class DockerLocal(DockerBase): + """ Provides a handle for a local Docker based ipython server. """ + + OBJ_NAME = 'DockerLocal' + + CONFIG_VARS = OrderedDict([ + ]) + + def get_instance_status(self, instance): + return self.docker.container_status(instance) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py new file mode 100644 index 0000000..d02887b --- /dev/null +++ b/MolnsLib/DockerSSH.py @@ -0,0 +1,20 @@ +class DockerSSH(object): + + def __init__(self, docker): + self.docker = docker + + def exec_command(self, command, verbose=True): + # TODO + print("DockerSSH exec_command not yet implemented.") + + def open_sftp(self): + # TODO + print("DockerSSH open_sftp not yet implemented.") + + def connect(self, hostname, port=None, username=None, password=None, key_filename=None): + # TODO + print("DockerSSH connect not yet implemented.") + + def close(self): + # TODO + print("DockerSSH close not yet implemented.") diff --git a/MolnsLib/SSH.py b/MolnsLib/SSH.py new file mode 100644 index 0000000..9e99f5f --- /dev/null +++ b/MolnsLib/SSH.py @@ -0,0 +1,55 @@ +import paramiko +import time + + +class SSHException(Exception): + pass + + +class SSH: + def __init__(self): + self.ssh = paramiko.SSHClient() + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + def exec_command(self, command, verbose=True): + try: + stdout_data = [] + stderr_data = [] + session = self.ssh.get_transport().open_session() + session.exec_command(command) + nbytes = 4096 + # TODO add a timeout here, don't wait for commands forever. + while True: + if session.recv_ready(): + msg = session.recv(nbytes) + stdout_data.append(msg) + if session.recv_stderr_ready(): + msg = session.recv_stderr(nbytes) + stderr_data.append(msg) + if session.exit_status_ready(): + break + time.sleep(0.1) # Sleep briefly to prevent over-polling + + status = session.recv_exit_status() + str_return = ''.join(stdout_data).splitlines() + stderr_str = ''.join(stderr_data) + session.close() + if status != 0: + raise paramiko.SSHException( + "Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, "\n".join(str_return), stderr_str)) + if verbose: + print "EXECUTING...\t{0}".format(command) + return str_return + except paramiko.SSHException as e: + if verbose: + print "FAILED......\t{0}\t{1}".format(command, e) + raise SSHException("{0}\t{1}".format(command, e)) + + def open_sftp(self): + return self.ssh.open_sftp() + + def connect(self, hostname, port, username=None, key_filename=None): + return self.ssh.connect(hostname, port, username, key_filename=key_filename) + + def close(self): + self.ssh.close() diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index 9216fea..9302521 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -1,5 +1,6 @@ import os import collections +from SSH import SSH class ProviderException(Exception): @@ -38,6 +39,7 @@ def __init__(self, name, config=None, config_dir=None, **kwargs): self.config[k] = v for k,v in kwargs.iteritems(): self.__dict__[k] = v + self.ssh = SSH() def __getitem__(self, key): if key not in self.CONFIG_VARS.keys(): diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 129b5ce..cdb720a 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -1,4 +1,3 @@ - import json import logging import os @@ -9,10 +8,14 @@ import uuid import webbrowser import urllib2 +from SSH import SSH +from DockerSSH import DockerSSH + class SSHDeployException(Exception): pass + class SSHDeploy: ''' This class is used for deploy IPython @@ -31,8 +34,7 @@ class SSHDeploy: DEFAULT_PYURDME_TEMPDIR="/mnt/pyurdme_tmp" - - def __init__(self, config=None, config_dir=None): + def __init__(self, ssh, config=None, config_dir=None): if config is None: raise SSHDeployException("No config given") self.config = config @@ -43,16 +45,16 @@ def __init__(self, config=None, config_dir=None): self.endpoint = self.DEFAULT_PRIVATE_NOTEBOOK_PORT self.ssh_endpoint = self.DEFAULT_SSH_PORT self.keyfile = config.sshkeyfilename() - self.ssh = paramiko.SSHClient() - self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + if not isinstance(ssh, SSH) or not isinstance(ssh, DockerSSH): + raise SSHDeployException("Received incorrect SSH object.") + self.ssh = ssh self.profile = 'default' - self.profile_dir = "/home/%s/.ipython/profile_default/" %(self.username) + self.profile_dir = "/home/%s/.ipython/profile_default/" % (self.username) self.ipengine_env = 'export INSTANT_OS_CALL_METHOD=SUBPROCESS;export PYURDME_TMPDIR={0};'.format(self.DEFAULT_PYURDME_TEMPDIR) self.profile_dir_server = self.profile_dir self.profile_dir_client = self.profile_dir self.ipython_port = self.DEFAULT_IPCONTROLLER_PORT - def scp_command(self, hostname): return "scp -o 'StrictHostKeyChecking no' \ %s@%s:%ssecurity/ipcontroller-engine.json %ssecurity/" \ @@ -72,12 +74,12 @@ def prompt_for_password(self): print "Passwords do not match, try again." def create_ssl_cert(self, cert_directory, cert_name_prefix, hostname): - self.exec_command("mkdir -p '{0}'".format(cert_directory)) + self.ssh.exec_command("mkdir -p '{0}'".format(cert_directory)) user_cert = cert_directory + '{0}-user_cert.pem'.format(cert_name_prefix) ssl_key = cert_directory + '{0}-ssl_key.pem'.format(cert_name_prefix) ssl_cert = cert_directory + '{0}-ssl_cert.pem'.format(cert_name_prefix) ssl_subj = "/C=CN/ST=SH/L=STAR/O=Dis/CN=%s" % hostname - self.exec_command( + self.ssh.exec_command( "openssl req -new -newkey rsa:4096 -days 365 " '-nodes -x509 -subj %s -keyout %s -out %s' % (ssl_subj, ssl_key, ssl_cert)) @@ -94,7 +96,7 @@ def create_ipython_config(self, hostname, notebook_password=None): else: passwd = notebook_password try: - sha1pass_out = self.exec_command(sha1cmd % passwd , verbose=False) + sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) sha1pass = sha1pass_out[0].strip() except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, hostname, self.ssh_endpoint) @@ -147,14 +149,14 @@ def create_s3_config(self): s3_config_file = sftp.file(remote_file_name, 'w') config = {} config["provider_type"] = self.config.type - config["bucket_name"] = "molns_storage_{0}".format(self.get_cluster_id()) + config["bucket_name"] = "molns_storage_{0}".format(self.get_cluster_id()) config["credentials"] = self.config.get_config_credentials() s3_config_file.write(json.dumps(config)) s3_config_file.close() sftp.close() def get_cluster_id(self): - """ retreive the cluster id from the config. """ + """ Retrieve the cluster id from the config. """ filename = os.path.join(self.config_dir, 'cluster_id') if not os.path.isfile(filename): new_id = str(uuid.uuid4()) @@ -168,7 +170,6 @@ def get_cluster_id(self): raise SSHDeployException("error getting id for cluster from file, please check your file '{0}'".format(filename)) return idstr - def create_engine_config(self): sftp = self.ssh.open_sftp() remote_file_name='%sipengine_config.py' % self.profile_dir_server @@ -217,40 +218,7 @@ def _put_ipython_engine_file(self, file_data): def exec_command_list_switch(self, command_list): for command in command_list: - self.exec_command(command) - - def exec_command(self, command, verbose=True): - try: - stdout_data = [] - stderr_data = [] - session = self.ssh.get_transport().open_session() - session.exec_command(command) - nbytes = 4096 - #TODO add a timeout here, don't wait for commands forever. - while True: - if session.recv_ready(): - msg = session.recv(nbytes) - stdout_data.append(msg) - if session.recv_stderr_ready(): - msg = session.recv_stderr(nbytes) - stderr_data.append(msg) - if session.exit_status_ready(): - break - time.sleep(0.1) # Sleep breifly to prevent over-polling - - status = session.recv_exit_status() - str_return = ''.join(stdout_data).splitlines() - stderr_str = ''.join(stderr_data) - session.close() - if status != 0: - raise paramiko.SSHException("Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, "\n".join(str_return), stderr_str)) - if verbose: - print "EXECUTING...\t{0}".format(command) - return str_return - except paramiko.SSHException as e: - if verbose: - print "FAILED......\t{0}\t{1}".format(command,e) - raise SSHDeployException("{0}\t{1}".format(command,e)) + self.ssh.exec_command(command) def exec_multi_command(self, command, next_command): try: @@ -259,33 +227,34 @@ def exec_multi_command(self, command, next_command): stdin.flush() status = stdout.channel.recv_exit_status() if status != 0: - raise paramiko.SSHException("Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, stdout.read(), stderr.read())) + raise paramiko.SSHException("Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, stdout.read(), + stderr.read())) except paramiko.SSHException as e: - print "FAILED......\t{0}\t{1}".format(command,e) + print "FAILED......\t{0}\t{1}".format(command, e) raise e def connect(self, hostname, port): - print "Connecting to {0}:{1} keyfile={2}".format(hostname,port,self.keyfile) + print "Connecting to {0}:{1} keyfile={2}".format(hostname, port, self.keyfile) for i in range(self.MAX_NUMBER_SSH_CONNECT_ATTEMPTS): try: self.ssh.connect(hostname, port, username=self.username, - key_filename=self.keyfile) + key_filename=self.keyfile) print "SSH connection established" return except Exception as e: - print "Retry in {0} seconds...\t\t{1}".format(self.SSH_CONNECT_WAITTIME,e) + print "Retry in {0} seconds...\t\t{1}".format(self.SSH_CONNECT_WAITTIME, e) time.sleep(self.SSH_CONNECT_WAITTIME) - raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(hostname,self.ssh_endpoint)) + raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(hostname, self.ssh_endpoint)) def deploy_molns_webserver(self, ip_address): try: self.connect(ip_address, self.ssh_endpoint) - self.exec_command("sudo rm -rf /usr/local/molns_webroot") - self.exec_command("sudo mkdir -p /usr/local/molns_webroot") - self.exec_command("sudo chown ubuntu /usr/local/molns_webroot") - self.exec_command("git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") + self.ssh.exec_command("sudo rm -rf /usr/local/molns_webroot") + self.ssh.exec_command("sudo mkdir -p /usr/local/molns_webroot") + self.ssh.exec_command("sudo chown ubuntu /usr/local/molns_webroot") + self.ssh.exec_command("git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") self.exec_multi_command("cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format(self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') - self.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_WEBSERVER_PORT,self.DEFAULT_PRIVATE_WEBSERVER_PORT)) + self.ssh.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_WEBSERVER_PORT, self.DEFAULT_PRIVATE_WEBSERVER_PORT)) self.ssh.close() print "Deploying MOLNs webserver" url = "http://{0}/".format(ip_address) @@ -308,7 +277,7 @@ def deploy_molns_webserver(self, ip_address): def get_number_processors(self): cmd = 'python -c "import multiprocessing;print multiprocessing.cpu_count()"' try: - output = self.exec_command(cmd)[0].strip() + output = self.ssh.exec_command(cmd)[0].strip() return int(output) except Exception as e: raise SSHDeployException("Could not determine the number of processors on the remote system: {0}".format(e)) @@ -329,16 +298,16 @@ def deploy_stochss(self, ip_address, port=1443): print buff web_file.write(buff) web_file.close() - self.exec_command("sudo chown root /tmp/nginx.conf") - self.exec_command("sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf") + self.ssh.exec_command("sudo chown root /tmp/nginx.conf") + self.ssh.exec_command("sudo mv /tmp/nginx.conf /etc/nginx/nginx.conf") print "Starting Nginx" - self.exec_command("sudo nginx") + self.ssh.exec_command("sudo nginx") print "Modifying StochSS to not open a webbrowser (TODO: move to install)" - self.exec_command("sed -i 's/webbrowser.open_new(stochss_url)/pass/' /usr/local/stochss/run.ubuntu.sh") + self.ssh.exec_command("sed -i 's/webbrowser.open_new(stochss_url)/pass/' /usr/local/stochss/run.ubuntu.sh") print "Starting StochSS" - self.exec_command("cd /usr/local/stochss/ && screen -d -m ./run.ubuntu.sh") + self.ssh.exec_command("cd /usr/local/stochss/ && screen -d -m ./run.ubuntu.sh") print "Waiting for StochSS to become available:" stochss_url = "https://{0}/".format(ip_address) while True: @@ -354,7 +323,7 @@ def deploy_stochss(self, ip_address, port=1443): print "Configuring StochSS" admin_token = uuid.uuid4() create_and_exchange_admin_token = "python /usr/local/stochss/generate_admin_token.py {0}".format(admin_token) - self.exec_command(create_and_exchange_admin_token) + self.ssh.exec_command(create_and_exchange_admin_token) time.sleep(1) stochss_url = "{0}login?secret_key={1}".format(stochss_url, admin_token) print "StochSS available: {0}".format(stochss_url) @@ -371,36 +340,36 @@ def deploy_ipython_controller(self, ip_address, notebook_password=None): self.connect(ip_address, self.ssh_endpoint) # Set up the symlink to local scratch space - self.exec_command("sudo mkdir -p /mnt/molnsarea") - self.exec_command("sudo chown ubuntu /mnt/molnsarea") - self.exec_command("sudo mkdir -p /mnt/molnsarea/cache") - self.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") + self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea") + self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea") + self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea/cache") + self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") - self.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) + self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) # Setup symlink to the shared scratch space - self.exec_command("sudo mkdir -p /mnt/molnsshared") - self.exec_command("sudo chown ubuntu /mnt/molnsshared") - self.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsshared {0}".format('/home/ubuntu/shared')) + self.ssh.exec_command("sudo mkdir -p /mnt/molnsshared") + self.ssh.exec_command("sudo chown ubuntu /mnt/molnsshared") + self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsshared {0}".format('/home/ubuntu/shared')) # - self.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) - self.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) + self.ssh.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) + self.ssh.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) # #self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") - self.exec_command("mkdir -p .molns") + self.ssh.exec_command("mkdir -p .molns") self.create_s3_config() - self.exec_command("ipython profile create {0}".format(self.profile)) + self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_ipython_config(ip_address, notebook_password) self.create_engine_config() - self.exec_command("source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2} --log-to-file".format(ip_address, self.profile, self.ipython_port), '\n') + self.ssh.exec_command("source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2} --log-to-file".format(ip_address, self.profile, self.ipython_port), '\n') # Start one ipengine per processor num_procs = self.get_number_processors() num_engines = num_procs - 2 for _ in range(num_engines): - self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) - self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format(self.profile, self.ipengine_env)) - self.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_NOTEBOOK_PORT,self.DEFAULT_PRIVATE_NOTEBOOK_PORT)) + self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) + self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format(self.profile, self.ipengine_env)) + self.ssh.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_NOTEBOOK_PORT,self.DEFAULT_PRIVATE_NOTEBOOK_PORT)) self.ssh.close() except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) @@ -437,18 +406,18 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont self.connect(ip_address, self.ssh_endpoint) # Setup the symlink to local scratch space - self.exec_command("sudo mkdir -p /mnt/molnsarea") - self.exec_command("sudo chown ubuntu /mnt/molnsarea") - self.exec_command("sudo mkdir -p /mnt/molnsarea/cache") - self.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") + self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea") + self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea") + self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea/cache") + self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") - self.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) + self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) # - self.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) - self.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) + self.ssh.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) + self.ssh.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) # Setup config for object store - self.exec_command("mkdir -p .molns") + self.ssh.exec_command("mkdir -p .molns") self.create_s3_config() @@ -463,20 +432,20 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont controller_keyfile.close() print "Remote file {0} has {1} bytes".format(remote_file_name, sftp.stat(remote_file_name).st_size) sftp.close() - self.exec_command("chmod 0600 {0}".format(remote_file_name)) - self.exec_command("mkdir -p /home/ubuntu/shared") - self.exec_command("sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format(controler_ip)) + self.ssh.exec_command("chmod 0600 {0}".format(remote_file_name)) + self.ssh.exec_command("mkdir -p /home/ubuntu/shared") + self.ssh.exec_command("sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format(controler_ip)) # Update the Molnsutil package: TODO remove when molnsutil is stable #self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") - self.exec_command("ipython profile create {0}".format(self.profile)) + self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_engine_config() # Just write the engine_file to the engine self._put_ipython_engine_file(engine_file_data) # Start one ipengine per processor for _ in range(self.get_number_processors()): - self.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) + self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) self.ssh.close() diff --git a/molns.py b/molns.py index 9dc2b95..00295dd 100755 --- a/molns.py +++ b/molns.py @@ -88,6 +88,7 @@ def _get_controllerobj(cls, args, config): raise MOLNSException("controller '{0}' is not initialized, use 'molns controller setup {0}' to initialize the controller.".format(controller_name)) return controller_obj + class MOLNSController(MOLNSbase): @classmethod def controller_export(cls, args, config): @@ -356,7 +357,9 @@ def status_controller(cls, args, config): logging.debug("MOLNSController.status_controller(args={0})".format(args)) if len(args) > 0: controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return + # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) table_data = [] @@ -438,7 +441,7 @@ def start_controller(cls, args, config, password=None): print "Starting new controller" inst = controller_obj.start_instance() # deploying - sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) + sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password) sshdeploy.deploy_molns_webserver(inst.ip_address) #sshdeploy.deploy_stochss(inst.ip_address, port=443) @@ -466,19 +469,19 @@ def stop_controller(cls, args, config): if status == worker_obj.STATUS_RUNNING or status == worker_obj.STATUS_STOPPED: print "Terminating worker '{1}' running at {0}".format(i.ip_address, worker_name) worker_obj.terminate_instance(i) - else: print "No instance running for this controller" - @classmethod def terminate_controller(cls, args, config): """ Terminate the head node of a MOLNs controller. """ logging.debug("MOLNSController.terminate_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return instance_list = config.get_all_instances(controller_id=controller_obj.id) logging.debug("\tinstance_list={0}".format([str(i) for i in instance_list])) + print("\tinstance_list={0}".format([str(i) for i in instance_list])) # Check if they are running or stopped if len(instance_list) > 0: for i in instance_list: @@ -494,8 +497,6 @@ def terminate_controller(cls, args, config): if status == worker_obj.STATUS_RUNNING or status == worker_obj.STATUS_STOPPED: print "Terminating worker '{1}' running at {0}".format(i.ip_address, worker_name) worker_obj.terminate_instance(i) - - else: print "No instance running for this controller" @@ -544,7 +545,7 @@ def worker_group_export(cls, args, config): """ Export the configuration of a worker group. """ if len(args) < 1: raise MOLNSException("USAGE: molns worker export name [Filename]\n"\ - "\tExport the data from the worker group with the given name.") + "\tExport the data from the worker group with the given name.") worker_name = args[0] if len(args) > 1: filename = args[1] @@ -1242,6 +1243,20 @@ def clear_instances(cls, args, config): else: print "No instance found" +############################################################################################## + + +class MolnsLocal(MOLNSbase): + + @classmethod + def setup_local(cls): + #TODO + pass + + @classmethod + def start_local(cls): + # TODO + pass ############################################################################################## ############################################################################################## From 284e285d853413463ce33a3902b9a57ac095091a Mon Sep 17 00:00:00 2001 From: aviral26 Date: Fri, 3 Jun 2016 16:50:12 -0700 Subject: [PATCH 052/100] WIP. implement execute_command and connect --- MolnsLib/DockerProvider.py | 73 +++++++++++++++++++++++++------------- MolnsLib/DockerSSH.py | 9 +++-- MolnsLib/SSH.py | 4 +-- MolnsLib/ssh_deploy.py | 29 +++++++++------ molns.py | 7 ++-- 5 files changed, 78 insertions(+), 44 deletions(-) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 31c52bb..261d4e0 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -27,24 +27,46 @@ def __init__(self, name, config=None, config_dir=None, **kwargs): self.docker = Docker.Docker() self.ssh = DockerSSH(self.docker) - def get_container_status(self, container_id): + def _get_container_status(self, container_id): self.docker.container_status(container_id) def start_instance(self, num=1): """ Start given number (or 1) containers. """ - started_container_ids = [] + started_containers = [] for i in range(num): - started_container_ids.append(self.docker.create_container(self.provider.config["molns_image_name"])) - # TODO Store these IDs somewhere. - return started_container_ids - - def resume_instance(self, instance_ids): + container_id = self.docker.create_container(self.provider.config["molns_image_name"]) + stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=None + , provider_id=self.provider.id, controller_id=self.id) + started_containers.append(stored_container) + if num == 1: + return started_containers[0] + return started_containers + + def resume_instance(self, instances): + instance_ids = [] + if isinstance(instances, list): + for instance in instances: + instance_ids.append(instance.provider_instance_identifier) + else: + instance_ids.append(instances.provider_instance_identifier) self.docker.start_containers(instance_ids) - def stop_instance(self, instance_ids): + def stop_instance(self, instances): + instance_ids = [] + if isinstance(instances, list): + for instance in instances: + instance_ids.append(instance.provider_instance_identifier) + else: + instance_ids.append(instances.provider_instance_identifier) self.docker.stop_containers(instance_ids) - def terminate_instance(self, instance_ids): + def terminate_instance(self, instances): + instance_ids = [] + if isinstance(instances, list): + for instance in instances: + instance_ids.append(instance.provider_instance_identifier) + else: + instance_ids.append(instances.provider_instance_identifier) self.docker.terminate_containers(instance_ids) def exec_command(self, container_id, command): @@ -65,7 +87,9 @@ class DockerProvider(DockerBase): ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', - {'q': 'Docker Security Group name', 'default': 'molns', 'ask': True}) # Unused. + {'q': 'Docker Security Group name', 'default': 'molns', 'ask': True}), # Unused. + ('login_username', + {'default': 'ubuntu', 'ask': False}) # Unused. ]) @staticmethod @@ -121,7 +145,8 @@ def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n - ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu\n''' + ipython \n apt-get -y install sudo \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu && echo "ubuntu ALL= + (ALL) NOPASSWD: ALL" >> /etc/sudoers\nWORKDIR /home/ubuntu\n''' flag = False @@ -175,7 +200,7 @@ class DockerController(DockerBase): ]) def get_instance_status(self, instance): - return self.docker.container_status(instance) + return self.docker.container_status(instance.provider_instance_identifier) class DockerWorkerGroup(DockerController): @@ -187,15 +212,15 @@ class DockerWorkerGroup(DockerController): ('num_vms', {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) - - -class DockerLocal(DockerBase): - """ Provides a handle for a local Docker based ipython server. """ - - OBJ_NAME = 'DockerLocal' - - CONFIG_VARS = OrderedDict([ - ]) - - def get_instance_status(self, instance): - return self.docker.container_status(instance) +# +# +# class DockerLocal(DockerBase): +# """ Provides a handle for a local Docker based ipython server. """ +# +# OBJ_NAME = 'DockerLocal' +# +# CONFIG_VARS = OrderedDict([ +# ]) +# +# def get_instance_status(self, instance): +# return self.docker.container_status(instance) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index d02887b..694c286 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -2,18 +2,17 @@ class DockerSSH(object): def __init__(self, docker): self.docker = docker + self.container_id = None def exec_command(self, command, verbose=True): - # TODO - print("DockerSSH exec_command not yet implemented.") + self.docker.execute_command(self.container_id, command) def open_sftp(self): # TODO print("DockerSSH open_sftp not yet implemented.") - def connect(self, hostname, port=None, username=None, password=None, key_filename=None): - # TODO - print("DockerSSH connect not yet implemented.") + def connect(self, instance, port=None, username=None, key_filename=None): + self.container_id = instance.provider_instance_identifier def close(self): # TODO diff --git a/MolnsLib/SSH.py b/MolnsLib/SSH.py index 9e99f5f..34f34f1 100644 --- a/MolnsLib/SSH.py +++ b/MolnsLib/SSH.py @@ -48,8 +48,8 @@ def exec_command(self, command, verbose=True): def open_sftp(self): return self.ssh.open_sftp() - def connect(self, hostname, port, username=None, key_filename=None): - return self.ssh.connect(hostname, port, username, key_filename=key_filename) + def connect(self, instance, port, username=None, key_filename=None): + return self.ssh.connect(instance.ip_address, port, username, key_filename=key_filename) def close(self): self.ssh.close() diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index cdb720a..dbc5e41 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -45,8 +45,8 @@ def __init__(self, ssh, config=None, config_dir=None): self.endpoint = self.DEFAULT_PRIVATE_NOTEBOOK_PORT self.ssh_endpoint = self.DEFAULT_SSH_PORT self.keyfile = config.sshkeyfilename() - if not isinstance(ssh, SSH) or not isinstance(ssh, DockerSSH): - raise SSHDeployException("Received incorrect SSH object.") + if not (isinstance(ssh, SSH) or isinstance(ssh, DockerSSH)): + raise SSHDeployException("SSH object invalid.") self.ssh = ssh self.profile = 'default' self.profile_dir = "/home/%s/.ipython/profile_default/" % (self.username) @@ -233,22 +233,28 @@ def exec_multi_command(self, command, next_command): print "FAILED......\t{0}\t{1}".format(command, e) raise e - def connect(self, hostname, port): - print "Connecting to {0}:{1} keyfile={2}".format(hostname, port, self.keyfile) + def connect(self, instance, port=None): + if port is None: + port = self.ssh_endpoint + print "Connecting to {0}:{1} keyfile={2}".format(instance.ip_address, port, self.keyfile) for i in range(self.MAX_NUMBER_SSH_CONNECT_ATTEMPTS): try: - self.ssh.connect(hostname, port, username=self.username, + self.ssh.connect(instance, self.ssh_endpoint, username=self.username, key_filename=self.keyfile) - print "SSH connection established" + if not isinstance(self.ssh, DockerSSH): + print "SSH connection established" + else: + print "Ready to execute commands in local container." return except Exception as e: print "Retry in {0} seconds...\t\t{1}".format(self.SSH_CONNECT_WAITTIME, e) time.sleep(self.SSH_CONNECT_WAITTIME) - raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(hostname, self.ssh_endpoint)) + raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(instance.ip_address, self.ssh_endpoint)) - def deploy_molns_webserver(self, ip_address): + def deploy_molns_webserver(self, instance): + ip_address = instance.ip_address try: - self.connect(ip_address, self.ssh_endpoint) + self.connect(instance, self.ssh_endpoint) self.ssh.exec_command("sudo rm -rf /usr/local/molns_webroot") self.ssh.exec_command("sudo mkdir -p /usr/local/molns_webroot") self.ssh.exec_command("sudo chown ubuntu /usr/local/molns_webroot") @@ -332,12 +338,13 @@ def deploy_stochss(self, ip_address, port=1443): print "StochSS launch failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def deploy_ipython_controller(self, ip_address, notebook_password=None): + def deploy_ipython_controller(self, instance, notebook_password=None): + ip_address = instance.ip_address controller_hostname = '' engine_file_data = '' try: print "{0}:{1}".format(ip_address, self.ssh_endpoint) - self.connect(ip_address, self.ssh_endpoint) + self.connect(instance, self.ssh_endpoint) # Set up the symlink to local scratch space self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea") diff --git a/molns.py b/molns.py index 00295dd..1557644 100755 --- a/molns.py +++ b/molns.py @@ -442,8 +442,8 @@ def start_controller(cls, args, config, password=None): inst = controller_obj.start_instance() # deploying sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) - sshdeploy.deploy_ipython_controller(inst.ip_address, notebook_password=password) - sshdeploy.deploy_molns_webserver(inst.ip_address) + sshdeploy.deploy_ipython_controller(inst, notebook_password=password) + sshdeploy.deploy_molns_webserver(inst) #sshdeploy.deploy_stochss(inst.ip_address, port=443) @classmethod @@ -1371,6 +1371,8 @@ def run(self, args, config_dir=None): raise CommandException("command not found") ############################################### + + class Command(): def __init__(self, command, args_defs={}, description=None, function=None): self.command = command @@ -1382,6 +1384,7 @@ def __init__(self, command, args_defs={}, description=None, function=None): self.description = function.__doc__.strip() else: self.description = description + def __str__(self): ret = self.command+" " for k,v in self.args_defs.iteritems(): From b6c4a96731d9f5feb0d873e0af439fddf05ae04a Mon Sep 17 00:00:00 2001 From: aviral26 Date: Sat, 4 Jun 2016 17:18:24 -0700 Subject: [PATCH 053/100] add mock sft to docker provider --- MolnsLib/Docker.py | 10 ++++++ MolnsLib/DockerProvider.py | 3 ++ MolnsLib/DockerSSH.py | 63 +++++++++++++++++++++++++++++++++++--- MolnsLib/ssh_deploy.py | 4 +-- 4 files changed, 73 insertions(+), 7 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index ef92d50..237f8c8 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -123,3 +123,13 @@ def terminate_containers(self, container_ids): def terminate_container(self, container_id): self.client.remove_container(container_id) + + def put_archive(self, container_id, tar_file_bytes, target_path_in_container): + if self.start_container(container_id) is False: + print("ERROR Could not start container.") + return + print("Unpacking archive to: " + target_path_in_container) + if self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): + print "Copied file successfully." + else: + print "Failed to copy." diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 261d4e0..ebbefeb 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -92,6 +92,9 @@ class DockerProvider(DockerBase): {'default': 'ubuntu', 'ask': False}) # Unused. ]) + def get_config_credentials(self): + None + @staticmethod def __get_new_dockerfile_name(): import uuid diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 694c286..c257302 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -1,5 +1,9 @@ -class DockerSSH(object): +import StringIO +import tarfile +import os + +class DockerSSH(object): def __init__(self, docker): self.docker = docker self.container_id = None @@ -8,12 +12,61 @@ def exec_command(self, command, verbose=True): self.docker.execute_command(self.container_id, command) def open_sftp(self): - # TODO - print("DockerSSH open_sftp not yet implemented.") + return MockSFTP(self.docker, self.container_id) def connect(self, instance, port=None, username=None, key_filename=None): self.container_id = instance.provider_instance_identifier def close(self): - # TODO - print("DockerSSH close not yet implemented.") + self.container_id = None + + +class MockSFTPFileException(Exception): + pass + + +class MockSFTP: + def __init__(self, docker, container_id): + self.docker = docker + self.container_id = container_id + + def file(self, filename, flag): + return MockSFTPFile(filename, flag, self.docker, self.container_id) + + def close(self): + pass + + +class MockSFTPFile: + def __init__(self, filename, flag, docker, container_id): + self.filename = filename # Absolute path of file. + self.file_contents = "" + self.docker = docker + self.container_id = container_id + if flag is 'w': + self.flag = flag + else: + print("WARNING Unrecognized file mode.") + + def write(self, write_this): + self.file_contents += write_this + + def close(self): + # Make tarfile. + temp_tar = "transport.tar" + tar = tarfile.TarFile(temp_tar, "w") + string = StringIO.StringIO() + string.write(self.file_contents) + string.seek(0) + tar_file_info = tarfile.TarInfo(name=os.path.basename(self.filename)) + tar_file_info.size = len(string.buf) + tar.addfile(tarinfo=tar_file_info, fileobj=string) + tar.close() # Create temporary tar file to be copied into container. + + path_to_file = os.path.dirname(self.filename) + create_path_to_file_command = "sudo mkdir -p {0}".format(path_to_file) + self.docker.execute_command(self.container_id, create_path_to_file_command) + with open(temp_tar, mode='rb') as f: + tar_file_bytes = f.read() + self.docker.put_archive(self.container_id, tar_file_bytes, "/home/ubuntu/"+path_to_file) + os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index dbc5e41..3b03b62 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -96,12 +96,12 @@ def create_ipython_config(self, hostname, notebook_password=None): else: passwd = notebook_password try: - sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) + sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) # TODO What's being returned here? + print "YEP" sha1pass = sha1pass_out[0].strip() except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, hostname, self.ssh_endpoint) raise e - sftp = self.ssh.open_sftp() notebook_config_file = sftp.file(remote_file_name, 'w+') notebook_config_file.write('\n'.join([ From 11fcc1f77f0473f7ae5f55fff111278249070fbd Mon Sep 17 00:00:00 2001 From: aviral26 Date: Mon, 6 Jun 2016 19:47:54 -0700 Subject: [PATCH 054/100] ipython controller starts up --- MolnsLib/Docker.py | 34 ++++++++++++++++++++++++++++------ MolnsLib/DockerProvider.py | 21 +++++---------------- MolnsLib/DockerSSH.py | 16 ++++++++++++---- MolnsLib/SSH.py | 13 +++++++++++++ MolnsLib/ssh_deploy.py | 18 ++---------------- 5 files changed, 60 insertions(+), 42 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 237f8c8..4cf0570 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -1,5 +1,6 @@ import logging import re +import time from molns_provider import ProviderBase from Constants import Constants from docker import Client @@ -24,9 +25,11 @@ def __init__(self): logging.basicConfig(level=logging.DEBUG) def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): - """Creates a new container. Returns the container ID. """ - print " Using image {0}".format(image_id) - container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True) + """Creates a new container with elevated privileges. Returns the container ID. """ + print "Using image {0}".format(image_id) + hc = self.client.create_host_config(privileged=True) + container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, + host_config=hc) return container.get("Id") def stop_containers(self, container_ids): @@ -68,12 +71,13 @@ def start_container(self, container_id): def execute_command(self, container_id, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" - print("CONTAINER: {0} COMMAND: {1}".format(container_id, command)) + run_command = "/bin/bash -c \"" + command + "\"" + print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) if self.start_container(container_id) is False: - print(" Could not start container.") + print("Could not start container.") return None try: - exec_instance = self.client.exec_create(container_id, "/bin/bash -c \"" + command + "\"") + exec_instance = self.client.exec_create(container_id, run_command) response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: @@ -128,8 +132,26 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): if self.start_container(container_id) is False: print("ERROR Could not start container.") return + + # Prepend file path with /home/ubuntu/. Very hack-y. Should be refined. + if not target_path_in_container.startswith("/home/ubuntu/"): + target_path_in_container = "/home/ubuntu/" + target_path_in_container + print("Unpacking archive to: " + target_path_in_container) if self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): print "Copied file successfully." else: print "Failed to copy." + + def get_container_ip_address(self, container_id): + self.start_container(container_id) + ins = self.client.inspect_container(container_id) + print "Waiting for an IP Address..." + ip_address = str(ins.get("NetworkSettings").get("IPAddress")) + while True: + ip_address = str(ins.get("NetworkSettings").get("IPAddress")) + time.sleep(3) + if ip_address.startswith("1") is True: + break + print "IP ADDRESS: " + ip_address + return ip_address diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index ebbefeb..e5dbf96 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -35,8 +35,9 @@ def start_instance(self, num=1): started_containers = [] for i in range(num): container_id = self.docker.create_container(self.provider.config["molns_image_name"]) - stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=None - , provider_id=self.provider.id, controller_id=self.id) + stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, + ip_address=self.docker.get_container_ip_address(container_id) + , provider_id=self.provider.id, controller_id=self.id) started_containers.append(stored_container) if num == 1: return started_containers[0] @@ -148,8 +149,8 @@ def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n - ipython \n apt-get -y install sudo \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu && echo "ubuntu ALL= - (ALL) NOPASSWD: ALL" >> /etc/sudoers\nWORKDIR /home/ubuntu\n''' + ipython \ \n sudo \ \n screen \ \n iptables \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\n + RUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers\nWORKDIR /home/ubuntu\n''' flag = False @@ -215,15 +216,3 @@ class DockerWorkerGroup(DockerController): ('num_vms', {'q': 'Number of containers in group', 'default': '1', 'ask': True}), ]) -# -# -# class DockerLocal(DockerBase): -# """ Provides a handle for a local Docker based ipython server. """ -# -# OBJ_NAME = 'DockerLocal' -# -# CONFIG_VARS = OrderedDict([ -# ]) -# -# def get_instance_status(self, instance): -# return self.docker.container_status(instance) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index c257302..74d23c3 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -1,6 +1,7 @@ import StringIO import tarfile import os +import re class DockerSSH(object): @@ -9,7 +10,13 @@ def __init__(self, docker): self.container_id = None def exec_command(self, command, verbose=True): - self.docker.execute_command(self.container_id, command) + cmd = re.sub("\"", "\\\"", command) # Escape all occurrences of ". + ret_val, response = self.docker.execute_command(self.container_id, cmd) + print "RESPONSE: " + response + return response + + def exec_multi_command(self, command, for_compatibility): + return self.exec_command(command) def open_sftp(self): return MockSFTP(self.docker, self.container_id) @@ -64,9 +71,10 @@ def close(self): tar.close() # Create temporary tar file to be copied into container. path_to_file = os.path.dirname(self.filename) - create_path_to_file_command = "sudo mkdir -p {0}".format(path_to_file) - self.docker.execute_command(self.container_id, create_path_to_file_command) + # create_path_to_file_command = "sudo mkdir -p {0}".format(path_to_file) + # self.docker.execute_command(self.container_id, create_path_to_file_command) + print "PATH TO FILE: " + path_to_file with open(temp_tar, mode='rb') as f: tar_file_bytes = f.read() - self.docker.put_archive(self.container_id, tar_file_bytes, "/home/ubuntu/"+path_to_file) + self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/SSH.py b/MolnsLib/SSH.py index 34f34f1..4a0ae95 100644 --- a/MolnsLib/SSH.py +++ b/MolnsLib/SSH.py @@ -45,6 +45,19 @@ def exec_command(self, command, verbose=True): print "FAILED......\t{0}\t{1}".format(command, e) raise SSHException("{0}\t{1}".format(command, e)) + def exec_multi_command(self, command, next_command): + try: + stdin, stdout, stderr = self.ssh.exec_command(command) + stdin.write(next_command) + stdin.flush() + status = stdout.channel.recv_exit_status() + if status != 0: + raise paramiko.SSHException("Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, stdout.read(), + stderr.read())) + except paramiko.SSHException as e: + print "FAILED......\t{0}\t{1}".format(command, e) + raise e + def open_sftp(self): return self.ssh.open_sftp() diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 3b03b62..f551be1 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -96,8 +96,7 @@ def create_ipython_config(self, hostname, notebook_password=None): else: passwd = notebook_password try: - sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) # TODO What's being returned here? - print "YEP" + sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) sha1pass = sha1pass_out[0].strip() except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, hostname, self.ssh_endpoint) @@ -219,19 +218,6 @@ def _put_ipython_engine_file(self, file_data): def exec_command_list_switch(self, command_list): for command in command_list: self.ssh.exec_command(command) - - def exec_multi_command(self, command, next_command): - try: - stdin, stdout, stderr = self.ssh.exec_command(command) - stdin.write(next_command) - stdin.flush() - status = stdout.channel.recv_exit_status() - if status != 0: - raise paramiko.SSHException("Exit Code: {0}\tSTDOUT: {1}\tSTDERR: {2}\n\n".format(status, stdout.read(), - stderr.read())) - except paramiko.SSHException as e: - print "FAILED......\t{0}\t{1}".format(command, e) - raise e def connect(self, instance, port=None): if port is None: @@ -259,7 +245,7 @@ def deploy_molns_webserver(self, instance): self.ssh.exec_command("sudo mkdir -p /usr/local/molns_webroot") self.ssh.exec_command("sudo chown ubuntu /usr/local/molns_webroot") self.ssh.exec_command("git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") - self.exec_multi_command("cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format(self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') + self.ssh.exec_multi_command("cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format(self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') self.ssh.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_WEBSERVER_PORT, self.DEFAULT_PRIVATE_WEBSERVER_PORT)) self.ssh.close() print "Deploying MOLNs webserver" From 9f2b98cb44dc7ba53434037ac5bd6bf919ca94c0 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Tue, 7 Jun 2016 18:12:56 -0700 Subject: [PATCH 055/100] add volume, set user id in container --- MolnsLib/DockerProvider.py | 18 ++++++++----- MolnsLib/Utils.py | 9 ++++++- MolnsLib/{DockerSSH.py => docker_ssh.py} | 0 MolnsLib/docker_test.py | 2 +- MolnsLib/molns_provider.py | 2 +- MolnsLib/{SSH.py => ssh.py} | 0 MolnsLib/ssh_deploy.py | 13 +++++++--- MolnsLib/{Docker.py => super_docker.py} | 33 ++++++++++++++---------- molns.py | 4 +++ 9 files changed, 54 insertions(+), 27 deletions(-) rename MolnsLib/{DockerSSH.py => docker_ssh.py} (100%) rename MolnsLib/{SSH.py => ssh.py} (100%) rename MolnsLib/{Docker.py => super_docker.py} (84%) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index e5dbf96..078da57 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -2,10 +2,10 @@ import logging import time import os -import Docker +import super_docker import installSoftware import tempfile -from DockerSSH import DockerSSH +from docker_ssh import DockerSSH from collections import OrderedDict from molns_provider import ProviderBase, ProviderException @@ -24,7 +24,7 @@ class DockerBase(ProviderBase): def __init__(self, name, config=None, config_dir=None, **kwargs): ProviderBase.__init__(self, name, config, config_dir, **kwargs) - self.docker = Docker.Docker() + self.docker = super_docker.Docker() self.ssh = DockerSSH(self.docker) def _get_container_status(self, container_id): @@ -147,10 +147,14 @@ def check_molns_image(self): def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ - dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n + import pwd + user_id = pwd.getpwnam(os.getlogin()).pw_uid + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n \n# Add user ubuntu. \nRUN useradd -u {0} -ms /bin/bash ubuntu \n + # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n - ipython \ \n sudo \ \n screen \ \n iptables \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\n - RUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers\nWORKDIR /home/ubuntu\n''' + ipython \ \n sudo \ \n screen \ \n iptables \n + RUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \n + WORKDIR /home/ubuntu\n'''.format(user_id) flag = False @@ -187,7 +191,7 @@ def _create_dockerfile(self, commands): def _preprocess(self, command): """ Filters out any sudos in the command, prepends "shell only" commands with '/bin/bash -c'. """ - for shell_command in Docker.Docker.shell_commands: + for shell_command in super_docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command command = command.replace(shell_command, replace_string) diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 956751f..65a4223 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -20,4 +20,11 @@ def __init__(self, name): self._logger = logger def get(self): - return self._logger \ No newline at end of file + return self._logger + +debug_docker = False + + +def print_d(print_this): + if debug_docker: + print print_this diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/docker_ssh.py similarity index 100% rename from MolnsLib/DockerSSH.py rename to MolnsLib/docker_ssh.py diff --git a/MolnsLib/docker_test.py b/MolnsLib/docker_test.py index e14f55a..aba6fb5 100644 --- a/MolnsLib/docker_test.py +++ b/MolnsLib/docker_test.py @@ -1,4 +1,4 @@ -from Docker import Docker +from super_docker import Docker import tempfile import installSoftware from DockerProvider import DockerProvider diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index 9302521..c62fed2 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -1,6 +1,6 @@ import os import collections -from SSH import SSH +from ssh import SSH class ProviderException(Exception): diff --git a/MolnsLib/SSH.py b/MolnsLib/ssh.py similarity index 100% rename from MolnsLib/SSH.py rename to MolnsLib/ssh.py diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index f551be1..f9cefab 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -1,15 +1,15 @@ import json import logging import os -import paramiko +import Utils import string import sys import time import uuid import webbrowser import urllib2 -from SSH import SSH -from DockerSSH import DockerSSH +from ssh import SSH +from docker_ssh import DockerSSH class SSHDeployException(Exception): @@ -97,7 +97,12 @@ def create_ipython_config(self, hostname, notebook_password=None): passwd = notebook_password try: sha1pass_out = self.ssh.exec_command(sha1cmd % passwd, verbose=False) - sha1pass = sha1pass_out[0].strip() + if isinstance(sha1pass_out, list): + sha1pass = sha1pass_out[0].strip() + else: + sha1pass = sha1pass_out.strip() + Utils.print_d("SHA1PASS_OUT: " + sha1pass_out) + Utils.print_d("SHA1PASS: " + sha1pass) except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, hostname, self.ssh_endpoint) raise e diff --git a/MolnsLib/Docker.py b/MolnsLib/super_docker.py similarity index 84% rename from MolnsLib/Docker.py rename to MolnsLib/super_docker.py index 4cf0570..eaff246 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/super_docker.py @@ -5,6 +5,7 @@ from Constants import Constants from docker import Client from docker.errors import NotFound, NullResource, APIError +import Utils class Docker: @@ -27,9 +28,14 @@ def __init__(self): def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): """Creates a new container with elevated privileges. Returns the container ID. """ print "Using image {0}".format(image_id) - hc = self.client.create_host_config(privileged=True) + hc = self.client.create_host_config(privileged=True, binds={ + '/home/aviral/Desktop/molns/volume_home_ubuntu/': { + 'bind': '/home/ubuntu', + 'mode': 'rw' + } + }) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, - host_config=hc) + host_config=hc, volumes=['/home/ubuntu']) return container.get("Id") def stop_containers(self, container_ids): @@ -61,11 +67,11 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - logging.debug(Docker.LOG_TAG + " Starting container " + container_id) + Utils.print_d(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: - logging.error(Docker.LOG_TAG + " Something went wrong while starting container.", e) + print (Docker.LOG_TAG + " Something went wrong while starting container.", e) return False return True @@ -81,7 +87,7 @@ def execute_command(self, container_id, command): response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: - logging.error(Docker.LOG_TAG + " Could not execute command.", e) + print (Docker.LOG_TAG + " Could not execute command.", e) return None def build_image(self, dockerfile): @@ -100,7 +106,7 @@ def build_image(self, dockerfile): # id. exp = r'[a-z0-9]{12}' image_id = re.findall(exp, str(last_line))[0] - print("Image ID: {0}".format(image_id)) + Utils.print_d("Image ID: {0}".format(image_id)) return image_id except (Docker.ImageBuildException, IndexError) as e: print("ERROR {0}".format(e)) @@ -111,7 +117,7 @@ def image_exists(self, image_id): for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: - print("Image exists: " + str(image)) + Utils.print_d("Image exists: " + str(image)) return True return False @@ -137,21 +143,22 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): if not target_path_in_container.startswith("/home/ubuntu/"): target_path_in_container = "/home/ubuntu/" + target_path_in_container - print("Unpacking archive to: " + target_path_in_container) + Utils.print_d("Unpacking archive to: " + target_path_in_container) if self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - print "Copied file successfully." + Utils.print_d("Copied file successfully.") else: - print "Failed to copy." + Utils.print_d("Failed to copy.") def get_container_ip_address(self, container_id): self.start_container(container_id) ins = self.client.inspect_container(container_id) - print "Waiting for an IP Address..." + Utils.print_d("Waiting for an IP Address...") ip_address = str(ins.get("NetworkSettings").get("IPAddress")) while True: ip_address = str(ins.get("NetworkSettings").get("IPAddress")) - time.sleep(3) + if ip_address == "": + time.sleep(3) if ip_address.startswith("1") is True: break - print "IP ADDRESS: " + ip_address + Utils.print_d("IP ADDRESS: " + ip_address) return ip_address diff --git a/molns.py b/molns.py index 1557644..e3e36b5 100755 --- a/molns.py +++ b/molns.py @@ -1511,6 +1511,10 @@ def parseArgs(): if arg_list[0].startswith('--debug'): print "Turning on Debugging output" logger.setLevel(logging.DEBUG) #for Debugging + if arg_list[0].startswith('--docker-debug'): + print "Turning on debugging for Docker" + import MolnsLib.Utils + MolnsLib.Utils.debug_docker = True arg_list = arg_list[1:] if len(arg_list) == 0 or arg_list[0] =='help' or arg_list[0] == '-h': From 73b07a3578434958b64117c4b88381743fdfc8c3 Mon Sep 17 00:00:00 2001 From: aviral26 Date: Wed, 8 Jun 2016 12:59:06 -0700 Subject: [PATCH 056/100] initial working version --- MolnsLib/Constants.py | 2 +- MolnsLib/DockerProvider.py | 8 +++--- MolnsLib/super_docker.py | 31 ++++++++++------------- MolnsLib/docker_test.py => docker_test.py | 8 +++--- 4 files changed, 22 insertions(+), 27 deletions(-) rename MolnsLib/docker_test.py => docker_test.py (94%) diff --git a/MolnsLib/Constants.py b/MolnsLib/Constants.py index 90bf290..4cef8bd 100644 --- a/MolnsLib/Constants.py +++ b/MolnsLib/Constants.py @@ -8,4 +8,4 @@ class Constants: DOCKERFILE_NAME = "dockerfile_" DOKCER_IMAGE_ID_LENGTH = 12 DOCKER_IMAGE_PREFIX = "aviralcse/docker-provider-" - DOCKER_PY_IMAGE_ID_PREFIX_LENGTH = 7 \ No newline at end of file + DOCKER_PY_IMAGE_ID_PREFIX_LENGTH = 7 diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 078da57..6aaa8a3 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -6,7 +6,6 @@ import installSoftware import tempfile from docker_ssh import DockerSSH - from collections import OrderedDict from molns_provider import ProviderBase, ProviderException @@ -94,7 +93,7 @@ class DockerProvider(DockerBase): ]) def get_config_credentials(self): - None + return None @staticmethod def __get_new_dockerfile_name(): @@ -104,12 +103,11 @@ def __get_new_dockerfile_name(): def check_ssh_key(self): """ Returns true. (Implementation does not use SSH.) """ - print "reached_check_ssh_key" + print "reached check_ssh_key" return True def create_ssh_key(self): """ Returns true. """ - # TODO print "reached create_ssh_key" ssh_key_dir = os.path.join(self.config_dir, self.name) fp = open(ssh_key_dir, 'w') @@ -177,7 +175,7 @@ def _create_dockerfile(self, commands): else: dockerfile += ''' && \ \n ''' + self._preprocess(entry) - dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu''' + dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu\nVOLUME /home/ubuntu''' dockerfile_file = DockerProvider.__get_new_dockerfile_name() with open(dockerfile_file, 'w') as Dockerfile: diff --git a/MolnsLib/super_docker.py b/MolnsLib/super_docker.py index eaff246..493e607 100644 --- a/MolnsLib/super_docker.py +++ b/MolnsLib/super_docker.py @@ -5,7 +5,6 @@ from Constants import Constants from docker import Client from docker.errors import NotFound, NullResource, APIError -import Utils class Docker: @@ -28,14 +27,9 @@ def __init__(self): def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): """Creates a new container with elevated privileges. Returns the container ID. """ print "Using image {0}".format(image_id) - hc = self.client.create_host_config(privileged=True, binds={ - '/home/aviral/Desktop/molns/volume_home_ubuntu/': { - 'bind': '/home/ubuntu', - 'mode': 'rw' - } - }) + hc = self.client.create_host_config(privileged=True) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, - host_config=hc, volumes=['/home/ubuntu']) + host_config=hc) return container.get("Id") def stop_containers(self, container_ids): @@ -67,7 +61,7 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - Utils.print_d(Docker.LOG_TAG + " Starting container " + container_id) + print(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: @@ -106,7 +100,7 @@ def build_image(self, dockerfile): # id. exp = r'[a-z0-9]{12}' image_id = re.findall(exp, str(last_line))[0] - Utils.print_d("Image ID: {0}".format(image_id)) + print("Image ID: {0}".format(image_id)) return image_id except (Docker.ImageBuildException, IndexError) as e: print("ERROR {0}".format(e)) @@ -117,7 +111,7 @@ def image_exists(self, image_id): for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: - Utils.print_d("Image exists: " + str(image)) + print("Image exists: " + str(image)) return True return False @@ -135,24 +129,27 @@ def terminate_container(self, container_id): self.client.remove_container(container_id) def put_archive(self, container_id, tar_file_bytes, target_path_in_container): + """ Copies and unpacks a given tarfile in the container at specified location. + Location must exist in container.""" if self.start_container(container_id) is False: print("ERROR Could not start container.") return - # Prepend file path with /home/ubuntu/. Very hack-y. Should be refined. + # Prepend file path with /home/ubuntu/. Very hack-y. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): target_path_in_container = "/home/ubuntu/" + target_path_in_container - Utils.print_d("Unpacking archive to: " + target_path_in_container) + print("Unpacking archive to: " + target_path_in_container) if self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - Utils.print_d("Copied file successfully.") + print("Copied file successfully.") else: - Utils.print_d("Failed to copy.") + print("Failed to copy.") def get_container_ip_address(self, container_id): + """ Returns the IP Address of given container.""" self.start_container(container_id) ins = self.client.inspect_container(container_id) - Utils.print_d("Waiting for an IP Address...") + print("Waiting for an IP Address...") ip_address = str(ins.get("NetworkSettings").get("IPAddress")) while True: ip_address = str(ins.get("NetworkSettings").get("IPAddress")) @@ -160,5 +157,5 @@ def get_container_ip_address(self, container_id): time.sleep(3) if ip_address.startswith("1") is True: break - Utils.print_d("IP ADDRESS: " + ip_address) + print("IP ADDRESS: " + ip_address) return ip_address diff --git a/MolnsLib/docker_test.py b/docker_test.py similarity index 94% rename from MolnsLib/docker_test.py rename to docker_test.py index aba6fb5..39f351b 100644 --- a/MolnsLib/docker_test.py +++ b/docker_test.py @@ -1,9 +1,9 @@ -from super_docker import Docker +from MolnsLib.super_docker import Docker import tempfile -import installSoftware -from DockerProvider import DockerProvider +import MolnsLib.installSoftware +from MolnsLib.DockerProvider import DockerProvider -commands = installSoftware.InstallSW.command_list +commands = MolnsLib.installSoftware.InstallSW.command_list docker = Docker() # From 47988ccea17bf0ceb7cec5fd837e65dcf2a9de2d Mon Sep 17 00:00:00 2001 From: aviral26 Date: Thu, 9 Jun 2016 13:47:11 -0700 Subject: [PATCH 057/100] refactor --- .gitignore | 4 +++- MolnsLib/DockerProvider.py | 8 ++++---- MolnsLib/docker_ssh.py | 2 +- MolnsLib/super_docker.py | 8 ++++---- docker_file/Dockerfile | 17 ----------------- 5 files changed, 12 insertions(+), 27 deletions(-) delete mode 100644 docker_file/Dockerfile diff --git a/.gitignore b/.gitignore index 5f0e906..61cbc0d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ molns_install.log .ec2_creds_molns .idea/ *.tar.gz +*.tar NOTES qsubscript /dockerfile_* -\#qsubscript\# \ No newline at end of file +\#qsubscript\# +docker_test.py \ No newline at end of file diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 6aaa8a3..7e2a767 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -103,12 +103,12 @@ def __get_new_dockerfile_name(): def check_ssh_key(self): """ Returns true. (Implementation does not use SSH.) """ - print "reached check_ssh_key" + # print "reached check_ssh_key" return True def create_ssh_key(self): """ Returns true. """ - print "reached create_ssh_key" + # print "reached create_ssh_key" ssh_key_dir = os.path.join(self.config_dir, self.name) fp = open(ssh_key_dir, 'w') fp.write("This is a dummy key.") @@ -127,10 +127,10 @@ def create_molns_image(self): """ Create a molns image, save it on localhost and return ID of created image. """ # create Dockerfile and build container. try: - print("Creating Dockerfile...") + # print("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) - print("Image created.") + # print("Image created.") return image_id except Exception as e: logging.exception(e) diff --git a/MolnsLib/docker_ssh.py b/MolnsLib/docker_ssh.py index 74d23c3..3b0efb7 100644 --- a/MolnsLib/docker_ssh.py +++ b/MolnsLib/docker_ssh.py @@ -12,7 +12,7 @@ def __init__(self, docker): def exec_command(self, command, verbose=True): cmd = re.sub("\"", "\\\"", command) # Escape all occurrences of ". ret_val, response = self.docker.execute_command(self.container_id, cmd) - print "RESPONSE: " + response + # print "RESPONSE: " + response return response def exec_multi_command(self, command, for_compatibility): diff --git a/MolnsLib/super_docker.py b/MolnsLib/super_docker.py index 493e607..2840daf 100644 --- a/MolnsLib/super_docker.py +++ b/MolnsLib/super_docker.py @@ -26,7 +26,7 @@ def __init__(self): def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): """Creates a new container with elevated privileges. Returns the container ID. """ - print "Using image {0}".format(image_id) + # print "Using image {0}".format(image_id) hc = self.client.create_host_config(privileged=True) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, host_config=hc) @@ -61,7 +61,7 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - print(Docker.LOG_TAG + " Starting container " + container_id) + # print(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: @@ -86,7 +86,7 @@ def execute_command(self, container_id, command): def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ - print("Building image...") + # print("Building image...") image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(self.build_count) last_line = "" try: @@ -111,7 +111,7 @@ def image_exists(self, image_id): for image in self.client.images(): some_id = image["Id"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: - print("Image exists: " + str(image)) + # print("Image exists: " + str(image)) return True return False diff --git a/docker_file/Dockerfile b/docker_file/Dockerfile deleted file mode 100644 index 27a284c..0000000 --- a/docker_file/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:14.04 - -RUN apt-get update - -# Set up base environment. -RUN apt-get install -yy \ - software-properties-common \ - python-software-properties \ - wget \ - git \ - ipython - -# Create and switch to new user. -RUN useradd -ms /bin/bash ubuntu -USER ubuntu -ENV HOME /home/ubuntu -WORKDIR /home/ubuntu \ No newline at end of file From dc3fa921c7e3d011bec7ebcdc81b1fc3ebdd99ff Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 27 Sep 2016 16:57:19 -0700 Subject: [PATCH 058/100] Fix userid bug. iPython server starts up now. --- .gitignore | 3 ++- MolnsLib/Constants.py | 2 +- MolnsLib/{super_docker.py => Docker.py} | 12 ++++------- MolnsLib/DockerProvider.py | 26 ++++++++++++------------ MolnsLib/{docker_ssh.py => DockerSSH.py} | 0 MolnsLib/ssh_deploy.py | 2 +- docker_test.py | 2 +- 7 files changed, 22 insertions(+), 25 deletions(-) rename MolnsLib/{super_docker.py => Docker.py} (93%) rename MolnsLib/{docker_ssh.py => DockerSSH.py} (100%) diff --git a/.gitignore b/.gitignore index 61cbc0d..5443c50 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ NOTES qsubscript /dockerfile_* \#qsubscript\# -docker_test.py \ No newline at end of file +docker_test.py +.idea/ diff --git a/MolnsLib/Constants.py b/MolnsLib/Constants.py index 4cef8bd..62770a3 100644 --- a/MolnsLib/Constants.py +++ b/MolnsLib/Constants.py @@ -1,5 +1,5 @@ class Constants: - LOGGING_DIRECTORY = "~/MOLNS_LOG"; + LOGGING_DIRECTORY = "~/MOLNS_LOG" DOCKER_BASE_URL = "unix://var/run/docker.sock" DOCKER_DEFAULT_IMAGE = "aviralcse/docker-provider" DOCKER_DEFAULT_PORT = '9000' diff --git a/MolnsLib/super_docker.py b/MolnsLib/Docker.py similarity index 93% rename from MolnsLib/super_docker.py rename to MolnsLib/Docker.py index 2840daf..52679ac 100644 --- a/MolnsLib/super_docker.py +++ b/MolnsLib/Docker.py @@ -61,7 +61,7 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - # print(Docker.LOG_TAG + " Starting container " + container_id) + print(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: @@ -135,21 +135,18 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): print("ERROR Could not start container.") return - # Prepend file path with /home/ubuntu/. Very hack-y. TODO Should be refined. + # Prepend file path with /home/ubuntu/. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): target_path_in_container = "/home/ubuntu/" + target_path_in_container - print("Unpacking archive to: " + target_path_in_container) - if self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - print("Copied file successfully.") - else: + # print("Unpacking archive to: " + target_path_in_container) + if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): print("Failed to copy.") def get_container_ip_address(self, container_id): """ Returns the IP Address of given container.""" self.start_container(container_id) ins = self.client.inspect_container(container_id) - print("Waiting for an IP Address...") ip_address = str(ins.get("NetworkSettings").get("IPAddress")) while True: ip_address = str(ins.get("NetworkSettings").get("IPAddress")) @@ -157,5 +154,4 @@ def get_container_ip_address(self, container_id): time.sleep(3) if ip_address.startswith("1") is True: break - print("IP ADDRESS: " + ip_address) return ip_address diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 7e2a767..1c67893 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -2,10 +2,10 @@ import logging import time import os -import super_docker +import Docker import installSoftware import tempfile -from docker_ssh import DockerSSH +from DockerSSH import DockerSSH from collections import OrderedDict from molns_provider import ProviderBase, ProviderException @@ -23,7 +23,7 @@ class DockerBase(ProviderBase): def __init__(self, name, config=None, config_dir=None, **kwargs): ProviderBase.__init__(self, name, config, config_dir, **kwargs) - self.docker = super_docker.Docker() + self.docker = Docker.Docker() self.ssh = DockerSSH(self.docker) def _get_container_status(self, container_id): @@ -87,7 +87,7 @@ class DockerProvider(DockerBase): ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', - {'q': 'Docker Security Group name', 'default': 'molns', 'ask': True}), # Unused. + {'q': 'Docker Security Group name', 'default': 'molns', 'ask': False}), # Unused. ('login_username', {'default': 'ubuntu', 'ask': False}) # Unused. ]) @@ -127,7 +127,7 @@ def create_molns_image(self): """ Create a molns image, save it on localhost and return ID of created image. """ # create Dockerfile and build container. try: - # print("Creating Dockerfile...") + print("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) # print("Image created.") @@ -146,13 +146,13 @@ def check_molns_image(self): def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ import pwd - user_id = pwd.getpwnam(os.getlogin()).pw_uid - dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n \n# Add user ubuntu. \nRUN useradd -u {0} -ms /bin/bash ubuntu \n - # Set up base environment.\nRUN apt-get install -yy \ \n - software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n - ipython \ \n sudo \ \n screen \ \n iptables \n - RUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \n - WORKDIR /home/ubuntu\n'''.format(user_id) + + user_id = pwd.getpwnam(os.environ['SUDO_USER']).pw_uid + dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n\n# Add user ubuntu.\nRUN useradd -u {0} -ms /bin/bash ubuntu\n + # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ + \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \ \n sudo \ + \n screen \ \n iptables \nRUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + \nWORKDIR /home/ubuntu\n'''.format(user_id) flag = False @@ -189,7 +189,7 @@ def _create_dockerfile(self, commands): def _preprocess(self, command): """ Filters out any sudos in the command, prepends "shell only" commands with '/bin/bash -c'. """ - for shell_command in super_docker.Docker.shell_commands: + for shell_command in Docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command command = command.replace(shell_command, replace_string) diff --git a/MolnsLib/docker_ssh.py b/MolnsLib/DockerSSH.py similarity index 100% rename from MolnsLib/docker_ssh.py rename to MolnsLib/DockerSSH.py diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index f9cefab..0de9291 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -9,7 +9,7 @@ import webbrowser import urllib2 from ssh import SSH -from docker_ssh import DockerSSH +from DockerSSH import DockerSSH class SSHDeployException(Exception): diff --git a/docker_test.py b/docker_test.py index 39f351b..f75d735 100644 --- a/docker_test.py +++ b/docker_test.py @@ -1,4 +1,4 @@ -from MolnsLib.super_docker import Docker +from MolnsLib.Docker import Docker import tempfile import MolnsLib.installSoftware from MolnsLib.DockerProvider import DockerProvider From 76e7b106885b89eaa003921dd3b0e92f4b725d99 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 28 Sep 2016 17:26:31 -0700 Subject: [PATCH 059/100] Fix typo introduced by PyCharm --- MolnsLib/DockerProvider.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 1c67893..c1e9b76 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -83,7 +83,7 @@ class DockerProvider(DockerBase): {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'Local MOLNs image ID to use', 'default': '', 'ask': True}), + {'q': 'Local MOLNs Docker image ID to use (Leave blank to build new image)', 'default': '', 'ask': True}), ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', @@ -149,9 +149,9 @@ def _create_dockerfile(self, commands): user_id = pwd.getpwnam(os.environ['SUDO_USER']).pw_uid dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n\n# Add user ubuntu.\nRUN useradd -u {0} -ms /bin/bash ubuntu\n - # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ - \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \ \n sudo \ - \n screen \ \n iptables \nRUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n + python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \ \n sudo \ \n + screen \ \n iptables \nRUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers \nWORKDIR /home/ubuntu\n'''.format(user_id) flag = False From fda120bc04d94abcfa5f310b4ebbca0f97edfce5 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Thu, 29 Sep 2016 12:02:03 -0700 Subject: [PATCH 060/100] remove debugging print statements. --- MolnsLib/Docker.py | 4 ++-- NOTES | 54 ---------------------------------------------- README.md | 4 ++-- 3 files changed, 4 insertions(+), 58 deletions(-) delete mode 100644 NOTES diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 52679ac..5f22e75 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -61,7 +61,7 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - print(Docker.LOG_TAG + " Starting container " + container_id) + # print(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: @@ -72,7 +72,7 @@ def start_container(self, container_id): def execute_command(self, container_id, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" run_command = "/bin/bash -c \"" + command + "\"" - print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) + # print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) if self.start_container(container_id) is False: print("Could not start container.") return None diff --git a/NOTES b/NOTES deleted file mode 100644 index 04516c1..0000000 --- a/NOTES +++ /dev/null @@ -1,54 +0,0 @@ -1. DockerProvider - line 167, 176, 222 - - -pbs qsub - scheduler -comet, htcondor - -knot csc ucsb - stochss backend - -molns azure - -openstack installation on campus cluster - -gillespy - - - -1. sudo rm -rf /usr/local/pyurdme;sudo mkdir -p /usr/local/pyurdme;sudo chown ubuntu /usr/local/pyurdme - rm: invalid option -- 'p' -------no user ubuntu in container - -2. cp MOLNS_notebooks/*.ipynb .;rm -rf MOLNS_notebooks; - cp: target 'MOLNS_notebooks;' is not a directory - - - 3. sudo add-apt-repository -y ppa:fenics-packages/fenics - sudo: add-apt-repository: command not found - - -4. cd ipython && git checkout 3.0.0-molns_fixes && python setup.py submodule && sudo python setup.py install - rpc error: code = 2 desc = "oci runtime error: exec failed: exec: \"cd\": executable file not found in $PATH" ---------cd is a built in shell command - -5. Command: sudo gpasswd -a ubuntu fuse - gpasswd: user 'ubuntu' does not exist ------------ - - 6. ipython executable not found in path - - 7. wget not found in path - - - -DEBUG:root:Docker Container ID: 1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e Command: python -c "from IPython.external import mathjax; mathjax.install_mathjax(tag='2.2.0')" -DEBUG:requests.packages.urllib3.connectionpool:"POST /v1.22/containers/1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e/exec HTTP/1.1" 201 74 -DEBUG:requests.packages.urllib3.connectionpool:"POST /v1.22/exec/e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978/start HTTP/1.1" 200 None -DEBUG:requests.packages.urllib3.connectionpool:"GET /v1.22/exec/e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978/json HTTP/1.1" 200 458 -RETURN VALUE: {u'OpenStderr': True, u'OpenStdout': True, u'ContainerID': u'1ad1360d83bbec1b014c7bf3f056bc526e38c9b31fb4f36259c9f8aa29f9714e', u'DetachKeys': u'', u'CanRemove': False, u'Running': False, u'ProcessConfig': {u'tty': False, u'entrypoint': u'/bin/bash', u'arguments': [u'-c', u'python -c from', u'IPython.external', u'import', u'mathjax;', u'mathjax.install_mathjax(tag=2.2.0)'], u'privileged': False}, u'ExitCode': 1, u'ID': u'e04dc8a240fb3c35a117833239b1712fb23aaaa08fb8a216cdfd2008bd776978', u'OpenStdin': False} -ERROR -RESPONSE: File "", line 1 - from - ^ -SyntaxError: invalid syntax - - diff --git a/README.md b/README.md index 0cfee6c..55a1cab 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ MOLNs is a cloud appliance that will set up, start and manage a virtual platform for scalable, distributed computational experiments using (spatial) stochastic simulation software such as PyURDME (www.pyurdme.org) and StochKit/Gillespy (www.github.com/Gillespy/gillespy). In addition, MOLNs by default makes FEniCS/Dolfin available as-a Service. -Since MOLNs will configure and manage a virtual IPython Cluster (with a Notebook frontend), with Numpy, SciPy and Ipython Parallel enabled, it can also be useful for general contextualization and management of dynamic, cloud-agnostic (supports EC2 and OpenStack-based clouds) virtual IPython environments, even if you are not into spatial stochstic simulations in systems biology. +Since MOLNs will configure and manage a virtual IPython Cluster (with a Notebook frontend), with Numpy, SciPy and Ipython Parallel enabled, it can also be useful for general contextualization and management of dynamic, cloud-agnostic (supports EC2 and OpenStack-based clouds) virtual IPython environments, even if you are not into spatial stochastic simulations in systems biology. -Note: MOLNs is currenly compatible only with 'EC2-Classic', we are working on supporting Amazon VPC. +Note: MOLNs is currently compatible only with 'EC2-Classic', we are working on supporting Amazon VPC. ### Prerequisites ### To use MOLNs, you need valid credentials to an OpenStack cloud, Amazon Elastic Compute Cloud (EC2) or HP Helion public cloud. You also need Python, and the following packages: From 2d9e5ee8703e1b1819d8cb1ed8d92f1dc525bbc4 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Thu, 29 Sep 2016 12:05:29 -0700 Subject: [PATCH 061/100] make warning message clearer --- MolnsLib/DockerSSH.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 3b0efb7..55436f2 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -53,7 +53,7 @@ def __init__(self, filename, flag, docker, container_id): if flag is 'w': self.flag = flag else: - print("WARNING Unrecognized file mode.") + print("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) def write(self, write_this): self.file_contents += write_this @@ -73,7 +73,7 @@ def close(self): path_to_file = os.path.dirname(self.filename) # create_path_to_file_command = "sudo mkdir -p {0}".format(path_to_file) # self.docker.execute_command(self.container_id, create_path_to_file_command) - print "PATH TO FILE: " + path_to_file + # print "PATH TO FILE: " + path_to_file with open(temp_tar, mode='rb') as f: tar_file_bytes = f.read() self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) From fd2b45282677c7630996af27ccb09b9d5f67e347 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 4 Oct 2016 16:31:50 -0700 Subject: [PATCH 062/100] bind host port 8080 --- MolnsLib/Docker.py | 4 ++-- MolnsLib/DockerProvider.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 5f22e75..ac5c98c 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -27,8 +27,8 @@ def __init__(self): def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): """Creates a new container with elevated privileges. Returns the container ID. """ # print "Using image {0}".format(image_id) - hc = self.client.create_host_config(privileged=True) - container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, + hc = self.client.create_host_config(privileged=True, port_bindings={80: 8080}) + container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, ports=[80], host_config=hc) return container.get("Id") diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index c1e9b76..7288a15 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -83,7 +83,7 @@ class DockerProvider(DockerBase): {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'Local MOLNs Docker image ID to use (Leave blank to build new image)', 'default': '', 'ask': True}), + {'q': 'Local MOLNs image (Docker image ID) to use ', 'default': None, 'ask': True}), ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', @@ -175,7 +175,7 @@ def _create_dockerfile(self, commands): else: dockerfile += ''' && \ \n ''' + self._preprocess(entry) - dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu\nVOLUME /home/ubuntu''' + dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu\nVOLUME /home/ubuntu\n''' dockerfile_file = DockerProvider.__get_new_dockerfile_name() with open(dockerfile_file, 'w') as Dockerfile: From 6004148776957d58de6994cd562b854d300bb8d7 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 4 Oct 2016 22:51:58 -0700 Subject: [PATCH 063/100] update to latest ode solver --- MolnsLib/installSoftware.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index c518482..c2faf7e 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -85,21 +85,20 @@ class InstallSW: "cd /usr/local/ && git clone https://github.com/StochSS/stochkit.git StochKit", "cd /usr/local/StochKit && ./install.sh", - #"sudo rm -rf /usr/local/ode-1.0.3;sudo mkdir -p /usr/local/ode-1.0.3/;sudo chown ubuntu /usr/local/ode-1.0.3", - "wget https://github.com/StochSS/stochss/blob/master/ode-1.0.3.tgz?raw=true -q -O /tmp/ode.tgz", - #"cd /usr/local/ && tar -xzf /tmp/ode.tgz", + #"wget https://github.com/StochSS/stochss/blob/master/ode-1.0.4.tgz?raw=true -q -O /tmp/ode.tgz", + "wget https://github.com/StochSS/StochKit_ode/archive/master.tar.gz?raw=true -q -O /tmp/ode.tgz" "cd /tmp && tar -xzf /tmp/ode.tgz", - "sudo mv /tmp/ode-1.0.3 /usr/local/", + "sudo mv /tmp/StochKit_ode-master /usr/local/ode", "rm /tmp/ode.tgz", - "cd /usr/local/ode-1.0.3/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", - "cd /usr/local/ode-1.0.3/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode-1.0.3/ make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode/cvodes/ && tar -xzf \"cvodes-2.7.0.tar.gz\"", + "cd /usr/local/ode/cvodes/cvodes-2.7.0/ && ./configure --prefix=\"/usr/local/ode/cvodes/cvodes-2.7.0/cvodes\" 1>stdout.log 2>stderr.log", + "cd /usr/local/ode/cvodes/cvodes-2.7.0/ && make 1>stdout.log 2>stderr.log", + "cd /usr/local/ode/cvodes/cvodes-2.7.0/ && make install 1>stdout.log 2>stderr.log", + "cd /usr/local/ode/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode/ make 1>stdout.log 2>stderr.log", "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", - "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode-1.0.3/ python setup.py install" + "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode/ python setup.py install" ], From 8daa41d21627133e1cbc1b151e4b582bf80506b4 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Thu, 6 Oct 2016 12:01:03 -0700 Subject: [PATCH 064/100] Fixes to make developing provider types in db not cause failures in other branches --- MolnsLib/molns_datastore.py | 5 ++++- molns.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 92dc886..87bfdf9 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -137,7 +137,8 @@ def get_provider_handle(kind, ptype): if kind not in valid_handles: raise DatastoreException("Unknown kind {0}".format(kind)) if ptype not in VALID_PROVIDER_TYPES: - raise DatastoreException("Unknown {1} type {0}".format(ptype, kind)) + return None + #raise DatastoreException("Unknown {1} type {0}".format(ptype, kind)) cls_name = "{0}{1}".format(ptype, kind) pkg_name = "MolnsLib.{0}Provider".format(ptype) if pkg_name not in sys.modules: @@ -286,6 +287,8 @@ def _get_object_data(self, d_handle, kind, ptype, p): p_handle = get_provider_handle(kind, ptype) #logging.debug("{2}(name={0}, data={1})".format(name,data,p_handle)) + if p_handle is None: + return None ret = p_handle(name=p.name, config=data, config_dir=self.config_dir) ret.id = p.id ret.datastore = self diff --git a/molns.py b/molns.py index 6227b3c..0690387 100755 --- a/molns.py +++ b/molns.py @@ -411,7 +411,10 @@ def status_controller(cls, args, config): if len(instance_list) > 0: table_data = [] for i in instance_list: - provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + provider_obj = config.get_object_by_id(i.provider_id, 'Provider') + if provider_obj is None: + continue + provider_name = provider_obj.name controller_name = config.get_object_by_id(i.controller_id, 'Controller').name if i.worker_group_id is not None: worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name From c9562aa2414a82fe45edd4d1f3c3523a14fb190e Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 12 Oct 2016 17:25:52 -0700 Subject: [PATCH 065/100] refactor' --- .gitignore | 2 +- MolnsLib/Docker.py | 39 +-- MolnsLib/DockerProvider.py | 31 ++- MolnsLib/DockerSSH.py | 20 +- MolnsLib/Utils.py | 36 +-- MolnsLib/molns_datastore.py | 1 - MolnsLib/ssh_deploy.py | 3 +- molns.py | 464 +++++++++++++++++++----------------- 8 files changed, 309 insertions(+), 287 deletions(-) diff --git a/.gitignore b/.gitignore index 5443c50..bb640f0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ molns_install.log .idea/ *.tar.gz *.tar -NOTES +notes qsubscript /dockerfile_* \#qsubscript\# diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index ac5c98c..1ddae7a 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -1,6 +1,8 @@ import logging import re import time + +from MolnsLib.Utils import Log from molns_provider import ProviderBase from Constants import Constants from docker import Client @@ -16,18 +18,20 @@ class Docker: shell_commands = ["source"] class ImageBuildException(Exception): - def __init__(self): - super("Something went wrong while building docker container image.") + def __init__(self, message=None): + super("Something went wrong while building docker container image.\n{0}".format(message)) def __init__(self): self.client = Client(base_url=Constants.DOCKER_BASE_URL) self.build_count = 0 logging.basicConfig(level=logging.DEBUG) - def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE): - """Creates a new container with elevated privileges. Returns the container ID. """ - # print "Using image {0}".format(image_id) - hc = self.client.create_host_config(privileged=True, port_bindings={80: 8080}) + def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE, port_bindings={80: 8080}): + """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container + to 8080 of locahost by default""" + + Log.write_log("Using image {0}".format(image_id)) + hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, ports=[80], host_config=hc) return container.get("Id") @@ -61,11 +65,11 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - # print(Docker.LOG_TAG + " Starting container " + container_id) + Log.write_log(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: - print (Docker.LOG_TAG + " Something went wrong while starting container.", e) + Log.write_log(Docker.LOG_TAG + " Something went wrong while starting container.", e) return False return True @@ -74,19 +78,19 @@ def execute_command(self, container_id, command): run_command = "/bin/bash -c \"" + command + "\"" # print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) if self.start_container(container_id) is False: - print("Could not start container.") + Log.write_log(Docker.LOG_TAG + "Could not start container.") return None try: exec_instance = self.client.exec_create(container_id, run_command) response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: - print (Docker.LOG_TAG + " Could not execute command.", e) + Log.write_log(Docker.LOG_TAG + " Could not execute command.", e) return None def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ - # print("Building image...") + Log.write_log(Docker.LOG_TAG + "Building image...") image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(self.build_count) last_line = "" try: @@ -100,8 +104,8 @@ def build_image(self, dockerfile): # id. exp = r'[a-z0-9]{12}' image_id = re.findall(exp, str(last_line))[0] - print("Image ID: {0}".format(image_id)) - return image_id + Log.write_log(Docker.LOG_TAG + "Image ID: {0}".format(image_id)) + return [image_id, image_tag] except (Docker.ImageBuildException, IndexError) as e: print("ERROR {0}".format(e)) return None @@ -110,8 +114,10 @@ def image_exists(self, image_id): """Checks if an image with the given ID exists locally.""" for image in self.client.images(): some_id = image["Id"] + some_tags = image["RepoTags"] if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: - # print("Image exists: " + str(image)) + return True + if image_id in some_tags: return True return False @@ -132,16 +138,15 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): """ Copies and unpacks a given tarfile in the container at specified location. Location must exist in container.""" if self.start_container(container_id) is False: - print("ERROR Could not start container.") + Log.write_log(Docker.LOG_TAG + "ERROR Could not start container.") return # Prepend file path with /home/ubuntu/. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): target_path_in_container = "/home/ubuntu/" + target_path_in_container - # print("Unpacking archive to: " + target_path_in_container) if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - print("Failed to copy.") + Log.write_log(Docker.LOG_TAG + "Failed to copy.") def get_container_ip_address(self, container_id): """ Returns the IP Address of given container.""" diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 7288a15..c4d49bc 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -30,7 +30,7 @@ def _get_container_status(self, container_id): self.docker.container_status(container_id) def start_instance(self, num=1): - """ Start given number (or 1) containers. """ + """ Start given number of (or 1) containers. """ started_containers = [] for i in range(num): container_id = self.docker.create_container(self.provider.config["molns_image_name"]) @@ -83,7 +83,7 @@ class DockerProvider(DockerBase): {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', - {'q': 'Local MOLNs image (Docker image ID) to use ', 'default': None, 'ask': True}), + {'q': 'Local MOLNs image (Docker image ID or image tag) to use ', 'default': None, 'ask': True}), ('key_name', {'q': 'Docker Key Pair name', 'default': "docker-default", 'ask': False}), # Unused. ('group_name', @@ -103,16 +103,13 @@ def __get_new_dockerfile_name(): def check_ssh_key(self): """ Returns true. (Implementation does not use SSH.) """ - # print "reached check_ssh_key" return True def create_ssh_key(self): """ Returns true. """ - # print "reached create_ssh_key" ssh_key_dir = os.path.join(self.config_dir, self.name) - fp = open(ssh_key_dir, 'w') - fp.write("This is a dummy key.") - fp.close() + with open(ssh_key_dir, 'w') as fp: + fp.write("This is a dummy key.") os.chmod(ssh_key_dir, 0o600) def check_security_group(self): @@ -124,22 +121,24 @@ def create_seurity_group(self): return True def create_molns_image(self): - """ Create a molns image, save it on localhost and return ID of created image. """ - # create Dockerfile and build container. + """ Create a molns image, save it on localhost and return ID of created image. Because of the implementation in + a higher layer, this forces MOLNs to recognise docker images based only on their IDs and NOT names.""" + dockerfile = None try: - print("Creating Dockerfile...") dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) - # print("Image created.") return image_id except Exception as e: logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) + finally: + if dockerfile is not None: + os.remove(dockerfile) def check_molns_image(self): """ Check if the molns image exists. """ - if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None and self.config[ - 'molns_image_name'] != '': + if 'molns_image_name' in self.config and self.config['molns_image_name'] is not None \ + and self.config['molns_image_name'] != '': return self.docker.image_exists(self.config['molns_image_name']) return False @@ -180,15 +179,15 @@ def _create_dockerfile(self, commands): dockerfile_file = DockerProvider.__get_new_dockerfile_name() with open(dockerfile_file, 'w') as Dockerfile: Dockerfile.write(dockerfile) - print("Using as dockerfile : " + dockerfile_file) named_dockerfile = tempfile.NamedTemporaryFile() named_dockerfile.write(dockerfile) named_dockerfile.seek(0) return named_dockerfile - def _preprocess(self, command): - """ Filters out any sudos in the command, prepends "shell only" commands with '/bin/bash -c'. """ + @staticmethod + def _preprocess(command): + """ Filters out any "sudo" in the command, prepends "shell only" commands with '/bin/bash -c'. """ for shell_command in Docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 55436f2..9f13adf 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -4,24 +4,27 @@ import re +# "unused" arguments to some methods are added to maintain compatibility with existing upper level APIs. +from MolnsLib.Utils import Log + + class DockerSSH(object): def __init__(self, docker): self.docker = docker self.container_id = None - def exec_command(self, command, verbose=True): + def exec_command(self, command, unused): cmd = re.sub("\"", "\\\"", command) # Escape all occurrences of ". ret_val, response = self.docker.execute_command(self.container_id, cmd) - # print "RESPONSE: " + response return response - def exec_multi_command(self, command, for_compatibility): + def exec_multi_command(self, command, unused): return self.exec_command(command) def open_sftp(self): return MockSFTP(self.docker, self.container_id) - def connect(self, instance, port=None, username=None, key_filename=None): + def connect(self, instance, unused1, unused2, unused3): self.container_id = instance.provider_instance_identifier def close(self): @@ -53,7 +56,7 @@ def __init__(self, filename, flag, docker, container_id): if flag is 'w': self.flag = flag else: - print("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) + Log.write_log("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) def write(self, write_this): self.file_contents += write_this @@ -68,13 +71,12 @@ def close(self): tar_file_info = tarfile.TarInfo(name=os.path.basename(self.filename)) tar_file_info.size = len(string.buf) tar.addfile(tarinfo=tar_file_info, fileobj=string) - tar.close() # Create temporary tar file to be copied into container. + tar.close() path_to_file = os.path.dirname(self.filename) - # create_path_to_file_command = "sudo mkdir -p {0}".format(path_to_file) - # self.docker.execute_command(self.container_id, create_path_to_file_command) - # print "PATH TO FILE: " + path_to_file + with open(temp_tar, mode='rb') as f: tar_file_bytes = f.read() + self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 65a4223..2ac0432 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -1,30 +1,10 @@ -import os -import logging +class Log: + verbose = True -from MolnsLib.Constants import Constants + def __init__(self): + pass - -class Logger(object): - - def __init__(self, name): - name = name.replace('.log','') - logger = logging.getLogger('MOLNS.%s' % name) - logger.setLevel(logging.DEBUG) - if not logger.handlers: - file_name = os.path.join(Constants.LOGGING_DIRECTORY, '%s.log' % name) - handler = logging.FileHandler(file_name) - formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s %(message)s') - handler.setFormatter(formatter) - handler.setLevel(logging.DEBUG) - logger.addHandler(handler) - self._logger = logger - - def get(self): - return self._logger - -debug_docker = False - - -def print_d(print_this): - if debug_docker: - print print_this + @staticmethod + def write_log(message): + if Log.verbose: + print message diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index e901743..b282d8c 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -8,7 +8,6 @@ import logging import sys ############################################################# -#VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Rackspace'] VALID_PROVIDER_TYPES = ['OpenStack', 'EC2', 'Eucalyptus', 'Docker'] ############################################################# #### SCHEMA ################################################# diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 0de9291..5010e3e 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -331,8 +331,7 @@ def deploy_stochss(self, ip_address, port=1443): def deploy_ipython_controller(self, instance, notebook_password=None): ip_address = instance.ip_address - controller_hostname = '' - engine_file_data = '' + try: print "{0}:{1}".format(ip_address, self.ssh_endpoint) self.connect(instance, self.ssh_endpoint) diff --git a/molns.py b/molns.py index e3e36b5..2ab5fe6 100755 --- a/molns.py +++ b/molns.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import os import sys + +from MolnsLib.Utils import Log from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES, get_provider_handle from MolnsLib.molns_provider import ProviderException import subprocess @@ -10,7 +12,7 @@ import logging logger = logging.getLogger() -#logger.setLevel(logging.INFO) #for Debugging +# logger.setLevel(logging.INFO) #for Debugging logger.setLevel(logging.CRITICAL) @@ -30,7 +32,6 @@ def __str__(self): ############################################### class MOLNSbase(): - @classmethod def merge_config(cls, obj, config): for key, conf, value in obj.get_config_vars(): @@ -64,9 +65,10 @@ def _get_workerobj(cls, args, config): worker_obj = config.get_object(name=worker_name, kind='WorkerGroup') except DatastoreException: worker_obj = None - #logging.debug("controller_obj {0}".format(controller_obj)) + # logging.debug("controller_obj {0}".format(controller_obj)) if worker_obj is None: - print "worker group '{0}' is not initialized, use 'molns worker setup {0}' to initialize the controller.".format(worker_name) + print "worker group '{0}' is not initialized, use 'molns worker setup {0}' to initialize the controller.".format( + worker_name) else: print "No worker name specified, please specify a name" return worker_obj @@ -83,9 +85,11 @@ def _get_controllerobj(cls, args, config): controller_obj = config.get_object(name=controller_name, kind='Controller') except DatastoreException: controller_obj = None - #logging.debug("controller_obj {0}".format(controller_obj)) + # logging.debug("controller_obj {0}".format(controller_obj)) if controller_obj is None: - raise MOLNSException("controller '{0}' is not initialized, use 'molns controller setup {0}' to initialize the controller.".format(controller_name)) + raise MOLNSException( + "controller '{0}' is not initialized, use 'molns controller setup {0}' to initialize the controller.".format( + controller_name)) return controller_obj @@ -94,8 +98,8 @@ class MOLNSController(MOLNSbase): def controller_export(cls, args, config): """ Export the configuration of a controller. """ if len(args) < 1: - raise MOLNSException("USAGE: molns controller export name [Filename]\n"\ - "\tExport the data from the controller with the given name.") + raise MOLNSException("USAGE: molns controller export name [Filename]\n" \ + "\tExport the data from the controller with the given name.") controller_name = args[0] if len(args) > 1: filename = args[1] @@ -118,8 +122,8 @@ def controller_import(cls, args, config, json_data=None): """ Import the configuration of a controller. """ if json_data is None: if len(args) < 1: - raise MOLNSException("USAGE: molns controller import [Filename.json]\n"\ - "\Import the data from the controller with the given name.") + raise MOLNSException("USAGE: molns controller import [Filename.json]\n" \ + "\Import the data from the controller with the given name.") filename = args[0] with open(filename) as fd: data = json.load(fd) @@ -135,14 +139,17 @@ def controller_import(cls, args, config, json_data=None): controller_obj = config.get_object(controller_name, kind='Controller') msg += "Found existing controller\n" if controller_obj.provider.name != provider_obj.name: - raise MOLNSException("Import data has provider '{0}'. Controller {1} exists with provider {2}. provider conversion is not possible.".format(data['provider_name'], controller_obj.name, controller_obj.provider.name)) + raise MOLNSException( + "Import data has provider '{0}'. Controller {1} exists with provider {2}. provider conversion is not possible.".format( + data['provider_name'], controller_obj.name, controller_obj.provider.name)) except DatastoreException as e: - controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', provider_id=provider_obj.id) + controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', + provider_id=provider_obj.id) msg += "Creating new controller\n" cls.merge_config(controller_obj, data['config']) config.save_object(controller_obj, kind='Controller') msg += "Controller data imported\n" - return {'msg':msg} + return {'msg': msg} @classmethod def controller_get_config(cls, name=None, provider_type=None, config=None): @@ -168,8 +175,8 @@ def controller_get_config(cls, name=None, provider_type=None, config=None): if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) - p_hand = get_provider_handle('Controller',provider_type) - obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + p_hand = get_provider_handle('Controller', provider_type) + obj = p_hand('__tmp__', data={}, config_dir=config.config_dir) if obj is None: raise MOLNSException("Controller {0} not found".format(name)) @@ -195,16 +202,17 @@ def controller_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ - 'question':question, - 'key':key, + 'question': question, + 'key': key, 'value': myval, - 'type':'string' + 'type': 'string' }) return ret @classmethod def setup_controller(cls, args, config): - """Setup a controller. Set the provider configuration for the head node. Use 'worker setup' to set the configuration for worker nodes + """Setup a controller. Set the provider configuration for the head node. + Use 'worker setup' to set the configuration for worker nodes. """ logging.debug("MOLNSController.setup_controller(config={0})".format(config)) # name @@ -219,18 +227,20 @@ def setup_controller(cls, args, config): # provider providers = config.list_objects(kind='Provider') if len(providers) == 0: - print "No providers configured, please configure one ('molns provider setup') before initializing controller." + print "No providers configured, " \ + "please configure one ('molns provider setup') before initializing controller." return print "Select a provider:" - for n,p in enumerate(providers): - print "\t[{0}] {1}".format(n,p.name) - provider_ndx = int(raw_input_default("enter the number of provider:", default='0')) + for n, p in enumerate(providers): + print "\t[{0}] {1}".format(n, p.name) + provider_ndx = int(raw_input_default("Enter the number of provider:", default='0')) provider_id = providers[provider_ndx].id provider_obj = config.get_object(name=providers[provider_ndx].name, kind='Provider') logging.debug("using provider {0}".format(provider_obj)) # create object try: - controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', provider_id=provider_id) + controller_obj = config.create_object(ptype=provider_obj.type, name=controller_name, kind='Controller', + provider_id=provider_id) except DatastoreException as e: print e return @@ -242,7 +252,7 @@ def list_controller(cls, args, config): """ List all the currently configured controllers.""" controllers = config.list_objects(kind='Controller') if len(controllers) == 0: - return {'msg':"No controllers configured"} + return {'msg': "No controllers configured"} else: table_data = [] for c in controllers: @@ -252,19 +262,19 @@ def list_controller(cls, args, config): except DatastoreException as e: provider_name = 'ERROR: {0}'.format(e) table_data.append([c.name, provider_name]) - return {'type':'table','column_names':['name', 'provider'], 'data':table_data} + return {'type': 'table', 'column_names': ['name', 'provider'], 'data': table_data} @classmethod def show_controller(cls, args, config): """ Show all the details of a controller config. """ if len(args) == 0: raise MOLNSException("USAGE: molns controller show name") - return {'msg':str(config.get_object(name=args[0], kind='Controller'))} + return {'msg': str(config.get_object(name=args[0], kind='Controller'))} @classmethod def delete_controller(cls, args, config): """ Delete a controller config. """ - #print "MOLNSProvider.delete_provider(args={0}, config={1})".format(args, config) + # print "MOLNSProvider.delete_provider(args={0}, config={1})".format(args, config) if len(args) == 0: raise MOLNSException("USAGE: molns cluser delete name") config.delete_object(name=args[0], kind='Controller') @@ -277,7 +287,7 @@ def ssh_controller(cls, args, config): if controller_obj is None: return # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - #logging.debug("instance_list={0}".format(instance_list)) + # logging.debug("instance_list={0}".format(instance_list)) # Check if they are running ip = None if len(instance_list) > 0: @@ -289,9 +299,10 @@ def ssh_controller(cls, args, config): if ip is None: print "No active instance for this controller" return - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) - cmd = ['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)] + # print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) + # os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) + cmd = ['/usr/bin/ssh', '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null', '-i', + controller_obj.provider.sshkeyfilename(), 'ubuntu@{0}'.format(ip)] print " ".join(cmd) subprocess.call(cmd) print "SSH process completed" @@ -304,7 +315,7 @@ def upload_controller(cls, args, config): if controller_obj is None: return # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - #logging.debug("instance_list={0}".format(instance_list)) + # logging.debug("instance_list={0}".format(instance_list)) # Check if they are running ip = None if len(instance_list) > 0: @@ -316,9 +327,10 @@ def upload_controller(cls, args, config): if ip is None: print "No active instance for this controller" return - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) - cmd = ['/usr/bin/scp','-r','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] + # print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) + # os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) + cmd = ['/usr/bin/scp', '-r', '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null', '-i', + controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] print " ".join(cmd) subprocess.call(cmd) print "SCP process completed" @@ -331,7 +343,7 @@ def put_controller(cls, args, config): if controller_obj is None: return # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - #logging.debug("instance_list={0}".format(instance_list)) + # logging.debug("instance_list={0}".format(instance_list)) # Check if they are running ip = None if len(instance_list) > 0: @@ -343,14 +355,14 @@ def put_controller(cls, args, config): if ip is None: print "No active instance for this controller" return - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) - cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/shared'.format(ip)] + # print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) + # os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) + cmd = ['/usr/bin/scp', '-oStrictHostKeyChecking=no', '-oUserKnownHostsFile=/dev/null', '-i', + controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/shared'.format(ip)] print " ".join(cmd) subprocess.call(cmd) print "SSH process completed" - @classmethod def status_controller(cls, args, config): """ Get status of the head node of a MOLNs controller. """ @@ -365,7 +377,7 @@ def status_controller(cls, args, config): table_data = [] if len(instance_list) > 0: for i in instance_list: - #provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + # provider_name = config.get_object_by_id(i.provider_id, 'Provider').name try: p = config.get_object_by_id(i.provider_id, 'Provider') provider_name = p.name @@ -373,7 +385,9 @@ def status_controller(cls, args, config): provider_name = 'ERROR: {0}'.format(e) controller_name = config.get_object_by_id(i.controller_id, 'Controller').name status = controller_obj.get_instance_status(i) - table_data.append([controller_name, status, 'controller', provider_name, i.provider_instance_identifier, i.ip_address]) + table_data.append( + [controller_name, status, 'controller', provider_name, i.provider_instance_identifier, + i.ip_address]) else: return {'msg': "No instance running for this controller"} @@ -383,16 +397,18 @@ def status_controller(cls, args, config): for i in instance_list: worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name worker_obj = cls._get_workerobj([worker_name], config) - #provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + # provider_name = config.get_object_by_id(i.provider_id, 'Provider').name try: p = config.get_object_by_id(i.provider_id, 'Provider') provider_name = p.name except DatastoreException as e: provider_name = 'ERROR: {0}'.format(e) status = worker_obj.get_instance_status(i) - table_data.append([worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) - #table_print(['name','status','type','provider','instance id', 'IP address'],table_data) - r = {'type':'table', 'column_names':['name','status','type','provider','instance id', 'IP address'], 'data':table_data} + table_data.append( + [worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) + # table_print(['name','status','type','provider','instance id', 'IP address'],table_data) + r = {'type': 'table', 'column_names': ['name', 'status', 'type', 'provider', 'instance id', 'IP address'], + 'data': table_data} return r else: instance_list = config.get_all_instances() @@ -405,15 +421,15 @@ def status_controller(cls, args, config): worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name table_data.append([worker_name, 'worker', provider_name, i.provider_instance_identifier]) else: - table_data.append([controller_name, 'controller', provider_name, i.provider_instance_identifier]) + table_data.append( + [controller_name, 'controller', provider_name, i.provider_instance_identifier]) - r = {'type':'table', 'column_names':['name','type','provider','instance id'], 'data':table_data} - r['msg']= "\n\tUse 'molns status NAME' to see current status of each instance." + r = {'type': 'table', 'column_names': ['name', 'type', 'provider', 'instance id'], 'data': table_data} + r['msg'] = "\n\tUse 'molns status NAME' to see current status of each instance." return r else: return {'msg': "No instance found"} - @classmethod def start_controller(cls, args, config, password=None): """ Start the MOLNs controller. """ @@ -444,7 +460,7 @@ def start_controller(cls, args, config, password=None): sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_ipython_controller(inst, notebook_password=password) sshdeploy.deploy_molns_webserver(inst) - #sshdeploy.deploy_stochss(inst.ip_address, port=443) + # sshdeploy.deploy_stochss(inst.ip_address, port=443) @classmethod def stop_controller(cls, args, config): @@ -530,7 +546,8 @@ def connect_controller_to_local(cls, args, config): sshdeploy = SSHDeploy(config=controller_obj.provider, config_dir=config.config_dir) client_file_data = sshdeploy.get_ipython_client_file(inst.ip_address) home_dir = os.environ.get('HOME') - ipython_client_filename = os.path.join(home_dir, '.ipython/profile_{0}/'.format(profile_name), 'security/ipcontroller-client.json') + ipython_client_filename = os.path.join(home_dir, '.ipython/profile_{0}/'.format(profile_name), + 'security/ipcontroller-client.json') logging.debug("Writing file {0}".format(ipython_client_filename)) with open(ipython_client_filename, 'w') as fd: fd.write(client_file_data) @@ -544,7 +561,7 @@ class MOLNSWorkerGroup(MOLNSbase): def worker_group_export(cls, args, config): """ Export the configuration of a worker group. """ if len(args) < 1: - raise MOLNSException("USAGE: molns worker export name [Filename]\n"\ + raise MOLNSException("USAGE: molns worker export name [Filename]\n" \ "\tExport the data from the worker group with the given name.") worker_name = args[0] if len(args) > 1: @@ -569,8 +586,8 @@ def worker_group_import(cls, args, config, json_data=None): """ Import the configuration of a worker group. """ if json_data is None: if len(args) < 1: - raise MOLNSException("USAGE: molns worker import [Filename.json]\n"\ - "\Import the data from the worker with the given name.") + raise MOLNSException("USAGE: molns worker import [Filename.json]\n" \ + "\Import the data from the worker with the given name.") filename = args[0] with open(filename) as fd: data = json.load(fd) @@ -590,16 +607,21 @@ def worker_group_import(cls, args, config, json_data=None): worker_obj = config.get_object(worker_name, kind='WorkerGroup') msg += "Found existing worker group\n" if worker_obj.provider.name != provider_obj.name: - raise MOLNSException("Import data has provider '{0}'. Worker group {1} exists with provider {2}. provider conversion is not possible.".format(data['provider_name'], worker_obj.name, worker_obj.provider.name)) + raise MOLNSException( + "Import data has provider '{0}'. Worker group {1} exists with provider {2}. provider conversion is not possible.".format( + data['provider_name'], worker_obj.name, worker_obj.provider.name)) if worker_obj.controller.name != controller_obj.name: - raise MOLNSException("Import data has controller '{0}'. Worker group {1} exists with controller {2}. provider conversion is not possible.".format(data['controller_name'], worker_obj.name, worker_obj.controller.name)) + raise MOLNSException( + "Import data has controller '{0}'. Worker group {1} exists with controller {2}. provider conversion is not possible.".format( + data['controller_name'], worker_obj.name, worker_obj.controller.name)) except DatastoreException as e: - worker_obj = config.create_object(ptype=provider_obj.type, name=worker_name, kind='WorkerGroup', provider_id=provider_obj.id, controller_id=controller_obj.id) + worker_obj = config.create_object(ptype=provider_obj.type, name=worker_name, kind='WorkerGroup', + provider_id=provider_obj.id, controller_id=controller_obj.id) msg += "Creating new worker group\n" cls.merge_config(worker_obj, data['config']) config.save_object(worker_obj, kind='WorkerGroup') msg += "Worker group data imported\n" - return {'msg':msg} + return {'msg': msg} @classmethod def worker_group_get_config(cls, name=None, provider_type=None, config=None): @@ -625,8 +647,8 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("Unknown provider type '{0}'".format(provider_type)) - p_hand = get_provider_handle('WorkerGroup',provider_type) - obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + p_hand = get_provider_handle('WorkerGroup', provider_type) + obj = p_hand('__tmp__', data={}, config_dir=config.config_dir) if obj is None: raise MOLNSException("Worker group {0} not found".format(name)) ret = [] @@ -651,10 +673,10 @@ def worker_group_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ - 'question':question, - 'key':key, + 'question': question, + 'key': key, 'value': myval, - 'type':'string' + 'type': 'string' }) return ret @@ -672,31 +694,33 @@ def setup_worker_groups(cls, args, config): except DatastoreException as e: # provider providers = config.list_objects(kind='Provider') - if len(providers)==0: - print "No providers configured, please configure one ('molns provider setup') before initializing worker group." + if len(providers) == 0: + print "No providers configured, " \ + "please configure one ('molns provider setup') before initializing worker group." return print "Select a provider:" - for n,p in enumerate(providers): - print "\t[{0}] {1}".format(n,p.name) - provider_ndx = int(raw_input_default("enter the number of provider:", default='0')) + for n, p in enumerate(providers): + print "\t[{0}] {1}".format(n, p.name) + provider_ndx = int(raw_input_default("Enter the number of provider:", default='0')) provider_id = providers[provider_ndx].id provider_obj = config.get_object(name=providers[provider_ndx].name, kind='Provider') logging.debug("using provider {0}".format(provider_obj)) # controller controllers = config.list_objects(kind='Controller') - if len(controllers)==0: - print "No controllers configured, please configure one ('molns controller setup') before initializing worker group." + if len(controllers) == 0: + print "No controllers configured, " \ + "please configure one ('molns controller setup') before initializing worker group." return print "Select a controller:" - for n,p in enumerate(controllers): - print "\t[{0}] {1}".format(n,p.name) - controller_ndx = int(raw_input_default("enter the number of controller:", default='0')) - controller_id = controllers[controller_ndx].id + for n, p in enumerate(controllers): + print "\t[{0}] {1}".format(n, p.name) + controller_ndx = int(raw_input_default("Enter the number of controller:", default='0')) controller_obj = config.get_object(name=controllers[controller_ndx].name, kind='Controller') logging.debug("using controller {0}".format(controller_obj)) # create object try: - worker_obj = config.create_object(ptype=provider_obj.type, name=group_name, kind='WorkerGroup', provider_id=provider_id, controller_id=controller_obj.id) + worker_obj = config.create_object(ptype=provider_obj.type, name=group_name, kind='WorkerGroup', + provider_id=provider_id, controller_id=controller_obj.id) except DatastoreException as e: print e return @@ -712,7 +736,7 @@ def list_worker_groups(cls, args, config): else: table_data = [] for g in groups: - #provider_name = config.get_object_by_id(g.provider_id, 'Provider').name + # provider_name = config.get_object_by_id(g.provider_id, 'Provider').name try: p = config.get_object_by_id(g.provider_id, 'Provider') provider_name = p.name @@ -724,7 +748,7 @@ def list_worker_groups(cls, args, config): except DatastoreException as e: controller_name = 'ERROR: {0}'.format(e) table_data.append([g.name, provider_name, controller_name]) - return {'type':'table','column_names':['name', 'provider', 'controller'], 'data':table_data} + return {'type': 'table', 'column_names': ['name', 'provider', 'controller'], 'data': table_data} @classmethod def show_worker_groups(cls, args, config): @@ -756,12 +780,15 @@ def status_worker_groups(cls, args, config): table_data = [] for i in instance_list: status = worker_obj.get_instance_status(i) - #print "{0} type={3} ip={1} id={2}".format(status, i.ip_address, i.provider_instance_identifier, worker_obj.PROVIDER_TYPE) + # print "{0} type={3} ip={1} id={2}".format(status, i.ip_address, i.provider_instance_identifier, worker_obj.PROVIDER_TYPE) worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name provider_name = config.get_object_by_id(i.provider_id, 'Provider').name status = worker_obj.get_instance_status(i) - table_data.append([worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) - return {'type':'table','column_names':['name','status','type','provider','instance id', 'IP address'],'data':table_data} + table_data.append( + [worker_name, status, 'worker', provider_name, i.provider_instance_identifier, i.ip_address]) + return {'type': 'table', + 'column_names': ['name', 'status', 'type', 'provider', 'instance id', 'IP address'], + 'data': table_data} else: return {'msg': "No worker instances running for this cluster"} else: @@ -777,15 +804,14 @@ def start_worker_groups(cls, args, config): num_vms_to_start = int(num_vms) controller_ip = cls.__launch_workers__get_controller(worker_obj, config) if controller_ip is None: return - #logging.debug("\tcontroller_ip={0}".format(controller_ip)) + # logging.debug("\tcontroller_ip={0}".format(controller_ip)) try: inst_to_deploy = cls.__launch_worker__start_or_resume_vms(worker_obj, config, num_vms_to_start) - #logging.debug("\tinst_to_deploy={0}".format(inst_to_deploy)) + # logging.debug("\tinst_to_deploy={0}".format(inst_to_deploy)) cls.__launch_worker__deploy_engines(worker_obj, controller_ip, inst_to_deploy, config) except ProviderException as e: print "Could not start workers: {0}".format(e) - @classmethod def add_worker_groups(cls, args, config): """ Add workers of a MOLNs cluster. """ @@ -828,7 +854,6 @@ def __launch_workers__get_controller(cls, worker_obj, config): return return controller_ip - @classmethod def __launch_worker__start_or_resume_vms(cls, worker_obj, config, num_vms_to_start=0): # Check for any instances are assigned to this worker group @@ -846,12 +871,12 @@ def __launch_worker__start_or_resume_vms(cls, worker_obj, config, num_vms_to_sta print "Resuming worker at {0}".format(i.ip_address) inst_to_resume.append(i) num_vms_to_start -= 1 - #logging.debug("inst_to_resume={0}".format(inst_to_resume)) + # logging.debug("inst_to_resume={0}".format(inst_to_resume)) if len(inst_to_resume) > 0: worker_obj.resume_instance(inst_to_resume) inst_to_deploy.extend(inst_to_resume) inst_to_deploy.extend(cls.__launch_worker__start_vms(worker_obj, num_vms_to_start)) - #logging.debug("inst_to_deploy={0}".format(inst_to_deploy)) + # logging.debug("inst_to_deploy={0}".format(inst_to_deploy)) return inst_to_deploy @classmethod @@ -861,12 +886,11 @@ def __launch_worker__start_vms(cls, worker_obj, num_vms_to_start=0): if num_vms_to_start > 0: # Start a new instances print "Starting {0} new workers".format(num_vms_to_start) - inst_to_deploy = worker_obj.start_instance(num=num_vms_to_start) - if not isinstance(inst_to_deploy,list): + inst_to_deploy = worker_obj.start_instance(num=num_vms_to_start) + if not isinstance(inst_to_deploy, list): inst_to_deploy = [inst_to_deploy] return inst_to_deploy - @classmethod def __launch_worker__deploy_engines(cls, worker_obj, controller_ip, inst_to_deploy, config): print "Deploying on {0} workers".format(len(inst_to_deploy)) @@ -880,8 +904,11 @@ def __launch_worker__deploy_engines(cls, worker_obj, controller_ip, inst_to_depl logging.debug("__launch_worker__deploy_engines() workpool(size={0})".format(len(inst_to_deploy))) jobs = [] for i in inst_to_deploy: - logging.debug("multiprocessing.Process(target=engine_ssh.deploy_ipython_engine({0}, engine_file)".format(i.ip_address)) - p = multiprocessing.Process(target=engine_ssh.deploy_ipython_engine, args=(i.ip_address, controller_ip, engine_file, controller_ssh_keyfile,)) + logging.debug( + "multiprocessing.Process(target=engine_ssh.deploy_ipython_engine({0}, engine_file)".format( + i.ip_address)) + p = multiprocessing.Process(target=engine_ssh.deploy_ipython_engine, args=( + i.ip_address, controller_ip, engine_file, controller_ssh_keyfile,)) jobs.append(p) p.start() logging.debug("__launch_worker__deploy_engines() joining processes.") @@ -938,6 +965,7 @@ def terminate_worker_groups(cls, args, config): else: print "No workers running in the worker group" + ############################################### class MOLNSProvider(MOLNSbase): @@ -945,8 +973,8 @@ class MOLNSProvider(MOLNSbase): def provider_export(cls, args, config): """ Export the configuration of a provider. """ if len(args) < 1: - raise MOLNSException("USAGE: molns provider export name [Filename]\n"\ - "\tExport the data from the provider with the given name.") + raise MOLNSException("USAGE: molns provider export name [Filename]\n" \ + "\tExport the data from the provider with the given name.") provider_name = args[0] if len(args) > 1: filename = args[1] @@ -969,8 +997,8 @@ def provider_import(cls, args, config, json_data=None): """ Import the configuration of a provider. """ if json_data is None: if len(args) < 1: - raise MOLNSException("USAGE: molns provider import [Filename.json]\n"\ - "\Import the data from the provider with the given name.") + raise MOLNSException("USAGE: molns provider import [Filename.json]\n" \ + "\Import the data from the provider with the given name.") filename = args[0] with open(filename) as fd: data = json.load(fd) @@ -984,15 +1012,16 @@ def provider_import(cls, args, config, json_data=None): provider_obj = config.get_object(provider_name, kind='Provider') msg += "Found existing provider\n" if provider_obj.type != data['type']: - raise MOLNSException("Import data has provider type '{0}'. Provier {1} exists with type {2}. Type conversion is not possible.".format(data['type'], provider_obj.name, provider_obj.type)) + raise MOLNSException( + "Import data has provider type '{0}'. Provier {1} exists with type {2}. Type conversion is not possible.".format( + data['type'], provider_obj.name, provider_obj.type)) except DatastoreException as e: provider_obj = config.create_object(name=provider_name, ptype=data['type'], kind='Provider') msg += "Creating new provider\n" cls.merge_config(provider_obj, data['config']) config.save_object(provider_obj, kind='Provider') msg += "Provider data imported\n" - return {'msg':msg} - + return {'msg': msg} @classmethod def provider_get_config(cls, name=None, provider_type=None, config=None): @@ -1018,8 +1047,8 @@ def provider_get_config(cls, name=None, provider_type=None, config=None): if obj is None and provider_type is not None: if provider_type not in VALID_PROVIDER_TYPES: raise MOLNSException("unknown provider type '{0}'".format(provider_type)) - p_hand = get_provider_handle('Provider',provider_type) - obj = p_hand('__tmp__',data={},config_dir=config.config_dir) + p_hand = get_provider_handle('Provider', provider_type) + obj = p_hand('__tmp__', data={}, config_dir=config.config_dir) if obj is None: raise MOLNSException("provider {0} not found".format(name)) ret = [] @@ -1044,40 +1073,36 @@ def provider_get_config(cls, name=None, provider_type=None, config=None): if myval is not None and 'obfuscate' in conf and conf['obfuscate']: myval = '********' ret.append({ - 'question':question, - 'key':key, + 'question': question, + 'key': key, 'value': myval, - 'type':'string' + 'type': 'string' }) return ret @classmethod def provider_setup(cls, args, config): """ Setup a new provider. Create the MOLNS image and SSH key if necessary.""" - #print "MOLNSProvider.provider_setup(args={0})".format(args) if len(args) < 1: print "USAGE: molns provider setup name" print "\tCreates a new provider with the given name." return - # find the \n\tWhere PROVIDER_TYPE is one of: {0}".format(VALID_PROVIDER_TYPES) - # provider name - provider_name = args[0] # check if provider exists try: provider_obj = config.get_object(args[0], kind='Provider') - except DatastoreException as e: + except DatastoreException: # ask provider type print "Select a provider type:" - for n,p in enumerate(VALID_PROVIDER_TYPES): - print "\t[{0}] {1}".format(n,p) + for n, p in enumerate(VALID_PROVIDER_TYPES): + print "\t[{0}] {1}".format(n, p) while True: try: - provider_ndx = int(raw_input_default("enter the number of type:", default='0')) + provider_ndx = int(raw_input_default("Enter the number of type:", default='0')) provider_type = VALID_PROVIDER_TYPES[provider_ndx] break except (ValueError, IndexError): pass - logging.debug("provider type '{0}'".format(provider_type)) + logging.debug("Provider type '{0}'".format(provider_type)) # Create provider try: provider_obj = config.create_object(name=args[0], ptype=provider_type, kind='Provider') @@ -1091,7 +1116,6 @@ def provider_setup(cls, args, config): cls.provider_initialize(args[0], config) - @classmethod def provider_initialize(cls, provider_name, config): """ Create the MOLNS image and SSH key if necessary.""" @@ -1136,7 +1160,6 @@ def provider_initialize(cls, provider_name, config): print "Success." config.save_object(provider_obj, kind='Provider') - @classmethod def provider_rebuild(cls, args, config): """ Rebuild the MOLNS image.""" @@ -1161,7 +1184,7 @@ def provider_rebuild(cls, args, config): @classmethod def provider_list(cls, args, config): """ List all the currently configured providers.""" - #print "MOLNSProvider.provider_list(args={0}, config={1})".format(args, config) + # print "MOLNSProvider.provider_list(args={0}, config={1})".format(args, config) providers = config.list_objects(kind='Provider') if len(providers) == 0: print "No providers configured" @@ -1169,14 +1192,14 @@ def provider_list(cls, args, config): table_data = [] for p in providers: table_data.append([p.name, p.type]) - #table_print(['name', 'type'], table_data) - r = {'type':'table', 'column_names':['name', 'type'],'data':table_data} + # table_print(['name', 'type'], table_data) + r = {'type': 'table', 'column_names': ['name', 'type'], 'data': table_data} return r @classmethod def show_provider(cls, args, config): """ Show all the details of a provider config. """ - #print "MOLNSProvider.show_provider(args={0}, config={1})".format(args, config) + # print "MOLNSProvider.show_provider(args={0}, config={1})".format(args, config) if len(args) == 0: print "USAGE: molns provider show name" return @@ -1185,11 +1208,13 @@ def show_provider(cls, args, config): @classmethod def delete_provider(cls, args, config): """ Delete a provider config. """ - #print "MOLNSProvider.delete_provider(args={0}, config={1})".format(args, config) + # print "MOLNSProvider.delete_provider(args={0}, config={1})".format(args, config) if len(args) == 0: print "USAGE: molns provider delete name" return config.delete_object(name=args[0], kind='Provider') + + ############################################### class MOLNSInstances(MOLNSbase): @@ -1208,7 +1233,7 @@ def show_instances(cls, args, config): name = config.get_object_by_id(i.controller_id, 'Controller').name itype = 'controller' table_data.append([i.id, provider_name, i.provider_instance_identifier, itype, name]) - table_print(['ID', 'provider', 'instance id', 'type', 'name'],table_data) + table_print(['ID', 'provider', 'instance id', 'type', 'name'], table_data) else: print "No instance found" @@ -1230,7 +1255,6 @@ def delete_instance(cls, args, config): config.delete_instance(instance) print "instance {0} deleted".format(instance_id) - @classmethod def clear_instances(cls, args, config): """ delete all instances in the db """ @@ -1243,14 +1267,14 @@ def clear_instances(cls, args, config): else: print "No instance found" + ############################################################################################## class MolnsLocal(MOLNSbase): - @classmethod def setup_local(cls): - #TODO + # TODO pass @classmethod @@ -1258,58 +1282,65 @@ def start_local(cls): # TODO pass + ############################################################################################## ############################################################################################## ############################################################################################## ############################################################################################## ############################################################################################## ############################################################################################## -# Below is the API for the commmand line execution +# Below is the API for the command line execution + class CommandException(Exception): pass + def process_output_exception(e): logging.exception(e) sys.stderr.write("Error: {0}\n".format(e)) + def process_output(result): if result is not None: - if type(result)==dict and 'type' in result: + if type(result) == dict and 'type' in result: if result['type'] == 'table' and 'column_names' in result and 'data' in result: - table_print(result['column_names'],result['data']) + table_print(result['column_names'], result['data']) if result['type'] == 'file' and 'filename' in result and 'data' in result: - output_to_file(result['filename'],result['data']) - elif type(result)==dict and 'msg' in result: + output_to_file(result['filename'], result['data']) + elif type(result) == dict and 'msg' in result: print result['msg'] else: print result + def output_to_file(filename, data): - with open(filename,'w+') as fd: + with open(filename, 'w+') as fd: fd.write(data) + def table_print(column_names, data): - column_width = [0]*len(column_names) - for i,n in enumerate(column_names): + column_width = [0] * len(column_names) + for i, n in enumerate(column_names): column_width[i] = len(str(n)) for row in data: if len(row) != len(column_names): raise Exception("len(row) != len(column_names): {0} vs {1}".format(len(row), len(column_names))) - for i,n in enumerate(row): + for i, n in enumerate(row): if len(str(n)) > column_width[i]: column_width[i] = len(str(n)) - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' - out = " | ".join([ column_names[i].ljust(column_width[i]) for i in range(len(column_names))]) - print '| '+out+' |' - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' + out = "|".join(["-" * (column_width[i] + 2) for i in range(len(column_names))]) + print '|' + out + '|' + out = " | ".join([column_names[i].ljust(column_width[i]) for i in range(len(column_names))]) + print '| ' + out + ' |' + out = "|".join(["-" * (column_width[i] + 2) for i in range(len(column_names))]) + print '|' + out + '|' for row in data: - out = " | ".join([ str(n).ljust(column_width[i]) for i,n in enumerate(row)]) - print '| '+out+' |' - out = "|".join([ "-"*(column_width[i]+2) for i in range(len(column_names))]) - print '|'+out+'|' + out = " | ".join([str(n).ljust(column_width[i]) for i, n in enumerate(row)]) + print '| ' + out + ' |' + out = "|".join(["-" * (column_width[i] + 2) for i in range(len(column_names))]) + print '|' + out + '|' + def raw_input_default(q, default=None, obfuscate=False): if default is None or default == '': @@ -1324,6 +1355,7 @@ def raw_input_default(q, default=None, obfuscate=False): else: return ret.strip() + def raw_input_default_config(q, default=None, obj=None): """ Ask the user and process the response with a default value. """ if default is None: @@ -1342,27 +1374,29 @@ def raw_input_default_config(q, default=None, obj=None): else: return raw_input_default(q['q'], default=default, obfuscate=False) + def setup_object(obj): """ Setup a molns_datastore object using raw_input_default function. """ for key, conf, value in obj.get_config_vars(): - #print "key: {0}\nconf: {1}\nvalue: {2}".format(key, conf, value) obj[key] = raw_input_default_config(conf, default=value, obj=obj) + ############################################### -class SubCommand(): +class SubCommand: def __init__(self, command, subcommands): self.command = command self.subcommands = subcommands + def __str__(self): r = '' for c in self.subcommands: - r += self.command + " " + c.__str__() + "\n" + r += self.command + " " + c.__str__() + "\n" return r[:-1] + def __eq__(self, other): return self.command == other def run(self, args, config_dir=None): - #print "SubCommand().run({0}, {1})".format(self.command, args) if len(args) > 0: cmd = args[0] for c in self.subcommands: @@ -1370,10 +1404,11 @@ def run(self, args, config_dir=None): return c.run(args[1:], config_dir=config_dir) raise CommandException("command not found") + ############################################### -class Command(): +class Command: def __init__(self, command, args_defs={}, description=None, function=None): self.command = command self.args_defs = args_defs @@ -1386,13 +1421,13 @@ def __init__(self, command, args_defs={}, description=None, function=None): self.description = description def __str__(self): - ret = self.command+" " - for k,v in self.args_defs.iteritems(): + ret = self.command + " " + for k, v in self.args_defs.iteritems(): if v is None: ret += "[{0}] ".format(k) else: - ret += "[{0}={1}] ".format(k,v) - ret += "\n\t"+self.description + ret += "[{0}={1}] ".format(k, v) + ret += "\n\t" + self.description return ret def __eq__(self, other): @@ -1401,94 +1436,97 @@ def __eq__(self, other): def run(self, args, config_dir=None): config = MOLNSConfig(config_dir=config_dir) return self.function(args, config=config) + + ############################################### COMMAND_LIST = [ - # Commands to interact with the head-node. - Command('ssh', {'name':None}, + # Commands to interact with the head-node. + Command('ssh', {'name': None}, function=MOLNSController.ssh_controller), - Command('status', {'name':None}, + Command('status', {'name': None}, function=MOLNSController.status_controller), - Command('start', {'name':None}, + Command('start', {'name': None}, function=MOLNSController.start_controller), - Command('stop', {'name':None}, + Command('stop', {'name': None}, function=MOLNSController.stop_controller), - Command('terminate', {'name':None}, + Command('terminate', {'name': None}, function=MOLNSController.terminate_controller), - Command('put', {'name':None, 'file':None}, + Command('put', {'name': None, 'file': None}, function=MOLNSController.put_controller), - Command('upload', {'name':None, 'file':None}, + Command('upload', {'name': None, 'file': None}, function=MOLNSController.upload_controller), - #Command('local-connect', {'name':None}, - # function=MOLNSController.connect_controller_to_local), - # Commands to interact with controller - SubCommand('controller',[ - Command('setup', {'name':None}, + # Command('local-connect', {'name':None}, + # function=MOLNSController.connect_controller_to_local), + # Commands to interact with controller + SubCommand('controller', [ + Command('setup', {'name': None}, function=MOLNSController.setup_controller), - Command('list', {'name':None}, + Command('list', {'name': None}, function=MOLNSController.list_controller), - Command('show', {'name':None}, + Command('show', {'name': None}, function=MOLNSController.show_controller), - Command('delete', {'name':None}, + Command('delete', {'name': None}, function=MOLNSController.delete_controller), - Command('export',{'name':None}, + Command('export', {'name': None}, function=MOLNSController.controller_export), - Command('import',{'filename.json':None}, + Command('import', {'filename.json': None}, function=MOLNSController.controller_import), - ]), - # Commands to interact with Worker-Groups - SubCommand('worker',[ - Command('setup', {'name':None}, + ]), + # Commands to interact with Worker-Groups + SubCommand('worker', [ + Command('setup', {'name': None}, function=MOLNSWorkerGroup.setup_worker_groups), - Command('list', {'name':None}, + Command('list', {'name': None}, function=MOLNSWorkerGroup.list_worker_groups), - Command('show', {'name':None}, + Command('show', {'name': None}, function=MOLNSWorkerGroup.show_worker_groups), - Command('delete', {'name':None}, + Command('delete', {'name': None}, function=MOLNSWorkerGroup.delete_worker_groups), - Command('start', {'name':None}, + Command('start', {'name': None}, function=MOLNSWorkerGroup.start_worker_groups), - Command('add', {'name':None}, + Command('add', {'name': None}, function=MOLNSWorkerGroup.add_worker_groups), - Command('status', {'name':None}, + Command('status', {'name': None}, function=MOLNSWorkerGroup.status_worker_groups), - #Command('stop', {'name':None}, - # function=MOLNSWorkerGroup.stop_worker_groups), - Command('terminate', {'name':None}, + # Command('stop', {'name':None}, + # function=MOLNSWorkerGroup.stop_worker_groups), + Command('terminate', {'name': None}, function=MOLNSWorkerGroup.terminate_worker_groups), - Command('export',{'name':None}, + Command('export', {'name': None}, function=MOLNSWorkerGroup.worker_group_export), - Command('import',{'filename.json':None}, + Command('import', {'filename.json': None}, function=MOLNSWorkerGroup.worker_group_import), - ]), - # Commands to interact with Infrastructure-Providers - SubCommand('provider',[ - Command('setup',{'name':None}, + ]), + # Commands to interact with Infrastructure-Providers + SubCommand('provider', [ + Command('setup', {'name': None}, function=MOLNSProvider.provider_setup), - Command('rebuild',{'name':None}, + Command('rebuild', {'name': None}, function=MOLNSProvider.provider_rebuild), - Command('list',{'name':None}, + Command('list', {'name': None}, function=MOLNSProvider.provider_list), - Command('show',{'name':None}, + Command('show', {'name': None}, function=MOLNSProvider.show_provider), - Command('delete',{'name':None}, + Command('delete', {'name': None}, function=MOLNSProvider.delete_provider), - Command('export',{'name':None}, + Command('export', {'name': None}, function=MOLNSProvider.provider_export), - Command('import',{'filename.json':None}, + Command('import', {'filename.json': None}, function=MOLNSProvider.provider_import), - ]), - # Commands to interact with the instance DB - SubCommand('instancedb',[ - Command('list', {}, + ]), + # Commands to interact with the instance DB + SubCommand('instancedb', [ + Command('list', {}, function=MOLNSInstances.show_instances), - Command('delete', {'ID':None}, + Command('delete', {'ID': None}, function=MOLNSInstances.delete_instance), - Command('clear', {}, + Command('clear', {}, function=MOLNSInstances.clear_instances), - ]), + ]), + +] - ] def printHelp(): print "molns " @@ -1502,22 +1540,23 @@ def parseArgs(): if len(sys.argv) < 2 or sys.argv[1] == '-h': printHelp() return - + Log.verbose = False arg_list = sys.argv[1:] config_dir = './.molns/' + while len(arg_list) > 0 and arg_list[0].startswith('--'): + if arg_list[0].startswith('--config='): - config_dir = sys.argv[1].split('=',2)[1] + config_dir = sys.argv[1].split('=', 2)[1] + if arg_list[0].startswith('--debug'): print "Turning on Debugging output" - logger.setLevel(logging.DEBUG) #for Debugging - if arg_list[0].startswith('--docker-debug'): - print "Turning on debugging for Docker" - import MolnsLib.Utils - MolnsLib.Utils.debug_docker = True + logger.setLevel(logging.DEBUG) + Log.verbose = True + arg_list = arg_list[1:] - if len(arg_list) == 0 or arg_list[0] =='help' or arg_list[0] == '-h': + if len(arg_list) == 0 or arg_list[0] == 'help' or arg_list[0] == '-h': printHelp() return @@ -1535,7 +1574,6 @@ def parseArgs(): return print "unknown command: " + " ".join(arg_list) - #printHelp() print "use 'molns help' to see all possible commands" From d61603ea9e728474804c241387d80216a07047bd Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Thu, 13 Oct 2016 14:26:26 -0700 Subject: [PATCH 066/100] refactor code. enable use of image tags with DockerProvider --- .gitignore | 1 - MolnsLib/Constants.py | 5 +- MolnsLib/Docker.py | 92 ++++++++++++++--- MolnsLib/DockerProvider.py | 20 ++-- MolnsLib/DockerSSH.py | 6 +- MolnsLib/molns_datastore.py | 8 +- MolnsLib/ssh_deploy.py | 194 +++++++++++++++++++++--------------- molns.py | 2 + 8 files changed, 218 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index bb640f0..c7091ed 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,3 @@ qsubscript /dockerfile_* \#qsubscript\# docker_test.py -.idea/ diff --git a/MolnsLib/Constants.py b/MolnsLib/Constants.py index 62770a3..64e4b4a 100644 --- a/MolnsLib/Constants.py +++ b/MolnsLib/Constants.py @@ -1,7 +1,7 @@ class Constants: LOGGING_DIRECTORY = "~/MOLNS_LOG" DOCKER_BASE_URL = "unix://var/run/docker.sock" - DOCKER_DEFAULT_IMAGE = "aviralcse/docker-provider" + DOCKER_DEFAULT_IMAGE = "ubuntu:latest" DOCKER_DEFAULT_PORT = '9000' DOCKER_CONTAINER_RUNNING = "running" DOCKER_CONTAINER_EXITED = "exited" @@ -9,3 +9,6 @@ class Constants: DOKCER_IMAGE_ID_LENGTH = 12 DOCKER_IMAGE_PREFIX = "aviralcse/docker-provider-" DOCKER_PY_IMAGE_ID_PREFIX_LENGTH = 7 + DockerProvider = "Docker" + DockerNonExistentTag = "**NA**" + DockerImageDelimiter = "|||" diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 1ddae7a..9b8f68f 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -11,7 +11,7 @@ class Docker: - """ A wrapper over docker-py.""" + """ A wrapper over docker-py and some utility methods and classes. """ LOG_TAG = "Docker" @@ -26,13 +26,18 @@ def __init__(self): self.build_count = 0 logging.basicConfig(level=logging.DEBUG) - def create_container(self, image_id=Constants.DOCKER_DEFAULT_IMAGE, port_bindings={80: 8080}): + def create_container(self, image_str, port_bindings={80: 8080}): """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container to 8080 of locahost by default""" - Log.write_log("Using image {0}".format(image_id)) + docker_image = DockerImage.from_string(image_str) + + image = docker_image.image_id if docker_image.image_id is not Constants.DockerNonExistentTag \ + else docker_image.image_tag + + Log.write_log("Using image {0}".format(image)) hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) - container = self.client.create_container(image=image_id, command="/bin/bash", tty=True, detach=True, ports=[80], + container = self.client.create_container(image=image, command="/bin/bash", tty=True, detach=True, ports=[80], host_config=hc) return container.get("Id") @@ -102,22 +107,28 @@ def build_image(self, dockerfile): # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image # id. - exp = r'[a-z0-9]{12}' - image_id = re.findall(exp, str(last_line))[0] + image_id = get_docker_image_id_from_string(str(last_line)) Log.write_log(Docker.LOG_TAG + "Image ID: {0}".format(image_id)) - return [image_id, image_tag] + return str(DockerImage(image_id, image_tag)) + except (Docker.ImageBuildException, IndexError) as e: - print("ERROR {0}".format(e)) - return None + raise Docker.ImageBuildException(e) + + def image_exists(self, image_str): + """Checks if an image with the given ID/tag exists locally.""" + docker_image = DockerImage.from_string(image_str) + + if docker_image.image_id is Constants.DockerNonExistentTag \ + and docker_image.image_tag is Constants.DockerNonExistentTag: + raise InvalidDockerImageException("Neither image_id nor image_tag provided.") - def image_exists(self, image_id): - """Checks if an image with the given ID exists locally.""" for image in self.client.images(): some_id = image["Id"] some_tags = image["RepoTags"] - if image_id in some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: + if docker_image.image_id in \ + some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: return True - if image_id in some_tags: + if docker_image.image_tag in some_tags: return True return False @@ -160,3 +171,58 @@ def get_container_ip_address(self, container_id): if ip_address.startswith("1") is True: break return ip_address + + +def get_docker_image_id_from_string(some_string): + exp = r'[a-z0-9]{12}' + matches = re.findall(exp, some_string) + if len(matches) is 0: + return None + else: + return matches[0] + + +class InvalidDockerImageException(Exception): + def __init__(self, message): + super(message) + + +class DockerImage: + def __init__(self, image_id=None, image_tag=None): + if image_id in [None, Constants.DockerNonExistentTag] and image_tag in [None, Constants.DockerNonExistentTag]: + raise InvalidDockerImageException("Both image_id and image_tag cannot be None.") + + self.image_id = image_id if image_id is not None else Constants.DockerNonExistentTag + self.image_tag = image_tag if image_tag is not None else Constants.DockerNonExistentTag + + def __str__(self): + if self.image_id is Constants.DockerNonExistentTag and self.image_tag is Constants.DockerNonExistentTag: + raise InvalidDockerImageException( + "Cannot serialize DockerImage object because both image_id and image_tag are None.") + + return "{0}{1}{2}".format(self.image_id, Constants.DockerImageDelimiter, self.image_tag) + + @staticmethod + def from_string(serialized_docker_image): + temp = serialized_docker_image.split(Constants.DockerImageDelimiter) + + if len(temp) is 2: + return DockerImage(image_id=temp[0], image_tag=temp[1]) + + if len(temp) > 2 or len(temp) is 0: + raise InvalidDockerImageException("Unexpected format, cannot serialize to DockerImage.") + + temp = temp[0] + # Figure out if temp is image_id or image_name. + if DockerImage.looks_like_image_id(temp): + return DockerImage(image_id=temp) + else: + return DockerImage(image_tag=temp) + + @staticmethod + def looks_like_image_id(some_string): + possible_image_id = get_docker_image_id_from_string(some_string) + if some_string is possible_image_id: + return True + else: + return False diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index c4d49bc..c58778a 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -36,7 +36,8 @@ def start_instance(self, num=1): container_id = self.docker.create_container(self.provider.config["molns_image_name"]) stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=self.docker.get_container_ip_address(container_id) - , provider_id=self.provider.id, controller_id=self.id) + , provider_id=self.provider.id, controller_id=self.id, + provider_type=Constants.Constants.DockerProvider) started_containers.append(stored_container) if num == 1: return started_containers[0] @@ -89,7 +90,9 @@ class DockerProvider(DockerBase): ('group_name', {'q': 'Docker Security Group name', 'default': 'molns', 'ask': False}), # Unused. ('login_username', - {'default': 'ubuntu', 'ask': False}) # Unused. + {'default': 'ubuntu', 'ask': False}), # Unused. + ('provider_type', + {'default': Constants.Constants.DockerProvider, 'ask': False}) ]) def get_config_credentials(self): @@ -121,19 +124,18 @@ def create_seurity_group(self): return True def create_molns_image(self): - """ Create a molns image, save it on localhost and return ID of created image. Because of the implementation in - a higher layer, this forces MOLNs to recognise docker images based only on their IDs and NOT names.""" - dockerfile = None + """ Create a molns image, save it on localhost and return DockerImage ID of created image. """ + file_to_remove = None try: - dockerfile = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) + dockerfile, file_to_remove = self._create_dockerfile(installSoftware.InstallSW.get_command_list()) image_id = self.docker.build_image(dockerfile) return image_id except Exception as e: logging.exception(e) raise ProviderException("Failed to create molns image: {0}".format(e)) finally: - if dockerfile is not None: - os.remove(dockerfile) + if file_to_remove is not None: + os.remove(file_to_remove) def check_molns_image(self): """ Check if the molns image exists. """ @@ -183,7 +185,7 @@ def _create_dockerfile(self, commands): named_dockerfile.write(dockerfile) named_dockerfile.seek(0) - return named_dockerfile + return named_dockerfile, dockerfile_file @staticmethod def _preprocess(command): diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 9f13adf..70e8a7b 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -13,18 +13,18 @@ def __init__(self, docker): self.docker = docker self.container_id = None - def exec_command(self, command, unused): + def exec_command(self, command, verbose=None): cmd = re.sub("\"", "\\\"", command) # Escape all occurrences of ". ret_val, response = self.docker.execute_command(self.container_id, cmd) return response - def exec_multi_command(self, command, unused): + def exec_multi_command(self, command, verbose=None): return self.exec_command(command) def open_sftp(self): return MockSFTP(self.docker, self.container_id) - def connect(self, instance, unused1, unused2, unused3): + def connect(self, instance, endpoint, username=None, key_filename=None): self.container_id = instance.provider_instance_identifier def close(self): diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index b282d8c..c3901ee 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -90,7 +90,8 @@ class Instance(Base): provider_id = Column(Integer) ip_address = Column(String) provider_instance_identifier = Column(String) - + provider_type = Column(String) + def __str__(self): return "Instance({0}): provider_instance_identifier={1} provider_id={2} controller_id={3} worker_group_id={4}".format(self.id, self.provider_instance_identifier, self.provider_id, self.controller_id, self.worker_group_id) @@ -339,11 +340,12 @@ def get_instance_by_id(self, id): """ Create or get the value for an instance. """ return self.session.query(Instance).filter_by(id=id).first() - def get_instance(self, provider_instance_identifier, ip_address, provider_id=None, controller_id=None, worker_group_id=None): + def get_instance(self, provider_instance_identifier, ip_address, provider_id=None, controller_id=None, + worker_group_id=None, provider_type=None): """ Create or get the value for an instance. """ p = self.session.query(Instance).filter_by(provider_instance_identifier=provider_instance_identifier).first() if p is None: - p = Instance(provider_instance_identifier=provider_instance_identifier, ip_address=ip_address, provider_id=provider_id, controller_id=controller_id, worker_group_id=worker_group_id) + p = Instance(provider_instance_identifier=provider_instance_identifier, ip_address=ip_address, provider_id=provider_id, controller_id=controller_id, worker_group_id=worker_group_id, provider_type=provider_type) self.session.add(p) self.session.commit() #logging.debug("Creating instance: {0}".format(p)) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 5010e3e..69abe81 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -8,6 +8,8 @@ import uuid import webbrowser import urllib2 + +from MolnsLib.Constants import Constants from ssh import SSH from DockerSSH import DockerSSH @@ -32,7 +34,7 @@ class SSHDeploy: DEFAULT_SSH_PORT = 22 DEFAULT_IPCONTROLLER_PORT = 9000 - DEFAULT_PYURDME_TEMPDIR="/mnt/pyurdme_tmp" + DEFAULT_PYURDME_TEMPDIR = "/mnt/pyurdme_tmp" def __init__(self, ssh, config=None, config_dir=None): if config is None: @@ -50,15 +52,16 @@ def __init__(self, ssh, config=None, config_dir=None): self.ssh = ssh self.profile = 'default' self.profile_dir = "/home/%s/.ipython/profile_default/" % (self.username) - self.ipengine_env = 'export INSTANT_OS_CALL_METHOD=SUBPROCESS;export PYURDME_TMPDIR={0};'.format(self.DEFAULT_PYURDME_TEMPDIR) + self.ipengine_env = 'export INSTANT_OS_CALL_METHOD=SUBPROCESS;export PYURDME_TMPDIR={0};'.format( + self.DEFAULT_PYURDME_TEMPDIR) self.profile_dir_server = self.profile_dir self.profile_dir_client = self.profile_dir self.ipython_port = self.DEFAULT_IPCONTROLLER_PORT - def scp_command(self, hostname): + def scp_command(self, hostname): return "scp -o 'StrictHostKeyChecking no' \ %s@%s:%ssecurity/ipcontroller-engine.json %ssecurity/" \ - %(self.username, hostname, self.profile_dir_server, self.profile_dir_client) + % (self.username, hostname, self.profile_dir_server, self.profile_dir_client) def prompt_for_password(self): import getpass @@ -78,7 +81,7 @@ def create_ssl_cert(self, cert_directory, cert_name_prefix, hostname): user_cert = cert_directory + '{0}-user_cert.pem'.format(cert_name_prefix) ssl_key = cert_directory + '{0}-ssl_key.pem'.format(cert_name_prefix) ssl_cert = cert_directory + '{0}-ssl_cert.pem'.format(cert_name_prefix) - ssl_subj = "/C=CN/ST=SH/L=STAR/O=Dis/CN=%s" % hostname + ssl_subj = "/C=CN/ST=SH/L=STAR/O=Dis/CN=%s" % hostname self.ssh.exec_command( "openssl req -new -newkey rsa:4096 -days 365 " '-nodes -x509 -subj %s -keyout %s -out %s' % @@ -101,55 +104,55 @@ def create_ipython_config(self, hostname, notebook_password=None): sha1pass = sha1pass_out[0].strip() else: sha1pass = sha1pass_out.strip() - Utils.print_d("SHA1PASS_OUT: " + sha1pass_out) - Utils.print_d("SHA1PASS: " + sha1pass) + Utils.Log.write_log("SHA1PASS_OUT: " + sha1pass_out) + Utils.Log.write_log("SHA1PASS: " + sha1pass) except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, hostname, self.ssh_endpoint) raise e sftp = self.ssh.open_sftp() notebook_config_file = sftp.file(remote_file_name, 'w+') - notebook_config_file.write('\n'.join([ - "c = get_config()", - "c.IPKernelApp.pylab = 'inline'", - "c.NotebookApp.certfile = u'%s'" % ssl_cert, - "c.NotebookApp.keyfile = u'%s'" % ssl_key, - "c.NotebookApp.ip = '*'", - "c.NotebookApp.open_browser = False", - "c.NotebookApp.password = u'%s'" % sha1pass, - "c.NotebookApp.port = %d" % int(notebook_port), - #"c.Global.exec_lines = ['import dill', 'from IPython.utils import pickleutil', 'pickleutil.use_dill()', 'import logging','logging.getLogger(\'UFL\').setLevel(logging.ERROR)','logging.getLogger(\'FFC\').setLevel(logging.ERROR)']", - ])) + notebook_config_file.write('\n'.join([ + "c = get_config()", + "c.IPKernelApp.pylab = 'inline'", + "c.NotebookApp.certfile = u'%s'" % ssl_cert, + "c.NotebookApp.keyfile = u'%s'" % ssl_key, + "c.NotebookApp.ip = '*'", + "c.NotebookApp.open_browser = False", + "c.NotebookApp.password = u'%s'" % sha1pass, + "c.NotebookApp.port = %d" % int(notebook_port), + # "c.Global.exec_lines = ['import dill', 'from IPython.utils import pickleutil', 'pickleutil.use_dill()', 'import logging','logging.getLogger(\'UFL\').setLevel(logging.ERROR)','logging.getLogger(\'FFC\').setLevel(logging.ERROR)']", + ])) notebook_config_file.close() - - remote_file_name='%sipcontroller_config.py' % self.profile_dir_server + + remote_file_name = '%sipcontroller_config.py' % self.profile_dir_server notebook_config_file = sftp.file(remote_file_name, 'w+') notebook_config_file.write('\n'.join([ - "c = get_config()", - "c.IPControllerApp.log_level=20", - "c.HeartMonitor.period=10000", - "c.HeartMonitor.max_heartmonitor_misses=10", - ])) + "c = get_config()", + "c.IPControllerApp.log_level=20", + "c.HeartMonitor.period=10000", + "c.HeartMonitor.max_heartmonitor_misses=10", + ])) notebook_config_file.close() # IPython startup code - remote_file_name='{0}startup/molns_dill_startup.py'.format(self.profile_dir_server) + remote_file_name = '{0}startup/molns_dill_startup.py'.format(self.profile_dir_server) dill_init_file = sftp.file(remote_file_name, 'w+') dill_init_file.write('\n'.join([ - 'import dill', - 'from IPython.utils import pickleutil', - 'pickleutil.use_dill()', - 'import logging', - "logging.getLogger('UFL').setLevel(logging.ERROR)", - "logging.getLogger('FFC').setLevel(logging.ERROR)" - "import cloud", - "logging.getLogger('Cloud').setLevel(logging.ERROR)" - ])) + 'import dill', + 'from IPython.utils import pickleutil', + 'pickleutil.use_dill()', + 'import logging', + "logging.getLogger('UFL').setLevel(logging.ERROR)", + "logging.getLogger('FFC').setLevel(logging.ERROR)" + "import cloud", + "logging.getLogger('Cloud').setLevel(logging.ERROR)" + ])) dill_init_file.close() sftp.close() def create_s3_config(self): sftp = self.ssh.open_sftp() - remote_file_name='.molns/s3.json' + remote_file_name = '.molns/s3.json' s3_config_file = sftp.file(remote_file_name, 'w') config = {} config["provider_type"] = self.config.type @@ -169,21 +172,22 @@ def get_cluster_id(self): wfd.write(new_id) with open(filename) as fd: idstr = fd.readline().rstrip() - logging.debug("get_cluster_id() file {0} found id = {1}".format(filename,idstr)) + logging.debug("get_cluster_id() file {0} found id = {1}".format(filename, idstr)) if idstr is None or len(idstr) == 0: - raise SSHDeployException("error getting id for cluster from file, please check your file '{0}'".format(filename)) + raise SSHDeployException( + "error getting id for cluster from file, please check your file '{0}'".format(filename)) return idstr def create_engine_config(self): sftp = self.ssh.open_sftp() - remote_file_name='%sipengine_config.py' % self.profile_dir_server + remote_file_name = '%sipengine_config.py' % self.profile_dir_server notebook_config_file = sftp.file(remote_file_name, 'w+') notebook_config_file.write('\n'.join([ - "c = get_config()", - "c.IPEngineApp.log_level=20", - "c.IPEngineApp.log_to_file = True", - "c.Global.exec_lines = ['import dill', 'from IPython.utils import pickleutil', 'pickleutil.use_dill()']", - ])) + "c = get_config()", + "c.IPEngineApp.log_level=20", + "c.IPEngineApp.log_to_file = True", + "c.Global.exec_lines = ['import dill', 'from IPython.utils import pickleutil', 'pickleutil.use_dill()']", + ])) notebook_config_file.close() sftp.close() self.create_s3_config() @@ -196,7 +200,7 @@ def _get_ipython_client_file(self): engine_file.close() sftp.close() return file_data - + def _put_ipython_client_file(self, file_data): sftp = self.ssh.open_sftp() engine_file = sftp.file(self.profile_dir_server + 'security/ipcontroller-client.json', 'w+') @@ -212,7 +216,7 @@ def _get_ipython_engine_file(self): engine_file.close() sftp.close() return file_data - + def _put_ipython_engine_file(self, file_data): sftp = self.ssh.open_sftp() engine_file = sftp.file(self.profile_dir_server + 'security/ipcontroller-engine.json', 'w+') @@ -223,7 +227,7 @@ def _put_ipython_engine_file(self, file_data): def exec_command_list_switch(self, command_list): for command in command_list: self.ssh.exec_command(command) - + def connect(self, instance, port=None): if port is None: port = self.ssh_endpoint @@ -249,9 +253,14 @@ def deploy_molns_webserver(self, instance): self.ssh.exec_command("sudo rm -rf /usr/local/molns_webroot") self.ssh.exec_command("sudo mkdir -p /usr/local/molns_webroot") self.ssh.exec_command("sudo chown ubuntu /usr/local/molns_webroot") - self.ssh.exec_command("git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") - self.ssh.exec_multi_command("cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format(self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') - self.ssh.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_WEBSERVER_PORT, self.DEFAULT_PRIVATE_WEBSERVER_PORT)) + self.ssh.exec_command( + "git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") + self.ssh.exec_multi_command( + "cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format( + self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') + self.ssh.exec_command( + "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( + self.DEFAULT_PUBLIC_WEBSERVER_PORT, self.DEFAULT_PRIVATE_WEBSERVER_PORT)) self.ssh.close() print "Deploying MOLNs webserver" url = "http://{0}/".format(ip_address) @@ -262,7 +271,7 @@ def deploy_molns_webserver(self, instance): sys.stdout.flush() break except Exception as e: - #sys.stdout.write("{0}".format(e)) + # sys.stdout.write("{0}".format(e)) sys.stdout.write(".") sys.stdout.flush() time.sleep(1) @@ -286,7 +295,8 @@ def deploy_stochss(self, ip_address, port=1443): print "Configure Nginx" (ssl_key, ssl_cert) = self.create_ssl_cert('/home/ubuntu/.nginx_cert/', 'stochss', ip_address) sftp = self.ssh.open_sftp() - with open(os.path.dirname(os.path.abspath(__file__))+os.sep+'..'+os.sep+'templates'+os.sep+'nginx.conf') as fd: + with open(os.path.dirname( + os.path.abspath(__file__)) + os.sep + '..' + os.sep + 'templates' + os.sep + 'nginx.conf') as fd: web_file = sftp.file("/tmp/nginx.conf", 'w+') buff = fd.read() buff = string.replace(buff, '###LISTEN_PORT###', str(port)) @@ -312,14 +322,15 @@ def deploy_stochss(self, ip_address, port=1443): req = urllib2.urlopen(stochss_url) break except Exception as e: - #sys.stdout.write("{0}".format(e)) + # sys.stdout.write("{0}".format(e)) sys.stdout.write(".") sys.stdout.flush() time.sleep(1) print "Success!" print "Configuring StochSS" admin_token = uuid.uuid4() - create_and_exchange_admin_token = "python /usr/local/stochss/generate_admin_token.py {0}".format(admin_token) + create_and_exchange_admin_token = "python /usr/local/stochss/generate_admin_token.py {0}".format( + admin_token) self.ssh.exec_command(create_and_exchange_admin_token) time.sleep(1) stochss_url = "{0}login?secret_key={1}".format(stochss_url, admin_token) @@ -335,44 +346,65 @@ def deploy_ipython_controller(self, instance, notebook_password=None): try: print "{0}:{1}".format(ip_address, self.ssh_endpoint) self.connect(instance, self.ssh_endpoint) - + # Set up the symlink to local scratch space self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea") self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea") self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea/cache") self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") - self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) - + self.ssh.exec_command( + "test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) + # Setup symlink to the shared scratch space self.ssh.exec_command("sudo mkdir -p /mnt/molnsshared") self.ssh.exec_command("sudo chown ubuntu /mnt/molnsshared") - self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsshared {0}".format('/home/ubuntu/shared')) + self.ssh.exec_command( + "test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsshared {0}".format('/home/ubuntu/shared')) # self.ssh.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) self.ssh.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) # - #self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") + # self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") self.ssh.exec_command("mkdir -p .molns") self.create_s3_config() self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_ipython_config(ip_address, notebook_password) self.create_engine_config() - self.ssh.exec_command("source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2} --log-to-file".format(ip_address, self.profile, self.ipython_port), '\n') - # Start one ipengine per processor - num_procs = self.get_number_processors() - num_engines = num_procs - 2 - for _ in range(num_engines): - self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) - self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format(self.profile, self.ipengine_env)) - self.ssh.exec_command("sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(self.DEFAULT_PUBLIC_NOTEBOOK_PORT,self.DEFAULT_PRIVATE_NOTEBOOK_PORT)) - self.ssh.close() + + # If provider is Docker, then ipython controller and ipengines aren't started + + if instance.provider_type != Constants.DockerProvider: + Utils.Log.write_log("Provider type is NOT Docker.") + self.ssh.exec_command( + "source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2}--log-to-file".format( + ip_address, self.profile, self.ipython_port), '\n') + # Start one ipengine per processor + num_procs = self.get_number_processors() + num_engines = num_procs - 2 + for _ in range(num_engines): + self.ssh.exec_command( + "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format( + self.profile, self.ipengine_env)) + + self.ssh.exec_command( + "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format( + self.profile, self.ipengine_env)) + + self.ssh.exec_command( + "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( + self.DEFAULT_PUBLIC_NOTEBOOK_PORT, self.DEFAULT_PRIVATE_NOTEBOOK_PORT)) + except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - url = "http://%s" %(ip_address) - print "\nThe URL for your MOLNs cluster is: %s." % url + + finally: + self.ssh.close() + + url = "http://%s" % (ip_address) + print "\nThe URL for your MOLNs head node is: %s." % url def get_ipython_engine_file(self, ip_address): try: @@ -396,30 +428,28 @@ def get_ipython_client_file(self, ip_address): print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, controller_ssh_keyfile): try: print "{0}:{1}".format(ip_address, self.ssh_endpoint) self.connect(ip_address, self.ssh_endpoint) - + # Setup the symlink to local scratch space self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea") self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea") self.ssh.exec_command("sudo mkdir -p /mnt/molnsarea/cache") self.ssh.exec_command("sudo chown ubuntu /mnt/molnsarea/cache") - - self.ssh.exec_command("test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) + self.ssh.exec_command( + "test -e {0} && sudo rm {0} ; sudo ln -s /mnt/molnsarea {0}".format('/home/ubuntu/localarea')) # self.ssh.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) self.ssh.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) # Setup config for object store self.ssh.exec_command("mkdir -p .molns") self.create_s3_config() - - + # SSH mount the controller on each engine - remote_file_name='.ssh/id_dsa' + remote_file_name = '.ssh/id_dsa' with open(controller_ssh_keyfile) as fd: sftp = self.ssh.open_sftp() controller_keyfile = sftp.file(remote_file_name, 'w') @@ -431,10 +461,12 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont sftp.close() self.ssh.exec_command("chmod 0600 {0}".format(remote_file_name)) self.ssh.exec_command("mkdir -p /home/ubuntu/shared") - self.ssh.exec_command("sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format(controler_ip)) + self.ssh.exec_command( + "sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format( + controler_ip)) # Update the Molnsutil package: TODO remove when molnsutil is stable - #self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") + # self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_engine_config() @@ -442,7 +474,9 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont self._put_ipython_engine_file(engine_file_data) # Start one ipengine per processor for _ in range(self.get_number_processors()): - self.ssh.exec_command("{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format(self.profile, self.ipengine_env)) + self.ssh.exec_command( + "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format( + self.profile, self.ipengine_env)) self.ssh.close() diff --git a/molns.py b/molns.py index 2ab5fe6..1d3f45b 100755 --- a/molns.py +++ b/molns.py @@ -2,6 +2,7 @@ import os import sys +from MolnsLib.Constants import Constants from MolnsLib.Utils import Log from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES, get_provider_handle from MolnsLib.molns_provider import ProviderException @@ -456,6 +457,7 @@ def start_controller(cls, args, config, password=None): # Start a new instance print "Starting new controller" inst = controller_obj.start_instance() + # deploying sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_ipython_controller(inst, notebook_password=password) From dbc4003910c1a32d5ce067f74624c9bc5961d94f Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 14 Oct 2016 16:36:10 -0700 Subject: [PATCH 067/100] map both notebook and web server ports in case of DockerProvider, modify MolnsLanding page to not break when using DockerProvider, add capability to run multiple DockerControllers at the same time --- MolnsLib/Docker.py | 9 ++-- MolnsLib/DockerProvider.py | 15 +++++- MolnsLib/molns_datastore.py | 2 +- MolnsLib/molns_landing_page.py | 91 ++++++++++++++++++++++++++++++++++ MolnsLib/molns_provider.py | 3 +- MolnsLib/ssh_deploy.py | 17 +++++-- molns.py | 2 +- 7 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 MolnsLib/molns_landing_page.py diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 9b8f68f..6bd8e11 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -2,6 +2,7 @@ import re import time +from MolnsLib import ssh_deploy from MolnsLib.Utils import Log from molns_provider import ProviderBase from Constants import Constants @@ -26,7 +27,8 @@ def __init__(self): self.build_count = 0 logging.basicConfig(level=logging.DEBUG) - def create_container(self, image_str, port_bindings={80: 8080}): + def create_container(self, image_str, port_bindings={ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT: 8080, + ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT: 8081}): """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container to 8080 of locahost by default""" @@ -37,8 +39,9 @@ def create_container(self, image_str, port_bindings={80: 8080}): Log.write_log("Using image {0}".format(image)) hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) - container = self.client.create_container(image=image, command="/bin/bash", tty=True, detach=True, ports=[80], - host_config=hc) + container = self.client.create_container(image=image, command="/bin/bash", tty=True, detach=True, ports=[ + ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT, ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT + ], host_config=hc) return container.get("Id") def stop_containers(self, container_ids): diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index c58778a..65e8049 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -7,6 +7,8 @@ import tempfile from DockerSSH import DockerSSH from collections import OrderedDict + +from MolnsLib import ssh_deploy from molns_provider import ProviderBase, ProviderException @@ -33,7 +35,12 @@ def start_instance(self, num=1): """ Start given number of (or 1) containers. """ started_containers = [] for i in range(num): - container_id = self.docker.create_container(self.provider.config["molns_image_name"]) + container_id = self.docker.create_container(self.provider.config["molns_image_name"], + port_bindings={ + ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT: + self.config['web_server_port'], + ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT: + self.config['notebook_port']}) stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=self.docker.get_container_ip_address(container_id) , provider_id=self.provider.id, controller_id=self.id, @@ -204,6 +211,12 @@ class DockerController(DockerBase): OBJ_NAME = 'DockerController' CONFIG_VARS = OrderedDict([ + ('web_server_port', + {'q': 'Port to use for web server', 'default': "8080", + 'ask': True}), + ('notebook_port', + {'q': 'Port to use for jupyter notebook', 'default': "8081", + 'ask': True}) ]) def get_instance_status(self, instance): diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index c3901ee..76bf90c 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -128,7 +128,7 @@ def get_provider_handle(kind, ptype): pkg_name = "MolnsLib.{0}Provider".format(ptype) if pkg_name not in sys.modules: logging.debug("loading {0} from {1}".format(cls_name, pkg_name)) - pkg = dynamic_module_import(pkg_name) + # pkg = dynamic_module_import(pkg_name) pkg = dynamic_module_import(pkg_name) try: #logging.debug("dir(pkg={0})={1}".format(pkg, dir(pkg))) diff --git a/MolnsLib/molns_landing_page.py b/MolnsLib/molns_landing_page.py new file mode 100644 index 0000000..14cdfe8 --- /dev/null +++ b/MolnsLib/molns_landing_page.py @@ -0,0 +1,91 @@ +class MolnsLandingPage: + def __init__(self, port): + self.molns_landing_page = """ + + + + + MOLNs + + + + + + +
+ +
+
+
+ +
+

MOLNs

+

A cloud computing appliance for spatial stochastic simulation of biochemical systems.

+

+ + + To the IPython Interface +

+ + + + +

Please note that due to the self-signed certification, you will see a warning before you can view the page. Please accept the warning and proceed.

+
+
+
+
+
+   +
+
+

+

+ +

+
+
+
+
+
+

Write PyURDME models as sharable IPython notebooks

+ + PyURDME API reference + +
+
+

Advanced analysis with Python scientific libraries

+ + +
+
+

Large scale computational experiments made easy

+ +
+
+ +
+
+
+
+

Visit MOLNs on GitHub

 +
+
+
+ + + + + +
+ +
+ + +""".format(port) diff --git a/MolnsLib/molns_provider.py b/MolnsLib/molns_provider.py index c62fed2..fbee62f 100644 --- a/MolnsLib/molns_provider.py +++ b/MolnsLib/molns_provider.py @@ -14,7 +14,8 @@ class ProviderBase: STATUS_STOPPED = 'stopped' STATUS_TERMINATED = 'terminated' - SecurityGroupRule = collections.namedtuple("SecurityGroupRule", ["ip_protocol", "from_port", "to_port", "cidr_ip", "src_group_name"]) + SecurityGroupRule = collections.namedtuple("SecurityGroupRule", ["ip_protocol", "from_port", "to_port", "cidr_ip", + "src_group_name"]) FIREWALL_RULES = [ SecurityGroupRule("tcp", "22", "22", "0.0.0.0/0", None), diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 69abe81..b4d31f0 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -246,8 +246,12 @@ def connect(self, instance, port=None): time.sleep(self.SSH_CONNECT_WAITTIME) raise SSHDeployException("ssh connect Failed!!!\t{0}:{1}".format(instance.ip_address, self.ssh_endpoint)) - def deploy_molns_webserver(self, instance): + def deploy_molns_webserver(self, instance, controller_obj): ip_address = instance.ip_address + + if instance.provider_type == Constants.DockerProvider: + ip_address = "0.0.0.0:{0}".format(controller_obj.config["web_server_port"]) + try: self.connect(instance, self.ssh_endpoint) self.ssh.exec_command("sudo rm -rf /usr/local/molns_webroot") @@ -255,6 +259,14 @@ def deploy_molns_webserver(self, instance): self.ssh.exec_command("sudo chown ubuntu /usr/local/molns_webroot") self.ssh.exec_command( "git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") + + # If DockerProvider, replace index page. + if instance.provider_type == Constants.DockerProvider: + from molns_landing_page import MolnsLandingPage + from pipes import quote + index_page = MolnsLandingPage(controller_obj.config["notebook_port"]).molns_landing_page + self.ssh.exec_command("echo {0} > /usr/local/molns_webroot/index.html".format(quote(index_page))) + self.ssh.exec_multi_command( "cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format( self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') @@ -271,7 +283,6 @@ def deploy_molns_webserver(self, instance): sys.stdout.flush() break except Exception as e: - # sys.stdout.write("{0}".format(e)) sys.stdout.write(".") sys.stdout.flush() time.sleep(1) @@ -403,7 +414,7 @@ def deploy_ipython_controller(self, instance, notebook_password=None): finally: self.ssh.close() - url = "http://%s" % (ip_address) + url = "https://%s" % (ip_address) print "\nThe URL for your MOLNs head node is: %s." % url def get_ipython_engine_file(self, ip_address): diff --git a/molns.py b/molns.py index 1d3f45b..e9ef0d4 100755 --- a/molns.py +++ b/molns.py @@ -461,7 +461,7 @@ def start_controller(cls, args, config, password=None): # deploying sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) sshdeploy.deploy_ipython_controller(inst, notebook_password=password) - sshdeploy.deploy_molns_webserver(inst) + sshdeploy.deploy_molns_webserver(inst, controller_obj) # sshdeploy.deploy_stochss(inst.ip_address, port=443) @classmethod From 29388c92d5959ae4d38bdb6e17a642d05066c1fa Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Fri, 14 Oct 2016 20:44:24 -0700 Subject: [PATCH 068/100] bug fix if the provider is not defined --- molns.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/molns.py b/molns.py index 0690387..7b85f72 100755 --- a/molns.py +++ b/molns.py @@ -1215,7 +1215,11 @@ def show_instances(cls, args, config): if len(instance_list) > 0: table_data = [] for i in instance_list: - provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + provider_obj = config.get_object_by_id(i.provider_id, 'Provider') + if provider_obj is None: + continue + provider_name = provider_obj.name + #print "provider_obj.type",provider_obj.type if i.worker_group_id is not None: name = config.get_object_by_id(i.worker_id, 'WorkerGroup').name itype = 'worker' From 968f14563a5d37e8a950d7d5f496bf8de81f489d Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sat, 15 Oct 2016 16:44:16 -0700 Subject: [PATCH 069/100] Removed provider_type from instance obj, for backwards compatability --- MolnsLib/molns_datastore.py | 3 +-- MolnsLib/ssh_deploy.py | 10 ++++++---- molns.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 76bf90c..ec4f251 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -90,7 +90,6 @@ class Instance(Base): provider_id = Column(Integer) ip_address = Column(String) provider_instance_identifier = Column(String) - provider_type = Column(String) def __str__(self): return "Instance({0}): provider_instance_identifier={1} provider_id={2} controller_id={3} worker_group_id={4}".format(self.id, self.provider_instance_identifier, self.provider_id, self.controller_id, self.worker_group_id) @@ -345,7 +344,7 @@ def get_instance(self, provider_instance_identifier, ip_address, provider_id=Non """ Create or get the value for an instance. """ p = self.session.query(Instance).filter_by(provider_instance_identifier=provider_instance_identifier).first() if p is None: - p = Instance(provider_instance_identifier=provider_instance_identifier, ip_address=ip_address, provider_id=provider_id, controller_id=controller_id, worker_group_id=worker_group_id, provider_type=provider_type) + p = Instance(provider_instance_identifier=provider_instance_identifier, ip_address=ip_address, provider_id=provider_id, controller_id=controller_id, worker_group_id=worker_group_id) self.session.add(p) self.session.commit() #logging.debug("Creating instance: {0}".format(p)) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index b4d31f0..e271b31 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -248,9 +248,11 @@ def connect(self, instance, port=None): def deploy_molns_webserver(self, instance, controller_obj): ip_address = instance.ip_address + logging.debug('deploy_molns_webserver(): controller_obj.provider.type={0}\n',controller_obj.provider.type) - if instance.provider_type == Constants.DockerProvider: + if controller_obj.provider.type == Constants.DockerProvider: ip_address = "0.0.0.0:{0}".format(controller_obj.config["web_server_port"]) + logging.debug('deploy_molns_webserver(): ip_address={0}\n',ip_address) try: self.connect(instance, self.ssh_endpoint) @@ -261,7 +263,7 @@ def deploy_molns_webserver(self, instance, controller_obj): "git clone https://github.com/Molns/MOLNS_web_landing_page.git /usr/local/molns_webroot") # If DockerProvider, replace index page. - if instance.provider_type == Constants.DockerProvider: + if controller_obj.provider.type == Constants.DockerProvider: from molns_landing_page import MolnsLandingPage from pipes import quote index_page = MolnsLandingPage(controller_obj.config["notebook_port"]).molns_landing_page @@ -351,7 +353,7 @@ def deploy_stochss(self, ip_address, port=1443): print "StochSS launch failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def deploy_ipython_controller(self, instance, notebook_password=None): + def deploy_ipython_controller(self, instance, controller_obj, notebook_password=None): ip_address = instance.ip_address try: @@ -386,7 +388,7 @@ def deploy_ipython_controller(self, instance, notebook_password=None): # If provider is Docker, then ipython controller and ipengines aren't started - if instance.provider_type != Constants.DockerProvider: + if controller_obj.provider.type != Constants.DockerProvider: Utils.Log.write_log("Provider type is NOT Docker.") self.ssh.exec_command( "source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2}--log-to-file".format( diff --git a/molns.py b/molns.py index e9ef0d4..fb2d528 100755 --- a/molns.py +++ b/molns.py @@ -460,7 +460,7 @@ def start_controller(cls, args, config, password=None): # deploying sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) - sshdeploy.deploy_ipython_controller(inst, notebook_password=password) + sshdeploy.deploy_ipython_controller(inst, controller_obj, notebook_password=password) sshdeploy.deploy_molns_webserver(inst, controller_obj) # sshdeploy.deploy_stochss(inst.ip_address, port=443) From 1e4a3aed30bf714bb7fdb12a7a5c199d6d8d2a65 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Sun, 16 Oct 2016 13:12:43 -0700 Subject: [PATCH 070/100] bug fix --- MolnsLib/installSoftware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index c2faf7e..ddfd83c 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -86,7 +86,7 @@ class InstallSW: "cd /usr/local/StochKit && ./install.sh", #"wget https://github.com/StochSS/stochss/blob/master/ode-1.0.4.tgz?raw=true -q -O /tmp/ode.tgz", - "wget https://github.com/StochSS/StochKit_ode/archive/master.tar.gz?raw=true -q -O /tmp/ode.tgz" + "wget https://github.com/StochSS/StochKit_ode/archive/master.tar.gz?raw=true -q -O /tmp/ode.tgz", "cd /tmp && tar -xzf /tmp/ode.tgz", "sudo mv /tmp/StochKit_ode-master /usr/local/ode", "rm /tmp/ode.tgz", From 89159d68a286e5f7eeef622fd5ac7fa0fc8882d6 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 21 Oct 2016 15:37:15 -0700 Subject: [PATCH 071/100] make DockerProvider data available as volume at user provided location --- MolnsLib/Docker.py | 81 +++++++++++++++++++++---- MolnsLib/DockerProvider.py | 44 ++++++++------ MolnsLib/DockerSSH.py | 8 ++- MolnsLib/Utils.py | 10 +++ MolnsLib/{Constants.py => constants.py} | 8 ++- MolnsLib/installSoftware.py | 6 +- MolnsLib/molns_landing_page.py | 6 +- MolnsLib/ssh_deploy.py | 56 ++++++++--------- molns.py | 19 +++--- 9 files changed, 167 insertions(+), 71 deletions(-) rename MolnsLib/{Constants.py => constants.py} (64%) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 6bd8e11..47867a0 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -2,10 +2,10 @@ import re import time -from MolnsLib import ssh_deploy -from MolnsLib.Utils import Log +from Utils import Log +import constants from molns_provider import ProviderBase -from Constants import Constants +from constants import Constants from docker import Client from docker.errors import NotFound, NullResource, APIError @@ -27,22 +27,67 @@ def __init__(self): self.build_count = 0 logging.basicConfig(level=logging.DEBUG) - def create_container(self, image_str, port_bindings={ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT: 8080, - ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT: 8081}): + @staticmethod + def get_container_volume_from_working_dir(working_directory): + import os + return os.path.join("/home/ubuntu/", os.path.basename(working_directory)) + + def create_container(self, image_str, working_directory=None, name=None, + port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: 8080, + Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: 8081}): """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container to 8080 of locahost by default""" docker_image = DockerImage.from_string(image_str) + volume_dir = Docker.get_container_volume_from_working_dir(working_directory) + if name is None: + import uuid + random_str = str(uuid.uuid4()) + name = constants.Constants.MolnsDockerContainerNamePrefix + random_str[:8] image = docker_image.image_id if docker_image.image_id is not Constants.DockerNonExistentTag \ else docker_image.image_tag Log.write_log("Using image {0}".format(image)) - hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) - container = self.client.create_container(image=image, command="/bin/bash", tty=True, detach=True, ports=[ - ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT, ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT - ], host_config=hc) - return container.get("Id") + import os + if Docker._verify_directory(working_directory) is False: + Log.write_log(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " + "be created.") + hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) + container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True, + ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, + Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT], + host_config=hc) + + else: + container_mount_point = '/home/ubuntu/{0}'.format(os.path.basename(working_directory)) + hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings, + binds={working_directory: {'bind': container_mount_point, + 'mode': 'rw'}}) + + container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True, + ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, + Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT], + volumes=container_mount_point, host_config=hc, + working_dir=volume_dir) + + container_id = container.get("Id") + + return container_id + + # noinspection PyBroadException + @staticmethod + def _verify_directory(working_directory): + import os, Utils + if working_directory is None: + return False + try: + if not os.path.exists(working_directory): + os.makedirs(working_directory) + os.chown(working_directory, Utils.get_user_id(), Utils.get_group_id()) + return True + except: + return False def stop_containers(self, container_ids): """Stops given containers.""" @@ -98,8 +143,10 @@ def execute_command(self, container_id, command): def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ + import uuid Log.write_log(Docker.LOG_TAG + "Building image...") - image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(self.build_count) + random_string = str(uuid.uuid4()) + image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(random_string[:]) last_line = "" try: for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag): @@ -148,6 +195,14 @@ def terminate_containers(self, container_ids): def terminate_container(self, container_id): self.client.remove_container(container_id) + def get_working_directory(self, container_id): + return self.client.inspect_container(container_id)["Config"]["WorkingDir"] + + def get_home_directory(self, container_id): + env_vars = self.client.inspect_container(container_id)["Config"]["Env"] + home = [i for i in env_vars if i.startswith("HOME")] + return home[0].split("=")[1] + def put_archive(self, container_id, tar_file_bytes, target_path_in_container): """ Copies and unpacks a given tarfile in the container at specified location. Location must exist in container.""" @@ -157,8 +212,10 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): # Prepend file path with /home/ubuntu/. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): - target_path_in_container = "/home/ubuntu/" + target_path_in_container + import os + target_path_in_container = os.path.join("/home/ubuntu/", target_path_in_container) + Log.write_log("target path in container: {0}".format(target_path_in_container)) if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): Log.write_log(Docker.LOG_TAG + "Failed to copy.") diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 65e8049..e2ec401 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -1,4 +1,5 @@ -import Constants +import constants +from constants import Constants import logging import time import os @@ -35,16 +36,17 @@ def start_instance(self, num=1): """ Start given number of (or 1) containers. """ started_containers = [] for i in range(num): - container_id = self.docker.create_container(self.provider.config["molns_image_name"], + container_id = self.docker.create_container(self.provider.config["molns_image_name"], name=self.name, port_bindings={ - ssh_deploy.SSHDeploy.DEFAULT_PUBLIC_WEBSERVER_PORT: + Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: self.config['web_server_port'], - ssh_deploy.SSHDeploy.DEFAULT_PRIVATE_NOTEBOOK_PORT: - self.config['notebook_port']}) + Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: + self.config['notebook_port']}, + working_directory=self.config["working_directory"]) stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=self.docker.get_container_ip_address(container_id) , provider_id=self.provider.id, controller_id=self.id, - provider_type=Constants.Constants.DockerProvider) + provider_type=constants.Constants.DockerProvider) started_containers.append(stored_container) if num == 1: return started_containers[0] @@ -86,9 +88,10 @@ class DockerProvider(DockerBase): OBJ_NAME = 'DockerProvider' + # TODO ask user for directory to store stuff at CONFIG_VARS = OrderedDict([ ('ubuntu_image_name', - {'q': 'Base Ubuntu image to use', 'default': Constants.Constants.DOCKER_DEFAULT_IMAGE, + {'q': 'Base Ubuntu image to use', 'default': constants.Constants.DOCKER_DEFAULT_IMAGE, 'ask': True}), ('molns_image_name', {'q': 'Local MOLNs image (Docker image ID or image tag) to use ', 'default': None, 'ask': True}), @@ -99,7 +102,7 @@ class DockerProvider(DockerBase): ('login_username', {'default': 'ubuntu', 'ask': False}), # Unused. ('provider_type', - {'default': Constants.Constants.DockerProvider, 'ask': False}) + {'default': constants.Constants.DockerProvider, 'ask': False}) ]) def get_config_credentials(self): @@ -108,7 +111,7 @@ def get_config_credentials(self): @staticmethod def __get_new_dockerfile_name(): import uuid - filename = Constants.Constants.DOCKERFILE_NAME + str(uuid.uuid4()) + filename = constants.Constants.DOCKERFILE_NAME + str(uuid.uuid4()) return filename def check_ssh_key(self): @@ -153,14 +156,14 @@ def check_molns_image(self): def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ - import pwd + import Utils - user_id = pwd.getpwnam(os.environ['SUDO_USER']).pw_uid + user_id = Utils.get_user_id() dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n\n# Add user ubuntu.\nRUN useradd -u {0} -ms /bin/bash ubuntu\n # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \ \n sudo \ \n screen \ \n iptables \nRUN echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers - \nWORKDIR /home/ubuntu\n'''.format(user_id) + \nWORKDIR /home/ubuntu\n\nUSER ubuntu\nENV HOME /home/ubuntu'''.format(user_id) flag = False @@ -183,7 +186,7 @@ def _create_dockerfile(self, commands): else: dockerfile += ''' && \ \n ''' + self._preprocess(entry) - dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu\nVOLUME /home/ubuntu\n''' + dockerfile += '''\n\n\n''' dockerfile_file = DockerProvider.__get_new_dockerfile_name() with open(dockerfile_file, 'w') as Dockerfile: @@ -196,27 +199,34 @@ def _create_dockerfile(self, commands): @staticmethod def _preprocess(command): - """ Filters out any "sudo" in the command, prepends "shell only" commands with '/bin/bash -c'. """ + """ Prepends "shell only" commands with '/bin/bash -c'. """ for shell_command in Docker.Docker.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command command = command.replace(shell_command, replace_string) command += "\"" - return command.replace("sudo", "") + return command + + +def get_default_working_directory(config=None): + if config is None: + raise Exception("Config should not be None.") + return os.path.join(config.config_dir, "docker_controller_working_dirs", config.name) class DockerController(DockerBase): """ Provider handle for a Docker controller. """ OBJ_NAME = 'DockerController' - CONFIG_VARS = OrderedDict([ ('web_server_port', {'q': 'Port to use for web server', 'default': "8080", 'ask': True}), ('notebook_port', {'q': 'Port to use for jupyter notebook', 'default': "8081", - 'ask': True}) + 'ask': True}), + ('working_directory', + {'q': 'Working directory for this controller', 'default': get_default_working_directory, 'ask': True}) ]) def get_instance_status(self, instance): diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 70e8a7b..4f77c96 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -63,7 +63,9 @@ def write(self, write_this): def close(self): # Make tarfile. - temp_tar = "transport.tar" + import uuid + rand_str = str(uuid.uuid4()) + temp_tar = "transport-{0}.tar".format(rand_str[:8]) tar = tarfile.TarFile(temp_tar, "w") string = StringIO.StringIO() string.write(self.file_contents) @@ -75,8 +77,12 @@ def close(self): path_to_file = os.path.dirname(self.filename) + if not path_to_file.startswith("/home"): + path_to_file = os.path.join(self.docker.get_home_directory(self.container_id), path_to_file) + with open(temp_tar, mode='rb') as f: tar_file_bytes = f.read() + Log.write_log("path to file: {0}".format(path_to_file)) self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 2ac0432..0c43534 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -1,3 +1,13 @@ +def get_user_id(): + import pwd, os + return pwd.getpwnam(os.environ['SUDO_USER']).pw_uid + + +def get_group_id(): + import grp, os + return grp.getgrnam((os.environ['SUDO_USER'])).gr_gid + + class Log: verbose = True diff --git a/MolnsLib/Constants.py b/MolnsLib/constants.py similarity index 64% rename from MolnsLib/Constants.py rename to MolnsLib/constants.py index 64e4b4a..2c1164c 100644 --- a/MolnsLib/Constants.py +++ b/MolnsLib/constants.py @@ -7,8 +7,14 @@ class Constants: DOCKER_CONTAINER_EXITED = "exited" DOCKERFILE_NAME = "dockerfile_" DOKCER_IMAGE_ID_LENGTH = 12 - DOCKER_IMAGE_PREFIX = "aviralcse/docker-provider-" + DOCKER_IMAGE_PREFIX = "molns-docker-provider-" DOCKER_PY_IMAGE_ID_PREFIX_LENGTH = 7 DockerProvider = "Docker" DockerNonExistentTag = "**NA**" DockerImageDelimiter = "|||" + MolnsDockerContainerNamePrefix = "Molns-" + DEFAULT_PRIVATE_NOTEBOOK_PORT = 8081 + DEFAULT_PUBLIC_NOTEBOOK_PORT = 443 + DEFAULT_PRIVATE_WEBSERVER_PORT = 8001 + DEFAULT_PUBLIC_WEBSERVER_PORT = 80 + diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index c6f619e..b352a3e 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -46,7 +46,7 @@ class InstallSW: # EC2/S3 and OpenStack APIs "sudo pip install boto", "sudo apt-get -y install pandoc", - # This set of packages is needed for OpenStack, as molnsutil uses them for hybrid cloud deployment + # This set of packages is needed for OpenStack, as molns_util uses them for hybrid cloud deployment "sudo apt-get -y install libxml2-dev libxslt1-dev python-dev", "sudo pip install python-novaclient", "sudo easy_install -U pip", @@ -112,7 +112,7 @@ class InstallSW: [ "sudo rm -rf /usr/local/pyurdme && sudo mkdir -p /usr/local/pyurdme && sudo chown ubuntu /usr/local/pyurdme", "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", #"cd /usr/local/pyurdme && git checkout develop", # for development only - "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* $HOME/.ipython/profile_default/static/custom/", # TODO added $HOME here. Is it okay? + "cp /usr/local/pyurdme/pyurdme/data/three.js_templates/js/* $HOME/.ipython/profile_default/static/custom/", "source /usr/local/pyurdme/pyurdme_init && python -c 'import pyurdme'", ], @@ -129,7 +129,7 @@ class InstallSW: "sudo pip install jsonschema jsonpointer", #redo this install to be sure it has not been removed. - "sync", # This is critial for some infrastructures. + "sync", # This is critical for some infrastructures. ] # How many time do we try to install each package. diff --git a/MolnsLib/molns_landing_page.py b/MolnsLib/molns_landing_page.py index 14cdfe8..0238f26 100644 --- a/MolnsLib/molns_landing_page.py +++ b/MolnsLib/molns_landing_page.py @@ -1,6 +1,8 @@ +from pipes import quote + class MolnsLandingPage: def __init__(self, port): - self.molns_landing_page = """ + self.molns_landing_page = quote(""" @@ -88,4 +90,4 @@ def __init__(self, port):
-""".format(port) +""".format(port)) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index e271b31..52c5a8e 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -9,7 +9,9 @@ import webbrowser import urllib2 -from MolnsLib.Constants import Constants +from constants import Constants + +from Docker import Docker from ssh import SSH from DockerSSH import DockerSSH @@ -25,10 +27,6 @@ class SSHDeploy: DEFAULT_STOCHSS_PORT = 1443 DEFAULT_INTERNAL_STOCHSS_PORT = 8080 DEFAULT_GAE_ADMIN_PORT = 8000 - DEFAULT_PRIVATE_NOTEBOOK_PORT = 8081 - DEFAULT_PUBLIC_NOTEBOOK_PORT = 443 - DEFAULT_PRIVATE_WEBSERVER_PORT = 8001 - DEFAULT_PUBLIC_WEBSERVER_PORT = 80 SSH_CONNECT_WAITTIME = 5 MAX_NUMBER_SSH_CONNECT_ATTEMPTS = 25 DEFAULT_SSH_PORT = 22 @@ -44,7 +42,7 @@ def __init__(self, ssh, config=None, config_dir=None): if config_dir is None: self.config_dir = os.path.join(os.path.dirname(__file__), '/../.molns/') self.username = config['login_username'] - self.endpoint = self.DEFAULT_PRIVATE_NOTEBOOK_PORT + self.endpoint = Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT self.ssh_endpoint = self.DEFAULT_SSH_PORT self.keyfile = config.sshkeyfilename() if not (isinstance(ssh, SSH) or isinstance(ssh, DockerSSH)): @@ -265,16 +263,15 @@ def deploy_molns_webserver(self, instance, controller_obj): # If DockerProvider, replace index page. if controller_obj.provider.type == Constants.DockerProvider: from molns_landing_page import MolnsLandingPage - from pipes import quote index_page = MolnsLandingPage(controller_obj.config["notebook_port"]).molns_landing_page - self.ssh.exec_command("echo {0} > /usr/local/molns_webroot/index.html".format(quote(index_page))) + self.ssh.exec_command("echo {0} > /usr/local/molns_webroot/index.html".format(index_page)) self.ssh.exec_multi_command( "cd /usr/local/molns_webroot; python -m SimpleHTTPServer {0} > ~/.molns_webserver.log 2>&1 &".format( - self.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') + Constants.DEFAULT_PRIVATE_WEBSERVER_PORT), '\n') self.ssh.exec_command( "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( - self.DEFAULT_PUBLIC_WEBSERVER_PORT, self.DEFAULT_PRIVATE_WEBSERVER_PORT)) + Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, Constants.DEFAULT_PRIVATE_WEBSERVER_PORT)) self.ssh.close() print "Deploying MOLNs webserver" url = "http://{0}/".format(ip_address) @@ -353,7 +350,7 @@ def deploy_stochss(self, ip_address, port=1443): print "StochSS launch failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - def deploy_ipython_controller(self, instance, controller_obj, notebook_password=None): + def deploy_ipython_controller(self, instance, controller_obj, notebook_password=None, resume=False): ip_address = instance.ip_address try: @@ -378,20 +375,30 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.ssh.exec_command("sudo mkdir -p {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) self.ssh.exec_command("sudo chown ubuntu {0}".format(self.DEFAULT_PYURDME_TEMPDIR)) # - # self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") - self.ssh.exec_command("mkdir -p .molns") - self.create_s3_config() + # self.exec_command("cd /usr/local/molns_util && git pull && sudo python setup.py install") - self.ssh.exec_command("ipython profile create {0}".format(self.profile)) - self.create_ipython_config(ip_address, notebook_password) - self.create_engine_config() + home_dir = "" + if controller_obj.provider.type == Constants.DockerProvider: + home_dir = "/home/ubuntu/" + + # If its not a DockerController being resumed, then create config files and move sample notebooks to volume. + if not (controller_obj.provider.type == Constants.DockerProvider and resume is True): + self.ssh.exec_command("mkdir -p {0}.molns".format(home_dir)) + self.create_s3_config() + self.ssh.exec_command("ipython profile create {0}".format(self.profile)) + self.create_ipython_config(ip_address, notebook_password) + self.create_engine_config() + if controller_obj.provider.type == Constants.DockerProvider: + self.ssh.exec_command("mv {0}*.ipynb {1}".format(home_dir, + Docker.get_container_volume_from_working_dir( + controller_obj.config["working_directory"]))) # If provider is Docker, then ipython controller and ipengines aren't started if controller_obj.provider.type != Constants.DockerProvider: - Utils.Log.write_log("Provider type is NOT Docker.") self.ssh.exec_command( - "source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} --port={2}--log-to-file".format( + "source /usr/local/pyurdme/pyurdme_init; screen -d -m ipcontroller --profile={1} --ip='*' --location={0} " + "--port={2}--log-to-file".format( ip_address, self.profile, self.ipython_port), '\n') # Start one ipengine per processor num_procs = self.get_number_processors() @@ -407,7 +414,7 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.ssh.exec_command( "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( - self.DEFAULT_PUBLIC_NOTEBOOK_PORT, self.DEFAULT_PRIVATE_NOTEBOOK_PORT)) + Constants.DEFAULT_PUBLIC_NOTEBOOK_PORT, Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT)) except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) @@ -478,8 +485,8 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont "sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format( controler_ip)) - # Update the Molnsutil package: TODO remove when molnsutil is stable - # self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") + # Update the Molnsutil package: TODO remove when molns_util is stable + # self.exec_command("cd /usr/local/molns_util && git pull && sudo python setup.py install") self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_engine_config() @@ -496,8 +503,3 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont except Exception as e: print "Failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] - - -if __name__ == "__main__": - sshdeploy = SSHDeploy() - sshdeploy.deploy_ipython_controller() diff --git a/molns.py b/molns.py index fb2d528..e613443 100755 --- a/molns.py +++ b/molns.py @@ -2,7 +2,6 @@ import os import sys -from MolnsLib.Constants import Constants from MolnsLib.Utils import Log from MolnsLib.molns_datastore import Datastore, DatastoreException, VALID_PROVIDER_TYPES, get_provider_handle from MolnsLib.molns_provider import ProviderException @@ -12,6 +11,8 @@ import json import logging +from MolnsLib import constants + logger = logging.getLogger() # logger.setLevel(logging.INFO) #for Debugging logger.setLevel(logging.CRITICAL) @@ -434,6 +435,7 @@ def status_controller(cls, args, config): @classmethod def start_controller(cls, args, config, password=None): """ Start the MOLNs controller. """ + resume = False logging.debug("MOLNSController.start_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) if controller_obj is None: @@ -452,6 +454,7 @@ def start_controller(cls, args, config, password=None): print "Resuming instance at {0}".format(i.ip_address) controller_obj.resume_instance(i) inst = i + resume=True break if inst is None: # Start a new instance @@ -460,7 +463,7 @@ def start_controller(cls, args, config, password=None): # deploying sshdeploy = SSHDeploy(controller_obj.ssh, config=controller_obj.provider, config_dir=config.config_dir) - sshdeploy.deploy_ipython_controller(inst, controller_obj, notebook_password=password) + sshdeploy.deploy_ipython_controller(inst, controller_obj, notebook_password=password, resume=resume) sshdeploy.deploy_molns_webserver(inst, controller_obj) # sshdeploy.deploy_stochss(inst.ip_address, port=443) @@ -1530,7 +1533,7 @@ def run(self, args, config_dir=None): ] -def printHelp(): +def print_help(): print "molns " print " --config=[Config Directory=./.molns/]" print "\tSpecify an alternate config location. (Must be first argument.)" @@ -1538,11 +1541,11 @@ def printHelp(): print c -def parseArgs(): +def parse_args(): if len(sys.argv) < 2 or sys.argv[1] == '-h': - printHelp() + print_help() return - Log.verbose = False + Log.verbose = True arg_list = sys.argv[1:] config_dir = './.molns/' @@ -1559,7 +1562,7 @@ def parseArgs(): arg_list = arg_list[1:] if len(arg_list) == 0 or arg_list[0] == 'help' or arg_list[0] == '-h': - printHelp() + print_help() return if arg_list[0] in COMMAND_LIST: @@ -1580,4 +1583,4 @@ def parseArgs(): if __name__ == "__main__": - parseArgs() + parse_args() From 5a052a39c13cb1a5fc0f1fa10fc83e34a503ce7a Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 25 Oct 2016 11:33:23 -0700 Subject: [PATCH 072/100] decorate Docker output --- MolnsLib/Docker.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 47867a0..ba56fc2 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -14,7 +14,7 @@ class Docker: """ A wrapper over docker-py and some utility methods and classes. """ - LOG_TAG = "Docker" + LOG_TAG = "Docker " shell_commands = ["source"] @@ -150,7 +150,7 @@ def build_image(self, dockerfile): last_line = "" try: for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag): - print(line) + print(Docker._decorate(line)) if "errorDetail" in line: raise Docker.ImageBuildException() last_line = line @@ -164,6 +164,10 @@ def build_image(self, dockerfile): except (Docker.ImageBuildException, IndexError) as e: raise Docker.ImageBuildException(e) + @staticmethod + def _decorate(some_line): + return some_line[11:-4].rstrip() + def image_exists(self, image_str): """Checks if an image with the given ID/tag exists locally.""" docker_image = DockerImage.from_string(image_str) From 083eb23cf14fb692d1e7af0f2eaab0ce9c54421d Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 25 Oct 2016 11:40:33 -0700 Subject: [PATCH 073/100] remove test file --- docker_test.py | 87 -------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 docker_test.py diff --git a/docker_test.py b/docker_test.py deleted file mode 100644 index f75d735..0000000 --- a/docker_test.py +++ /dev/null @@ -1,87 +0,0 @@ -from MolnsLib.Docker import Docker -import tempfile -import MolnsLib.installSoftware -from MolnsLib.DockerProvider import DockerProvider - -commands = MolnsLib.installSoftware.InstallSW.command_list - -docker = Docker() -# -# container = docker.create_container() -# -# docker.start_container(container) -# -# print "is container running: {0}".format(docker.is_container_running(container)) -# -# -# for entry in command_list: -# if isinstance(entry, list): -# for sub_entry in entry: -# ret_val, response = docker.execute_command(container, sub_entry) -# print "RETURN VALUE: {0}".format(ret_val) -# if ret_val is None or ret_val.get('ExitCode') != 0: -# print "ERROR" -# print "RESPONSE: {0}".format(response) -# print "__" -# print "__" -# -# else: -# ret_val, response = docker.execute_command(container, entry) -# print "RETURN VALUE: {0}".format(ret_val) -# if ret_val is None or ret_val.get('ExitCode') != 0: -# print "ERROR" -# print "RESPONSE: {0}".format(response) -# print "__" -# print "__" -def create_dockerfile(commands): - dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n# Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \n# Add user ubuntu.\nRUN useradd -ms /bin/bash ubuntu\nWORKDIR /home/ubuntu''' - - flag = False - - for entry in commands: - if isinstance(entry, list): - dockerfile += '''\n\nRUN ''' - first = True - flag = False - for sub_entry in entry: - if first is True: - dockerfile += _preprocess(sub_entry) - first = False - else: - dockerfile += ''' && \ \n ''' + _preprocess(sub_entry) - else: - if flag is False: - dockerfile += '''\n\nRUN ''' - flag = True - dockerfile += _preprocess(entry) - else: - dockerfile += ''' && \ \n ''' + _preprocess(entry) - - dockerfile += '''\n\nUSER ubuntu\nENV HOME /home/ubuntu''' - - return dockerfile - - -def _preprocess(command): - """ Filters out any sudos in the command, prepends shell only commands with '/bin/bash -c'. """ - for shell_command in Docker.shell_commands: - if shell_command in command: - replace_string = "/bin/bash -c \"" + shell_command - command = command.replace(shell_command, replace_string) - command += "\"" - return command.replace("sudo", "") - -print("Creating Dockerfile...") -dockerfile = create_dockerfile(commands) -print("---------------Dockerfile----------------") -print(dockerfile) -print("-----------------------------------------") -print("Building image...") -tmpfh = tempfile.NamedTemporaryFile() -tmpfh.write(dockerfile) -print("tmph name: " + tmpfh.name) -tmpfh.seek(0) -image_tag = docker.build_image(tmpfh) - -print("Image created.") - From b0d9bc6257c9072ff1dd7d0e4f132fafdbb7ff5a Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 8 Nov 2016 16:28:32 -0800 Subject: [PATCH 074/100] ensure sudo mode for docker --- MolnsLib/Docker.py | 52 ++++++++++++++++++++++++++++++++------ MolnsLib/DockerProvider.py | 20 +++++++-------- MolnsLib/Utils.py | 16 ++++++++++-- MolnsLib/constants.py | 9 ++++++- 4 files changed, 75 insertions(+), 22 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index ba56fc2..19f233a 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -2,7 +2,7 @@ import re import time -from Utils import Log +from Utils import Log, ensure_sudo_mode import constants from molns_provider import ProviderBase from constants import Constants @@ -10,6 +10,10 @@ from docker.errors import NotFound, NullResource, APIError +class InvalidVolumeName(Exception): + pass + + class Docker: """ A wrapper over docker-py and some utility methods and classes. """ @@ -22,6 +26,7 @@ class ImageBuildException(Exception): def __init__(self, message=None): super("Something went wrong while building docker container image.\n{0}".format(message)) + @ensure_sudo_mode def __init__(self): self.client = Client(base_url=Constants.DOCKER_BASE_URL) self.build_count = 0 @@ -32,9 +37,10 @@ def get_container_volume_from_working_dir(working_directory): import os return os.path.join("/home/ubuntu/", os.path.basename(working_directory)) + @ensure_sudo_mode def create_container(self, image_str, working_directory=None, name=None, - port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: 8080, - Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: 8081}): + port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: ('127.0.0.1', 8080), + Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: ('127.0.0.1', 8081)}): """Creates a new container with elevated privileges. Returns the container ID. Maps port 80 of container to 8080 of locahost by default""" @@ -51,6 +57,9 @@ def create_container(self, image_str, working_directory=None, name=None, Log.write_log("Using image {0}".format(image)) import os if Docker._verify_directory(working_directory) is False: + if working_directory is not None: + raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the controller environment, and unfortunately the provided name for working directory of the controller cannot be one of these. Please configure this controller again with a different volume name and retry. Here is a list of forbidden names: \n{0}".format(Constants.ForbiddenVolumeNames)) + Log.write_log(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " "be created.") hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) @@ -76,28 +85,32 @@ def create_container(self, image_str, working_directory=None, name=None, return container_id # noinspection PyBroadException + @ensure_sudo_mode @staticmethod def _verify_directory(working_directory): import os, Utils - if working_directory is None: + if working_directory is None or os.path.basename(working_directory) in Constants.ForbiddenVolumeNames: return False try: if not os.path.exists(working_directory): os.makedirs(working_directory) - os.chown(working_directory, Utils.get_user_id(), Utils.get_group_id()) + os.chown(working_directory, Utils.get_sudo_user_id(), Utils.get_sudo_group_id()) return True except: return False + @ensure_sudo_mode def stop_containers(self, container_ids): """Stops given containers.""" for container_id in container_ids: self.stop_container(container_id) + @ensure_sudo_mode def stop_container(self, container_id): """Stops the container with given ID.""" self.client.stop(container_id) + @ensure_sudo_mode def container_status(self, container_id): """Checks if container with given ID running.""" status = ProviderBase.STATUS_TERMINATED @@ -111,11 +124,13 @@ def container_status(self, container_id): pass return status + @ensure_sudo_mode def start_containers(self, container_ids): """Starts each container in given list of container IDs.""" for container_id in container_ids: self.start_container(container_id) + @ensure_sudo_mode def start_container(self, container_id): """ Start the container with given ID.""" Log.write_log(Docker.LOG_TAG + " Starting container " + container_id) @@ -126,6 +141,7 @@ def start_container(self, container_id): return False return True + @ensure_sudo_mode def execute_command(self, container_id, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" run_command = "/bin/bash -c \"" + command + "\"" @@ -141,6 +157,7 @@ def execute_command(self, container_id, command): Log.write_log(Docker.LOG_TAG + " Could not execute command.", e) return None + @ensure_sudo_mode def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ import uuid @@ -168,6 +185,7 @@ def build_image(self, dockerfile): def _decorate(some_line): return some_line[11:-4].rstrip() + @ensure_sudo_mode def image_exists(self, image_str): """Checks if an image with the given ID/tag exists locally.""" docker_image = DockerImage.from_string(image_str) @@ -186,6 +204,7 @@ def image_exists(self, image_str): return True return False + @ensure_sudo_mode def terminate_containers(self, container_ids): """ Terminates containers with given container ids.""" for container_id in container_ids: @@ -196,23 +215,39 @@ def terminate_containers(self, container_ids): except NotFound: pass + @ensure_sudo_mode def terminate_container(self, container_id): self.client.remove_container(container_id) + @ensure_sudo_mode + def get_mapped_ports(self, container_id): + container_ins = self.client.inspect_container(container_id) + mapped_ports = container_ins['HostConfig']['PortBindings'] + ret_val = [] + if mapped_ports is None: + Log.write_log("No mapped ports for {0}".format(container_id)) + return + for k, v in mapped_ports.iteritems(): + host_port = v[0]['HostPort'] + ret_val.append(host_port) + return ret_val + + @ensure_sudo_mode def get_working_directory(self, container_id): return self.client.inspect_container(container_id)["Config"]["WorkingDir"] + @ensure_sudo_mode def get_home_directory(self, container_id): env_vars = self.client.inspect_container(container_id)["Config"]["Env"] home = [i for i in env_vars if i.startswith("HOME")] return home[0].split("=")[1] + @ensure_sudo_mode def put_archive(self, container_id, tar_file_bytes, target_path_in_container): """ Copies and unpacks a given tarfile in the container at specified location. Location must exist in container.""" if self.start_container(container_id) is False: - Log.write_log(Docker.LOG_TAG + "ERROR Could not start container.") - return + raise Exception("ERROR Could not start container.") # Prepend file path with /home/ubuntu/. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): @@ -223,6 +258,7 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): Log.write_log(Docker.LOG_TAG + "Failed to copy.") + @ensure_sudo_mode def get_container_ip_address(self, container_id): """ Returns the IP Address of given container.""" self.start_container(container_id) @@ -289,4 +325,4 @@ def looks_like_image_id(some_string): if some_string is possible_image_id: return True else: - return False + return False \ No newline at end of file diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index e2ec401..5b25e68 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -1,15 +1,14 @@ -import constants -from constants import Constants import logging -import time import os -import Docker -import installSoftware import tempfile -from DockerSSH import DockerSSH +import time from collections import OrderedDict -from MolnsLib import ssh_deploy +import Docker +import constants +import installSoftware +from DockerSSH import DockerSSH +from constants import Constants from molns_provider import ProviderBase, ProviderException @@ -39,9 +38,9 @@ def start_instance(self, num=1): container_id = self.docker.create_container(self.provider.config["molns_image_name"], name=self.name, port_bindings={ Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: - self.config['web_server_port'], + ('127.0.0.1', self.config['web_server_port']), Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: - self.config['notebook_port']}, + ('127.0.0.1', self.config['notebook_port'])}, working_directory=self.config["working_directory"]) stored_container = self.datastore.get_instance(provider_instance_identifier=container_id, ip_address=self.docker.get_container_ip_address(container_id) @@ -88,7 +87,6 @@ class DockerProvider(DockerBase): OBJ_NAME = 'DockerProvider' - # TODO ask user for directory to store stuff at CONFIG_VARS = OrderedDict([ ('ubuntu_image_name', {'q': 'Base Ubuntu image to use', 'default': constants.Constants.DOCKER_DEFAULT_IMAGE, @@ -158,7 +156,7 @@ def _create_dockerfile(self, commands): """ Create Dockerfile from given commands. """ import Utils - user_id = Utils.get_user_id() + user_id = Utils.get_sudo_user_id() dockerfile = '''FROM ubuntu:14.04\nRUN apt-get update\n\n# Add user ubuntu.\nRUN useradd -u {0} -ms /bin/bash ubuntu\n # Set up base environment.\nRUN apt-get install -yy \ \n software-properties-common \ \n python-software-properties \ \n wget \ \n curl \ \n git \ \n ipython \ \n sudo \ \n diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 0c43534..3931260 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -1,13 +1,21 @@ -def get_user_id(): +def get_sudo_user_id(): import pwd, os return pwd.getpwnam(os.environ['SUDO_USER']).pw_uid -def get_group_id(): +def get_sudo_group_id(): import grp, os return grp.getgrnam((os.environ['SUDO_USER'])).gr_gid +def ensure_sudo_mode(some_function): + import os + import sys + if sys.platform.startswith("linux") and os.getuid() != 0: + raise NoPrivilegedMode("\n\nOn Linux platforms, 'docker' is a priviledged command. To use 'docker' functionality, please run in sudo mode or as root user.") + return some_function + + class Log: verbose = True @@ -18,3 +26,7 @@ def __init__(self): def write_log(message): if Log.verbose: print message + + +class NoPrivilegedMode(Exception): + pass diff --git a/MolnsLib/constants.py b/MolnsLib/constants.py index 2c1164c..a85601d 100644 --- a/MolnsLib/constants.py +++ b/MolnsLib/constants.py @@ -1,4 +1,8 @@ +import os + + class Constants: + DockerWorkingDirectoryPrefix = "working_dir_" LOGGING_DIRECTORY = "~/MOLNS_LOG" DOCKER_BASE_URL = "unix://var/run/docker.sock" DOCKER_DEFAULT_IMAGE = "ubuntu:latest" @@ -13,8 +17,11 @@ class Constants: DockerNonExistentTag = "**NA**" DockerImageDelimiter = "|||" MolnsDockerContainerNamePrefix = "Molns-" + MolnsExecHelper = "molns_exec_helper.py" DEFAULT_PRIVATE_NOTEBOOK_PORT = 8081 DEFAULT_PUBLIC_NOTEBOOK_PORT = 443 DEFAULT_PRIVATE_WEBSERVER_PORT = 8001 DEFAULT_PUBLIC_WEBSERVER_PORT = 80 - + DEFAULT_QSUB_SSH_PORT = 22 + ForbiddenVolumeNames = [".ssh", ".ipython", ".molns", "ipython", "localarea", "shared"] + ConfigDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".molns/") From b5cde243d48bbc1b453f7c75ede85ce1d9753e4d Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 8 Nov 2016 16:34:23 -0800 Subject: [PATCH 075/100] correctly incorporate all changes --- MolnsLib/DockerSSH.py | 15 ++++++++++----- MolnsLib/ssh.py | 5 ++++- __init__.py | 0 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 __init__.py diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 4f77c96..3b642f6 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -5,8 +5,6 @@ # "unused" arguments to some methods are added to maintain compatibility with existing upper level APIs. -from MolnsLib.Utils import Log - class DockerSSH(object): def __init__(self, docker): @@ -27,6 +25,9 @@ def open_sftp(self): def connect(self, instance, endpoint, username=None, key_filename=None): self.container_id = instance.provider_instance_identifier + def connect_cluster_node(self, ip_address, port, username, keyfile): + raise DockerSSHException("This invocation has been in error.") + def close(self): self.container_id = None @@ -35,6 +36,10 @@ class MockSFTPFileException(Exception): pass +class DockerSSHException(Exception): + pass + + class MockSFTP: def __init__(self, docker, container_id): self.docker = docker @@ -55,8 +60,8 @@ def __init__(self, filename, flag, docker, container_id): self.container_id = container_id if flag is 'w': self.flag = flag - else: - Log.write_log("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) + # else: + # print("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) def write(self, write_this): self.file_contents += write_this @@ -83,6 +88,6 @@ def close(self): with open(temp_tar, mode='rb') as f: tar_file_bytes = f.read() - Log.write_log("path to file: {0}".format(path_to_file)) + # print("path to file: {0}".format(path_to_file)) self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/ssh.py b/MolnsLib/ssh.py index 4a0ae95..d4f7d0a 100644 --- a/MolnsLib/ssh.py +++ b/MolnsLib/ssh.py @@ -64,5 +64,8 @@ def open_sftp(self): def connect(self, instance, port, username=None, key_filename=None): return self.ssh.connect(instance.ip_address, port, username, key_filename=key_filename) + def connect_cluster_node(self, ip_address, port, username, key_filename): + return self.ssh.connect(ip_address, port, username, key_filename=key_filename) + def close(self): - self.ssh.close() + self.ssh.close() \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 From a9a56afa382ddd6183786254f05ffca4f83826bc Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 11 Nov 2016 11:35:47 -0800 Subject: [PATCH 076/100] install molns and cluster_execution --- MolnsLib/ssh_deploy.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 52c5a8e..c2cf86a 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -405,12 +405,24 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= num_engines = num_procs - 2 for _ in range(num_engines): self.ssh.exec_command( - "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipengine --profile={0} --debug".format( + "{1}source /usr/local/pyurdme/; screen -d -m ipengine --profile={0} --debug".format( self.profile, self.ipengine_env)) + self.ssh.exec_command( + "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format( + self.profile, self.ipengine_env)) + else: + print "pip installing pyurdme..." + self.ssh.exec_command( + "sudo pip install /usr/local/pyurdme/; screen -d -m ipython notebook --profile={0}".format( + self.profile)) - self.ssh.exec_command( - "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format( - self.profile, self.ipengine_env)) + print "Installing cluster_execution..." + # Install cluster_execution + self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git; " + "PYTHONPATH=`pwd`:$PYTHONPATH;") + + print "Installing molns..." + self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") self.ssh.exec_command( "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( From 421c483e3ee117e6cecb4cc07c17018e8bffea2a Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 11 Nov 2016 11:38:43 -0800 Subject: [PATCH 077/100] install paramiko' --- MolnsLib/ssh_deploy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index c2cf86a..3b7fa93 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -416,6 +416,9 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= "sudo pip install /usr/local/pyurdme/; screen -d -m ipython notebook --profile={0}".format( self.profile)) + print "Installing paramiko..." + self.ssh.exec_command("sudo pip install paramiko") + print "Installing cluster_execution..." # Install cluster_execution self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git; " From fd9a1163de46199b649122478190d6133ccee3e1 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Fri, 11 Nov 2016 13:06:56 -0800 Subject: [PATCH 078/100] Added get command --- molns.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/molns.py b/molns.py index 80b49a6..ced258c 100755 --- a/molns.py +++ b/molns.py @@ -316,6 +316,33 @@ def upload_controller(cls, args, config): subprocess.call(cmd) print "SCP process completed" + @classmethod + def get_controller(cls, args, config): + """ Copy a controller's file to the local filesystem. """ + logging.debug("MOLNSController.put_controller(args={0})".format(args)) + controller_obj = cls._get_controllerobj(args, config) + if controller_obj is None: return + # Check if any instances are assigned to this controller + instance_list = config.get_controller_instances(controller_id=controller_obj.id) + #logging.debug("instance_list={0}".format(instance_list)) + # Check if they are running + ip = None + if len(instance_list) > 0: + for i in instance_list: + status = controller_obj.get_instance_status(i) + logging.debug("instance={0} has status={1}".format(i, status)) + if status == controller_obj.STATUS_RUNNING: + ip = i.ip_address + if ip is None: + print "No active instance for this controller" + return + #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) + #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) + cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), 'ubuntu@{0}:{1}'.format(ip, args[1]), '.'] + print " ".join(cmd) + subprocess.call(cmd) + print "SSH process completed" + @classmethod def put_controller(cls, args, config): """ Copy a local file to the controller's shared area. """ @@ -1388,6 +1415,8 @@ def run(self, args, config_dir=None): function=MOLNSController.stop_controller), Command('terminate', {'name':None}, function=MOLNSController.terminate_controller), + Command('get', {'name':None, 'file':None}, + function=MOLNSController.get_controller), Command('put', {'name':None, 'file':None}, function=MOLNSController.put_controller), Command('upload', {'name':None, 'file':None}, From 504ed83ae0430e3fef5fd088a574d391a5bc991f Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 11 Nov 2016 15:47:33 -0800 Subject: [PATCH 079/100] assume that user is added to docker group --- MolnsLib/Docker.py | 2 +- MolnsLib/Utils.py | 3 ++- MolnsLib/installSoftware.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 19f233a..9f57ed5 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -94,7 +94,7 @@ def _verify_directory(working_directory): try: if not os.path.exists(working_directory): os.makedirs(working_directory) - os.chown(working_directory, Utils.get_sudo_user_id(), Utils.get_sudo_group_id()) + # os.chown(working_directory, Utils.get_sudo_user_id(), Utils.get_sudo_group_id()) return True except: return False diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 3931260..d739f86 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -12,7 +12,8 @@ def ensure_sudo_mode(some_function): import os import sys if sys.platform.startswith("linux") and os.getuid() != 0: - raise NoPrivilegedMode("\n\nOn Linux platforms, 'docker' is a priviledged command. To use 'docker' functionality, please run in sudo mode or as root user.") + pass + #raise NoPrivilegedMode("\n\nOn Linux platforms, 'docker' is a priviledged command. To use 'docker' functionality, please run in sudo mode or as root user.") return some_function diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index b352a3e..f13527e 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -126,7 +126,7 @@ class InstallSW: "sudo apt-get -y remove python-scipy", "sudo pip install scipy", - "sudo pip install jsonschema jsonpointer", #redo this install to be sure it has not been removed. + "sudo pip install jsonschema jsonpointer", # redo this install to be sure it has not been removed. "sync", # This is critical for some infrastructures. From f7fa62431645ba785f27dda328f7285f8c2d215c Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 11 Nov 2016 16:42:55 -0800 Subject: [PATCH 080/100] use new test image --- MolnsLib/Utils.py | 19 +++++++++++++++---- MolnsLib/installSoftware.py | 5 ++--- MolnsLib/ssh_deploy.py | 8 +------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index d739f86..8dd066e 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -1,11 +1,22 @@ +def get_user_name(): + try: + import os + return os.environ['SUDO_USER'] + except KeyError: + import getpass + return getpass.getuser() + + def get_sudo_user_id(): - import pwd, os - return pwd.getpwnam(os.environ['SUDO_USER']).pw_uid + import pwd + u_name = get_user_name() + return pwd.getpwnam(u_name).pw_uid def get_sudo_group_id(): - import grp, os - return grp.getgrnam((os.environ['SUDO_USER'])).gr_gid + import grp + u_name = get_user_name() + return grp.getgrnam(u_name).gr_gid def ensure_sudo_mode(some_function): diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index f13527e..8e10dfb 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -56,8 +56,7 @@ class InstallSW: [ "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", - "cd /usr/local/ && git clone https://github.com/Molns/molnsutil.git", - "cd /usr/local/molnsutil && sudo python setup.py install" + "cd /usr/local/ && git clone https://github.com/aviral26/molnsutil.git && cd /usr/local/molnsutil && git checkout qsub_support" ], # So the workers can mount the controller via SSHfs @@ -127,7 +126,7 @@ class InstallSW: "sudo pip install scipy", "sudo pip install jsonschema jsonpointer", # redo this install to be sure it has not been removed. - + "sudo pip install paramiko", "sync", # This is critical for some infrastructures. ] diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 3b7fa93..0ec7cd5 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -411,20 +411,14 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= "{1}source /usr/local/pyurdme/pyurdme_init; screen -d -m ipython notebook --profile={0}".format( self.profile, self.ipengine_env)) else: - print "pip installing pyurdme..." self.ssh.exec_command( "sudo pip install /usr/local/pyurdme/; screen -d -m ipython notebook --profile={0}".format( self.profile)) - print "Installing paramiko..." - self.ssh.exec_command("sudo pip install paramiko") - - print "Installing cluster_execution..." # Install cluster_execution self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git; " - "PYTHONPATH=`pwd`:$PYTHONPATH;") + "source PYTHONPATH=`pwd`:/usr/local/:$PYTHONPATH;") - print "Installing molns..." self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") self.ssh.exec_command( From 04bae2059b3cf6208dd84477044ba8aa30d23a2b Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Mon, 14 Nov 2016 17:54:54 -0800 Subject: [PATCH 081/100] prepare image with cluster exec --- MolnsLib/Docker.py | 6 ++++-- MolnsLib/DockerProvider.py | 4 +++- MolnsLib/constants.py | 1 + MolnsLib/installSoftware.py | 2 +- MolnsLib/ssh_deploy.py | 30 ++++++++++++++++++++++++++++-- molns.py | 22 ---------------------- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index 9f57ed5..cb2e9f0 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -66,7 +66,8 @@ def create_container(self, image_str, working_directory=None, name=None, container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True, ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT], - host_config=hc) + host_config=hc, + environment={"PYTHONPATH": "/usr/local/"}) else: container_mount_point = '/home/ubuntu/{0}'.format(os.path.basename(working_directory)) @@ -78,7 +79,8 @@ def create_container(self, image_str, working_directory=None, name=None, ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT], volumes=container_mount_point, host_config=hc, - working_dir=volume_dir) + working_dir=volume_dir, + environment={"PYTHONPATH": "/usr/local/"}) container_id = container.get("Id") diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index 5b25e68..dfdbc72 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -224,7 +224,9 @@ class DockerController(DockerBase): {'q': 'Port to use for jupyter notebook', 'default': "8081", 'ask': True}), ('working_directory', - {'q': 'Working directory for this controller', 'default': get_default_working_directory, 'ask': True}) + {'q': 'Working directory for this controller', 'default': get_default_working_directory, 'ask': True}), + ('ssh_key_file', + {'q': 'SSH key to a qsub and docker enabled cluster', 'default': None, 'ask': True}) ]) def get_instance_status(self, instance): diff --git a/MolnsLib/constants.py b/MolnsLib/constants.py index a85601d..7ac7ffb 100644 --- a/MolnsLib/constants.py +++ b/MolnsLib/constants.py @@ -25,3 +25,4 @@ class Constants: DEFAULT_QSUB_SSH_PORT = 22 ForbiddenVolumeNames = [".ssh", ".ipython", ".molns", "ipython", "localarea", "shared"] ConfigDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".molns/") + ClusterKeyFileNameOnController = "molns_cluster_secretkey" diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 8e10dfb..bcf99fc 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -53,7 +53,7 @@ class InstallSW: "sudo pip install python-keystoneclient", "sudo pip install python-swiftclient", ], - + # TODO pip install molnsutil while building image [ "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", "cd /usr/local/ && git clone https://github.com/aviral26/molnsutil.git && cd /usr/local/molnsutil && git checkout qsub_support" diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 0ec7cd5..9e063ab 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -350,6 +350,31 @@ def deploy_stochss(self, ip_address, port=1443): print "StochSS launch failed: {0}\t{1}:{2}".format(e, ip_address, self.ssh_endpoint) raise sys.exc_info()[1], None, sys.exc_info()[2] + def __transfer_cluster_ssh_key_file(self, remote_target_dir, controller_obj): + local_ssh_key_file_path = controller_obj.config["ssh_key_file"] + + if local_ssh_key_file_path is None: + print "No SSH key file provided for cluster access." + return + + if not os.access(local_ssh_key_file_path, os.R_OK): + print "Skipping transfer of SSH key file. (Read access denied.) Cluster executions will not be possible." + return + + # Transfer secret key file. + sftp = self.ssh.open_sftp() + remote_file_abs_path = os.path.join(remote_target_dir, Constants.ClusterKeyFileNameOnController) + remote_ssh_key_file = sftp.file(remote_file_abs_path, 'w') + + with open(local_ssh_key_file_path, "r") as local_ssh_key_file: + remote_ssh_key_file.write(local_ssh_key_file.read()) + + remote_ssh_key_file.close() + sftp.close() + + # Give user ubuntu permission to access file. + self.ssh.exec_command("sudo chown ubuntu:ubuntu {0}".format(remote_file_abs_path)) + def deploy_ipython_controller(self, instance, controller_obj, notebook_password=None, resume=False): ip_address = instance.ip_address @@ -388,6 +413,7 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.ssh.exec_command("ipython profile create {0}".format(self.profile)) self.create_ipython_config(ip_address, notebook_password) self.create_engine_config() + self.__transfer_cluster_ssh_key_file(remote_target_dir=home_dir, controller_obj=controller_obj) if controller_obj.provider.type == Constants.DockerProvider: self.ssh.exec_command("mv {0}*.ipynb {1}".format(home_dir, Docker.get_container_volume_from_working_dir( @@ -416,9 +442,9 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.profile)) # Install cluster_execution - self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git; " - "source PYTHONPATH=`pwd`:/usr/local/:$PYTHONPATH;") + self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git") + # Install molns self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") self.ssh.exec_command( diff --git a/molns.py b/molns.py index e613443..fec11b3 100755 --- a/molns.py +++ b/molns.py @@ -1272,28 +1272,6 @@ def clear_instances(cls, args, config): else: print "No instance found" - -############################################################################################## - - -class MolnsLocal(MOLNSbase): - @classmethod - def setup_local(cls): - # TODO - pass - - @classmethod - def start_local(cls): - # TODO - pass - - -############################################################################################## -############################################################################################## -############################################################################################## -############################################################################################## -############################################################################################## -############################################################################################## # Below is the API for the command line execution From cd1596eb2ad36b572a98c204bc159ba3aa4853c8 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 15 Nov 2016 16:00:09 -0800 Subject: [PATCH 082/100] temporarily update molnsutil to latest version in sshDeploy --- MolnsLib/ssh_deploy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 9e063ab..4f8d596 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -447,6 +447,9 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= # Install molns self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") + # Temporary command until new image is created to use latest molnsutil. TODO pip install molnsutil + self.ssh.exec_command("cd /usr/local/molnsutil; git checkout qsub_support; git pull") + self.ssh.exec_command( "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format( Constants.DEFAULT_PUBLIC_NOTEBOOK_PORT, Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT)) From 1e0739744ecbf5f384aadee02eae49ed4ed3218f Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 23 Nov 2016 15:24:24 -0800 Subject: [PATCH 083/100] clean up code, use python logging instead of custom logging --- MolnsLib/Docker.py | 60 ++++++++++++++------------------------ MolnsLib/DockerProvider.py | 3 +- MolnsLib/DockerSSH.py | 2 +- MolnsLib/Utils.py | 3 +- MolnsLib/test.py | 5 ---- 5 files changed, 26 insertions(+), 47 deletions(-) delete mode 100644 MolnsLib/test.py diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index cb2e9f0..c10477b 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -1,8 +1,6 @@ import logging import re import time - -from Utils import Log, ensure_sudo_mode import constants from molns_provider import ProviderBase from constants import Constants @@ -26,7 +24,6 @@ class ImageBuildException(Exception): def __init__(self, message=None): super("Something went wrong while building docker container image.\n{0}".format(message)) - @ensure_sudo_mode def __init__(self): self.client = Client(base_url=Constants.DOCKER_BASE_URL) self.build_count = 0 @@ -37,7 +34,6 @@ def get_container_volume_from_working_dir(working_directory): import os return os.path.join("/home/ubuntu/", os.path.basename(working_directory)) - @ensure_sudo_mode def create_container(self, image_str, working_directory=None, name=None, port_bindings={Constants.DEFAULT_PUBLIC_WEBSERVER_PORT: ('127.0.0.1', 8080), Constants.DEFAULT_PRIVATE_NOTEBOOK_PORT: ('127.0.0.1', 8081)}): @@ -54,14 +50,19 @@ def create_container(self, image_str, working_directory=None, name=None, image = docker_image.image_id if docker_image.image_id is not Constants.DockerNonExistentTag \ else docker_image.image_tag - Log.write_log("Using image {0}".format(image)) + logging.info("Using image {0}".format(image)) import os if Docker._verify_directory(working_directory) is False: if working_directory is not None: - raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the controller environment, and unfortunately the provided name for working directory of the controller cannot be one of these. Please configure this controller again with a different volume name and retry. Here is a list of forbidden names: \n{0}".format(Constants.ForbiddenVolumeNames)) - - Log.write_log(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " - "be created.") + raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the " + "controller environment, and unfortunately the provided name for working " + "directory of the controller cannot be one of these. Please configure this " + "controller again with a different volume name and retry. " + "Here is a list of forbidden names: \n{0}" + .format(Constants.ForbiddenVolumeNames)) + + logging.warning(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " + "be created.") hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True, ports=[Constants.DEFAULT_PUBLIC_WEBSERVER_PORT, @@ -87,32 +88,27 @@ def create_container(self, image_str, working_directory=None, name=None, return container_id # noinspection PyBroadException - @ensure_sudo_mode @staticmethod def _verify_directory(working_directory): - import os, Utils + import os if working_directory is None or os.path.basename(working_directory) in Constants.ForbiddenVolumeNames: return False try: if not os.path.exists(working_directory): os.makedirs(working_directory) - # os.chown(working_directory, Utils.get_sudo_user_id(), Utils.get_sudo_group_id()) return True except: return False - @ensure_sudo_mode def stop_containers(self, container_ids): """Stops given containers.""" for container_id in container_ids: self.stop_container(container_id) - @ensure_sudo_mode def stop_container(self, container_id): """Stops the container with given ID.""" self.client.stop(container_id) - @ensure_sudo_mode def container_status(self, container_id): """Checks if container with given ID running.""" status = ProviderBase.STATUS_TERMINATED @@ -126,44 +122,40 @@ def container_status(self, container_id): pass return status - @ensure_sudo_mode def start_containers(self, container_ids): """Starts each container in given list of container IDs.""" for container_id in container_ids: self.start_container(container_id) - @ensure_sudo_mode def start_container(self, container_id): """ Start the container with given ID.""" - Log.write_log(Docker.LOG_TAG + " Starting container " + container_id) + logging.info(Docker.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: - Log.write_log(Docker.LOG_TAG + " Something went wrong while starting container.", e) + print (Docker.LOG_TAG + "Something went wrong while starting container.", e) return False return True - @ensure_sudo_mode def execute_command(self, container_id, command): """Executes given command as a shell command in the given container. Returns None is anything goes wrong.""" run_command = "/bin/bash -c \"" + command + "\"" # print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) if self.start_container(container_id) is False: - Log.write_log(Docker.LOG_TAG + "Could not start container.") + print (Docker.LOG_TAG + "Could not start container.") return None try: exec_instance = self.client.exec_create(container_id, run_command) response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: - Log.write_log(Docker.LOG_TAG + " Could not execute command.", e) + print (Docker.LOG_TAG + " Could not execute command.", e) return None - @ensure_sudo_mode def build_image(self, dockerfile): """ Build image from given Dockerfile object and return ID of the image created. """ import uuid - Log.write_log(Docker.LOG_TAG + "Building image...") + logging.info("Building image...") random_string = str(uuid.uuid4()) image_tag = Constants.DOCKER_IMAGE_PREFIX + "{0}".format(random_string[:]) last_line = "" @@ -177,7 +169,7 @@ def build_image(self, dockerfile): # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image # id. image_id = get_docker_image_id_from_string(str(last_line)) - Log.write_log(Docker.LOG_TAG + "Image ID: {0}".format(image_id)) + logging.info("Image ID: {0}".format(image_id)) return str(DockerImage(image_id, image_tag)) except (Docker.ImageBuildException, IndexError) as e: @@ -187,7 +179,6 @@ def build_image(self, dockerfile): def _decorate(some_line): return some_line[11:-4].rstrip() - @ensure_sudo_mode def image_exists(self, image_str): """Checks if an image with the given ID/tag exists locally.""" docker_image = DockerImage.from_string(image_str) @@ -206,7 +197,6 @@ def image_exists(self, image_str): return True return False - @ensure_sudo_mode def terminate_containers(self, container_ids): """ Terminates containers with given container ids.""" for container_id in container_ids: @@ -217,50 +207,44 @@ def terminate_containers(self, container_ids): except NotFound: pass - @ensure_sudo_mode def terminate_container(self, container_id): self.client.remove_container(container_id) - @ensure_sudo_mode def get_mapped_ports(self, container_id): container_ins = self.client.inspect_container(container_id) mapped_ports = container_ins['HostConfig']['PortBindings'] ret_val = [] if mapped_ports is None: - Log.write_log("No mapped ports for {0}".format(container_id)) + logging.info("No mapped ports for {0}".format(container_id)) return for k, v in mapped_ports.iteritems(): host_port = v[0]['HostPort'] ret_val.append(host_port) return ret_val - @ensure_sudo_mode def get_working_directory(self, container_id): return self.client.inspect_container(container_id)["Config"]["WorkingDir"] - @ensure_sudo_mode def get_home_directory(self, container_id): env_vars = self.client.inspect_container(container_id)["Config"]["Env"] home = [i for i in env_vars if i.startswith("HOME")] return home[0].split("=")[1] - @ensure_sudo_mode def put_archive(self, container_id, tar_file_bytes, target_path_in_container): """ Copies and unpacks a given tarfile in the container at specified location. Location must exist in container.""" if self.start_container(container_id) is False: - raise Exception("ERROR Could not start container.") + raise Exception("Could not start container.") # Prepend file path with /home/ubuntu/. TODO Should be refined. if not target_path_in_container.startswith("/home/ubuntu/"): import os target_path_in_container = os.path.join("/home/ubuntu/", target_path_in_container) - Log.write_log("target path in container: {0}".format(target_path_in_container)) + logging.info("target path in container: {0}".format(target_path_in_container)) if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - Log.write_log(Docker.LOG_TAG + "Failed to copy.") + logging.error(Docker.LOG_TAG + "Failed to copy.") - @ensure_sudo_mode def get_container_ip_address(self, container_id): """ Returns the IP Address of given container.""" self.start_container(container_id) @@ -327,4 +311,4 @@ def looks_like_image_id(some_string): if some_string is possible_image_id: return True else: - return False \ No newline at end of file + return False diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index dfdbc72..eb34782 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -2,11 +2,10 @@ import os import tempfile import time -from collections import OrderedDict - import Docker import constants import installSoftware +from collections import OrderedDict from DockerSSH import DockerSSH from constants import Constants from molns_provider import ProviderBase, ProviderException diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index 3b642f6..f9b1732 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -26,7 +26,7 @@ def connect(self, instance, endpoint, username=None, key_filename=None): self.container_id = instance.provider_instance_identifier def connect_cluster_node(self, ip_address, port, username, keyfile): - raise DockerSSHException("This invocation has been in error.") + raise DockerSSHException("This invocation means that an error has occurred.") def close(self): self.container_id = None diff --git a/MolnsLib/Utils.py b/MolnsLib/Utils.py index 8dd066e..a6b75f2 100644 --- a/MolnsLib/Utils.py +++ b/MolnsLib/Utils.py @@ -24,7 +24,8 @@ def ensure_sudo_mode(some_function): import sys if sys.platform.startswith("linux") and os.getuid() != 0: pass - #raise NoPrivilegedMode("\n\nOn Linux platforms, 'docker' is a priviledged command. To use 'docker' functionality, please run in sudo mode or as root user.") + raise NoPrivilegedMode("\n\nOn Linux platforms, 'docker' is a priviledged command. " + "To use 'docker' functionality, please run in sudo mode or as root user.") return some_function diff --git a/MolnsLib/test.py b/MolnsLib/test.py deleted file mode 100644 index 2bf4c00..0000000 --- a/MolnsLib/test.py +++ /dev/null @@ -1,5 +0,0 @@ -import pdb - -for i in range(10): - pdb.set_trace() - print "hi" From 6ca26e20924ea26d11248186382d5d3621348f3b Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 29 Nov 2016 12:19:03 -0800 Subject: [PATCH 084/100] add user friendly message --- MolnsLib/DockerProvider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index eb34782..b183d57 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -225,7 +225,7 @@ class DockerController(DockerBase): ('working_directory', {'q': 'Working directory for this controller', 'default': get_default_working_directory, 'ask': True}), ('ssh_key_file', - {'q': 'SSH key to a qsub and docker enabled cluster', 'default': None, 'ask': True}) + {'q': 'SSH key to a qsub and docker enabled cluster', 'default': "None", 'ask': True}) ]) def get_instance_status(self, instance): From 97f72884b9f655af092f88d282a57358162f6109 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 29 Nov 2016 13:07:14 -0800 Subject: [PATCH 085/100] fix to ignore docker providers --- MolnsLib/molns_datastore.py | 5 ++++- molns.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/MolnsLib/molns_datastore.py b/MolnsLib/molns_datastore.py index 00093c5..1403177 100644 --- a/MolnsLib/molns_datastore.py +++ b/MolnsLib/molns_datastore.py @@ -123,7 +123,8 @@ def get_provider_handle(kind, ptype): if kind not in valid_handles: raise DatastoreException("Unknown kind {0}".format(kind)) if ptype not in VALID_PROVIDER_TYPES: - raise DatastoreException("Unknown {1} type {0}".format(ptype, kind)) + #raise DatastoreException("Unknown {1} type {0}".format(ptype, kind)) + return None cls_name = "{0}{1}".format(ptype, kind) pkg_name = "MolnsLib.{0}Provider".format(ptype) if pkg_name not in sys.modules: @@ -272,6 +273,8 @@ def _get_object_data(self, d_handle, kind, ptype, p): p_handle = get_provider_handle(kind, ptype) #logging.debug("{2}(name={0}, data={1})".format(name,data,p_handle)) + if p_handle is None: + return None ret = p_handle(name=p.name, config=data, config_dir=self.config_dir) ret.id = p.id ret.datastore = self diff --git a/molns.py b/molns.py index ced258c..2f314bb 100755 --- a/molns.py +++ b/molns.py @@ -417,7 +417,9 @@ def status_controller(cls, args, config): if len(instance_list) > 0: table_data = [] for i in instance_list: - provider_name = config.get_object_by_id(i.provider_id, 'Provider').name + provider_obj = config.get_object_by_id(i.provider_id, 'Provider') + if provider_obj is None: continue + provider_name = provider_obj.name controller_name = config.get_object_by_id(i.controller_id, 'Controller').name if i.worker_group_id is not None: worker_name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name From bff44ebc6ba951ef061adfbdabfcea37abb9fd97 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 29 Nov 2016 15:01:59 -0800 Subject: [PATCH 086/100] add TODO --- MolnsLib/ssh_deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 4f8d596..45c9b4c 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -441,6 +441,7 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= "sudo pip install /usr/local/pyurdme/; screen -d -m ipython notebook --profile={0}".format( self.profile)) + # TODO remove next three commands after testing. Put them in the image instead. # Install cluster_execution self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git") From ae9c25162010297b50ded4886de284807a4f8354 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 2 Dec 2016 15:22:59 -0800 Subject: [PATCH 087/100] put_controller, upload_controller, get_controller --- MolnsLib/DockerSSH.py | 2 ++ molns.py | 74 ++++++++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index f9b1732..cd88a39 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -64,6 +64,8 @@ def __init__(self, filename, flag, docker, container_id): # print("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) def write(self, write_this): + if type(write_this) is not str: + raise TypeError("Expecting type str") self.file_contents += write_this def close(self): diff --git a/molns.py b/molns.py index 2d95b59..c5d0c09 100755 --- a/molns.py +++ b/molns.py @@ -315,10 +315,11 @@ def upload_controller(cls, args, config): """ Copy a local file to the controller's home directory. """ logging.debug("MOLNSController.upload_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - # logging.debug("instance_list={0}".format(instance_list)) + # Check if they are running ip = None if len(instance_list) > 0: @@ -329,23 +330,37 @@ def upload_controller(cls, args, config): ip = i.ip_address if ip is None: raise MOLNSException("No active instance for this controller") - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) - cmd = ['/usr/bin/scp','-r','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i', - controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/'.format(ip)] - print " ".join(cmd) - subprocess.call(cmd) - print "SCP process completed" + + file_to_transfer = args[1] + logging.debug("File to transfer: {0}".format(file_to_transfer)) + + remote_file_path = "/home/ubuntu/" + + sftp = controller_obj.ssh.open_sftp() + remote_fh = sftp.file(remote_file_path, "w") + try: + with open(file_to_transfer, "r") as fh: + remote_fh.write(fh.read()) + finally: + remote_fh.close() + sftp.close() + + print "Transferred {0} to {1}:{2}".format(file_to_transfer, ip, remote_file_path) @classmethod def get_controller(cls, args, config): """ Copy a controller's file to the local filesystem. """ - logging.debug("MOLNSController.put_controller(args={0})".format(args)) + logging.debug("MOLNSController.get_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return + + if controller_obj.provider_type == constants.Constants.DockerProvider: + raise NotImplementedError("Docker provider does not support this feature yet.") + # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - #logging.debug("instance_list={0}".format(instance_list)) + # Check if they are running ip = None if len(instance_list) > 0: @@ -357,8 +372,6 @@ def get_controller(cls, args, config): if ip is None: print "No active instance for this controller" return - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), 'ubuntu@{0}:{1}'.format(ip, args[1]), '.'] print " ".join(cmd) subprocess.call(cmd) @@ -369,10 +382,12 @@ def put_controller(cls, args, config): """ Copy a local file to the controller's and workers' shared area. """ logging.debug("MOLNSController.put_controller(args={0})".format(args)) controller_obj = cls._get_controllerobj(args, config) - if controller_obj is None: return + if controller_obj is None: + return + # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) - # logging.debug("instance_list={0}".format(instance_list)) + # Check if they are running ip = None if len(instance_list) > 0: @@ -383,13 +398,22 @@ def put_controller(cls, args, config): ip = i.ip_address if ip is None: raise MOLNSException("No active instance for this controller") - #print " ".join(['/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)]) - #os.execl('/usr/bin/ssh','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(),'ubuntu@{0}'.format(ip)) - cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i', - controller_obj.provider.sshkeyfilename(), args[1], 'ubuntu@{0}:/home/ubuntu/shared'.format(ip)] - print " ".join(cmd) - subprocess.call(cmd) - print "SSH process completed" + + file_to_transfer = args[1] + logging.debug("File to transfer: {0}".format(file_to_transfer)) + + remote_file_path = "/home/ubuntu/shared" + + sftp = controller_obj.ssh.open_sftp() + remote_fh = sftp.file(remote_file_path, "w") + try: + with open(file_to_transfer, "r") as fh: + remote_fh.write(fh.read()) + finally: + remote_fh.close() + sftp.close() + + print "Transferred {0} to {1}:{2}".format(file_to_transfer, ip, remote_file_path) @classmethod def is_controller_running(cls, args, config): @@ -1686,9 +1710,9 @@ def run(self, args, config_dir=None): function=MOLNSWorkerGroup.add_worker_groups), Command('status', {'name': None}, function=MOLNSWorkerGroup.status_worker_groups), - Command('stop', {'name':None}, + Command('stop', {'name':None}, function=MOLNSWorkerGroup.terminate_worker_groups), - Command('terminate', {'name':None}, + Command('terminate', {'name':None}, function=MOLNSWorkerGroup.terminate_worker_groups), Command('export', {'name': None}, function=MOLNSWorkerGroup.worker_group_export), From f360812e14df2f55a166b67582910fc6fb77a5fe Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 2 Dec 2016 15:24:51 -0800 Subject: [PATCH 088/100] remove type check --- MolnsLib/DockerSSH.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index cd88a39..f9b1732 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -64,8 +64,6 @@ def __init__(self, filename, flag, docker, container_id): # print("WARNING Unrecognized file mode. Filename: {0}, Flag: {1}".format(filename, flag)) def write(self, write_this): - if type(write_this) is not str: - raise TypeError("Expecting type str") self.file_contents += write_this def close(self): From 4101969e4b24badc743093a14a77c266cbb27c8b Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 2 Dec 2016 15:44:09 -0800 Subject: [PATCH 089/100] bug fixes --- MolnsLib/DockerSSH.py | 42 ++++++++++++++++++++++-------------------- MolnsLib/ssh_deploy.py | 2 +- molns.py | 33 +++++++++++++++++++-------------- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/MolnsLib/DockerSSH.py b/MolnsLib/DockerSSH.py index f9b1732..6f2032a 100644 --- a/MolnsLib/DockerSSH.py +++ b/MolnsLib/DockerSSH.py @@ -71,23 +71,25 @@ def close(self): import uuid rand_str = str(uuid.uuid4()) temp_tar = "transport-{0}.tar".format(rand_str[:8]) - tar = tarfile.TarFile(temp_tar, "w") - string = StringIO.StringIO() - string.write(self.file_contents) - string.seek(0) - tar_file_info = tarfile.TarInfo(name=os.path.basename(self.filename)) - tar_file_info.size = len(string.buf) - tar.addfile(tarinfo=tar_file_info, fileobj=string) - tar.close() - - path_to_file = os.path.dirname(self.filename) - - if not path_to_file.startswith("/home"): - path_to_file = os.path.join(self.docker.get_home_directory(self.container_id), path_to_file) - - with open(temp_tar, mode='rb') as f: - tar_file_bytes = f.read() - - # print("path to file: {0}".format(path_to_file)) - self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) - os.remove(temp_tar) # Remove temporary tar file. + try: + tar = tarfile.TarFile(temp_tar, "w") + string = StringIO.StringIO() + string.write(self.file_contents) + string.seek(0) + tar_file_info = tarfile.TarInfo(name=os.path.basename(self.filename)) + tar_file_info.size = len(string.buf) + tar.addfile(tarinfo=tar_file_info, fileobj=string) + tar.close() + + path_to_file = os.path.dirname(self.filename) + + if not path_to_file.startswith("/home"): + path_to_file = os.path.join(self.docker.get_home_directory(self.container_id), path_to_file) + + with open(temp_tar, mode='rb') as f: + tar_file_bytes = f.read() + + # print("path to file: {0}".format(path_to_file)) + self.docker.put_archive(self.container_id, tar_file_bytes, path_to_file) + finally: + os.remove(temp_tar) # Remove temporary tar file. diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 1233355..6acf6ea 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -493,7 +493,7 @@ def __transfer_cluster_ssh_key_file(self, remote_target_dir, controller_obj): return if not os.access(local_ssh_key_file_path, os.R_OK): - print "Skipping transfer of SSH key file. (Read access denied.) Cluster executions will not be possible." + print "Skipping transfer of SSH key file." return # Transfer secret key file. diff --git a/molns.py b/molns.py index c5d0c09..e304836 100755 --- a/molns.py +++ b/molns.py @@ -321,20 +321,22 @@ def upload_controller(cls, args, config): instance_list = config.get_controller_instances(controller_id=controller_obj.id) # Check if they are running - ip = None + inst = None if len(instance_list) > 0: for i in instance_list: status = controller_obj.get_instance_status(i) logging.debug("instance={0} has status={1}".format(i, status)) if status == controller_obj.STATUS_RUNNING: - ip = i.ip_address - if ip is None: + inst = i + if inst is None: raise MOLNSException("No active instance for this controller") file_to_transfer = args[1] logging.debug("File to transfer: {0}".format(file_to_transfer)) - remote_file_path = "/home/ubuntu/" + remote_file_path = os.path.join("/home/ubuntu/", os.path.basename(file_to_transfer)) + + controller_obj.ssh.connect(inst, SSHDeploy.DEFAULT_SSH_PORT, "ubuntu", controller_obj.provider.sshkeyfilename()) sftp = controller_obj.ssh.open_sftp() remote_fh = sftp.file(remote_file_path, "w") @@ -345,7 +347,7 @@ def upload_controller(cls, args, config): remote_fh.close() sftp.close() - print "Transferred {0} to {1}:{2}".format(file_to_transfer, ip, remote_file_path) + print "Transferred {0} to {1}@{2}:{3}".format(file_to_transfer, inst.ip_address, "ubuntu", remote_file_path) @classmethod def get_controller(cls, args, config): @@ -355,8 +357,8 @@ def get_controller(cls, args, config): if controller_obj is None: return - if controller_obj.provider_type == constants.Constants.DockerProvider: - raise NotImplementedError("Docker provider does not support this feature yet.") + if controller_obj.provider.type == constants.Constants.DockerProvider: + raise NotImplementedError("DockerController does not support this feature yet.") # Check if any instances are assigned to this controller instance_list = config.get_controller_instances(controller_id=controller_obj.id) @@ -372,7 +374,8 @@ def get_controller(cls, args, config): if ip is None: print "No active instance for this controller" return - cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i',controller_obj.provider.sshkeyfilename(), 'ubuntu@{0}:{1}'.format(ip, args[1]), '.'] + cmd = ['/usr/bin/scp','-oStrictHostKeyChecking=no','-oUserKnownHostsFile=/dev/null','-i', + controller_obj.provider.sshkeyfilename(), 'ubuntu@{0}:{1}'.format(ip, args[1]), '.'] print " ".join(cmd) subprocess.call(cmd) print "SSH process completed" @@ -389,20 +392,22 @@ def put_controller(cls, args, config): instance_list = config.get_controller_instances(controller_id=controller_obj.id) # Check if they are running - ip = None + inst = None if len(instance_list) > 0: for i in instance_list: status = controller_obj.get_instance_status(i) logging.debug("instance={0} has status={1}".format(i, status)) if status == controller_obj.STATUS_RUNNING: - ip = i.ip_address - if ip is None: + inst = i + if inst is None: raise MOLNSException("No active instance for this controller") file_to_transfer = args[1] logging.debug("File to transfer: {0}".format(file_to_transfer)) - remote_file_path = "/home/ubuntu/shared" + remote_file_path = os.path.join("/home/ubuntu/shared", os.path.basename(file_to_transfer)) + + controller_obj.ssh.connect(inst, SSHDeploy.DEFAULT_SSH_PORT, "ubuntu", controller_obj.provider.sshkeyfilename()) sftp = controller_obj.ssh.open_sftp() remote_fh = sftp.file(remote_file_path, "w") @@ -413,7 +418,7 @@ def put_controller(cls, args, config): remote_fh.close() sftp.close() - print "Transferred {0} to {1}:{2}".format(file_to_transfer, ip, remote_file_path) + print "Transferred {0} to {1}@{2}:{3}".format(file_to_transfer, inst.ip_address, "ubuntu", remote_file_path) @classmethod def is_controller_running(cls, args, config): @@ -1814,5 +1819,5 @@ def parse_args(): if __name__ == "__main__": logger = logging.getLogger() #logger.setLevel(logging.INFO) #for Debugging - logger.setLevel(logging.CRITICAL) + logger.setLevel(logging.DEBUG) parse_args() From 203157ffc2052fa0b42f87dc45b427bed6608f27 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Mon, 5 Dec 2016 10:45:06 -0800 Subject: [PATCH 090/100] remove __init__.py --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 From 4bc69a489a80fab53814b26b4b864d2433b0d8ea Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Mon, 5 Dec 2016 13:53:49 -0800 Subject: [PATCH 091/100] add __init__ --- __init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..ff7cc60 --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ +from molns import * From 6018104dc2734028670578cc46755addf2bb5870 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 13 Dec 2016 13:41:21 -0800 Subject: [PATCH 092/100] add dependencies of molns --- MolnsLib/installSoftware.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 3094bf7..8c44502 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -107,7 +107,9 @@ class InstallSW: # Gmsh for Finite Element meshes "sudo apt-get install -y gmsh", ], - + + ["sudo apt-get install docker", "sudo pip install docker-py", "sudo pip install sqlalchemy", + "sudo pip install boto", "sudo pip install python-novaclient", "sudo pip install paramiko"], # pyurdme [ "sudo rm -rf /usr/local/pyurdme && sudo mkdir -p /usr/local/pyurdme && sudo chown ubuntu /usr/local/pyurdme", "cd /usr/local/ && git clone https://github.com/MOLNs/pyurdme.git", From 828afcbedec07e40a0e76b29c3362b8e20eb7ae8 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 11 Jan 2017 11:19:52 -0800 Subject: [PATCH 093/100] bug fixes' --- MolnsLib/Docker.py | 4 ++-- MolnsLib/DockerProvider.py | 2 +- MolnsLib/ssh_deploy.py | 5 +++-- molns.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/MolnsLib/Docker.py b/MolnsLib/Docker.py index c10477b..caa4b84 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/Docker.py @@ -58,7 +58,7 @@ def create_container(self, image_str, working_directory=None, name=None, "controller environment, and unfortunately the provided name for working " "directory of the controller cannot be one of these. Please configure this " "controller again with a different volume name and retry. " - "Here is a list of forbidden names: \n{0}" + "Here is the list of forbidden names: \n{0}" .format(Constants.ForbiddenVolumeNames)) logging.warning(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " @@ -189,7 +189,7 @@ def image_exists(self, image_str): for image in self.client.images(): some_id = image["Id"] - some_tags = image["RepoTags"] + some_tags = image["RepoTags"] or [None] if docker_image.image_id in \ some_id[:(Constants.DOCKER_PY_IMAGE_ID_PREFIX_LENGTH + Constants.DOKCER_IMAGE_ID_LENGTH)]: return True diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index b183d57..e7035b7 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -208,7 +208,7 @@ def _preprocess(command): def get_default_working_directory(config=None): if config is None: raise Exception("Config should not be None.") - return os.path.join(config.config_dir, "docker_controller_working_dirs", config.name) + return os.path.realpath(os.path.join(config.config_dir, "docker_controller_working_dirs", config.name)) class DockerController(DockerBase): diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 6acf6ea..60b8ed8 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -493,7 +493,7 @@ def __transfer_cluster_ssh_key_file(self, remote_target_dir, controller_obj): return if not os.access(local_ssh_key_file_path, os.R_OK): - print "Skipping transfer of SSH key file." + print "No read access to SSH key file. Skipping transfer." return # Transfer secret key file. @@ -507,8 +507,9 @@ def __transfer_cluster_ssh_key_file(self, remote_target_dir, controller_obj): remote_ssh_key_file.close() sftp.close() - # Give user ubuntu permission to access file. + # Only user ubuntu has permission to access file. self.ssh.exec_command("sudo chown ubuntu:ubuntu {0}".format(remote_file_abs_path)) + self.ssh.exec_command("sudo chmod 400 {0}".format(remote_file_abs_path)) def deploy_ipython_controller(self, instance, controller_obj, notebook_password=None, reserved_cpus=2, resume=False): diff --git a/molns.py b/molns.py index e304836..f0523ec 100755 --- a/molns.py +++ b/molns.py @@ -1819,5 +1819,5 @@ def parse_args(): if __name__ == "__main__": logger = logging.getLogger() #logger.setLevel(logging.INFO) #for Debugging - logger.setLevel(logging.DEBUG) + #logger.setLevel(logging.DEBUG) parse_args() From 683d95cd36b2c837bcb0794db1a2646acbcab0be Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Tue, 31 Jan 2017 17:01:53 -0800 Subject: [PATCH 094/100] rename Docker to DockerProxy --- MolnsLib/DockerProvider.py | 6 +++--- MolnsLib/{Docker.py => DockerProxy.py} | 26 +++++++++++++------------- MolnsLib/ssh_deploy.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) rename MolnsLib/{Docker.py => DockerProxy.py} (93%) diff --git a/MolnsLib/DockerProvider.py b/MolnsLib/DockerProvider.py index e7035b7..cc04fe1 100644 --- a/MolnsLib/DockerProvider.py +++ b/MolnsLib/DockerProvider.py @@ -2,7 +2,7 @@ import os import tempfile import time -import Docker +import DockerProxy import constants import installSoftware from collections import OrderedDict @@ -24,7 +24,7 @@ class DockerBase(ProviderBase): def __init__(self, name, config=None, config_dir=None, **kwargs): ProviderBase.__init__(self, name, config, config_dir, **kwargs) - self.docker = Docker.Docker() + self.docker = DockerProxy.DockerProxy() self.ssh = DockerSSH(self.docker) def _get_container_status(self, container_id): @@ -197,7 +197,7 @@ def _create_dockerfile(self, commands): @staticmethod def _preprocess(command): """ Prepends "shell only" commands with '/bin/bash -c'. """ - for shell_command in Docker.Docker.shell_commands: + for shell_command in DockerProxy.DockerProxy.shell_commands: if shell_command in command: replace_string = "/bin/bash -c \"" + shell_command command = command.replace(shell_command, replace_string) diff --git a/MolnsLib/Docker.py b/MolnsLib/DockerProxy.py similarity index 93% rename from MolnsLib/Docker.py rename to MolnsLib/DockerProxy.py index caa4b84..122d911 100644 --- a/MolnsLib/Docker.py +++ b/MolnsLib/DockerProxy.py @@ -12,7 +12,7 @@ class InvalidVolumeName(Exception): pass -class Docker: +class DockerProxy: """ A wrapper over docker-py and some utility methods and classes. """ @@ -41,7 +41,7 @@ def create_container(self, image_str, working_directory=None, name=None, to 8080 of locahost by default""" docker_image = DockerImage.from_string(image_str) - volume_dir = Docker.get_container_volume_from_working_dir(working_directory) + volume_dir = DockerProxy.get_container_volume_from_working_dir(working_directory) if name is None: import uuid @@ -52,7 +52,7 @@ def create_container(self, image_str, working_directory=None, name=None, logging.info("Using image {0}".format(image)) import os - if Docker._verify_directory(working_directory) is False: + if DockerProxy._verify_directory(working_directory) is False: if working_directory is not None: raise InvalidVolumeName("\n\nMOLNs uses certain reserved names for its configuration files in the " "controller environment, and unfortunately the provided name for working " @@ -61,7 +61,7 @@ def create_container(self, image_str, working_directory=None, name=None, "Here is the list of forbidden names: \n{0}" .format(Constants.ForbiddenVolumeNames)) - logging.warning(Docker.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " + logging.warning(DockerProxy.LOG_TAG + "Unable to verify provided directory to use to as volume. Volume will NOT " "be created.") hc = self.client.create_host_config(privileged=True, port_bindings=port_bindings) container = self.client.create_container(image=image, name=name, command="/bin/bash", tty=True, detach=True, @@ -129,11 +129,11 @@ def start_containers(self, container_ids): def start_container(self, container_id): """ Start the container with given ID.""" - logging.info(Docker.LOG_TAG + " Starting container " + container_id) + logging.info(DockerProxy.LOG_TAG + " Starting container " + container_id) try: self.client.start(container=container_id) except (NotFound, NullResource) as e: - print (Docker.LOG_TAG + "Something went wrong while starting container.", e) + print (DockerProxy.LOG_TAG + "Something went wrong while starting container.", e) return False return True @@ -142,14 +142,14 @@ def execute_command(self, container_id, command): run_command = "/bin/bash -c \"" + command + "\"" # print("CONTAINER: {0} COMMAND: {1}".format(container_id, run_command)) if self.start_container(container_id) is False: - print (Docker.LOG_TAG + "Could not start container.") + print (DockerProxy.LOG_TAG + "Could not start container.") return None try: exec_instance = self.client.exec_create(container_id, run_command) response = self.client.exec_start(exec_instance) return [self.client.exec_inspect(exec_instance), response] except (NotFound, APIError) as e: - print (Docker.LOG_TAG + " Could not execute command.", e) + print (DockerProxy.LOG_TAG + " Could not execute command.", e) return None def build_image(self, dockerfile): @@ -161,9 +161,9 @@ def build_image(self, dockerfile): last_line = "" try: for line in self.client.build(fileobj=dockerfile, rm=True, tag=image_tag): - print(Docker._decorate(line)) + print(DockerProxy._decorate(line)) if "errorDetail" in line: - raise Docker.ImageBuildException() + raise DockerProxy.ImageBuildException() last_line = line # Return image ID. It's a hack around the fact that docker-py's build image command doesn't return an image @@ -172,8 +172,8 @@ def build_image(self, dockerfile): logging.info("Image ID: {0}".format(image_id)) return str(DockerImage(image_id, image_tag)) - except (Docker.ImageBuildException, IndexError) as e: - raise Docker.ImageBuildException(e) + except (DockerProxy.ImageBuildException, IndexError) as e: + raise DockerProxy.ImageBuildException(e) @staticmethod def _decorate(some_line): @@ -243,7 +243,7 @@ def put_archive(self, container_id, tar_file_bytes, target_path_in_container): logging.info("target path in container: {0}".format(target_path_in_container)) if not self.client.put_archive(container_id, target_path_in_container, tar_file_bytes): - logging.error(Docker.LOG_TAG + "Failed to copy.") + logging.error(DockerProxy.LOG_TAG + "Failed to copy.") def get_container_ip_address(self, container_id): """ Returns the IP Address of given container.""" diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 60b8ed8..0e5adb5 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -11,7 +11,7 @@ from constants import Constants -from Docker import Docker +from DockerProxy import DockerProxy from ssh import SSH from DockerSSH import DockerSSH @@ -554,7 +554,7 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.__transfer_cluster_ssh_key_file(remote_target_dir=home_dir, controller_obj=controller_obj) if controller_obj.provider.type == Constants.DockerProvider: self.ssh.exec_command("mv {0}*.ipynb {1}".format(home_dir, - Docker.get_container_volume_from_working_dir( + DockerProxy.get_container_volume_from_working_dir( controller_obj.config["working_directory"]))) # If provider is Docker, then ipython controller and ipengines aren't started From 48d7f138be658b80278ec6434976d4e3dcd1a4eb Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 14 Feb 2017 12:43:33 -0800 Subject: [PATCH 095/100] fixes when mounting sshfs --- MolnsLib/ssh_deploy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 21d254e..e11a260 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -455,7 +455,7 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont # SSH mount the controller on each engine - remote_file_name='.ssh/id_dsa' + remote_file_name='/home/ubuntu/.ssh/controller_ssh_key' with open(controller_ssh_keyfile) as fd: sftp = self.ssh.open_sftp() controller_keyfile = sftp.file(remote_file_name, 'w') @@ -466,8 +466,9 @@ def deploy_ipython_engine(self, ip_address, controler_ip, engine_file_data, cont print "Remote file {0} has {1} bytes".format(remote_file_name, sftp.stat(remote_file_name).st_size) sftp.close() self.exec_command("chmod 0600 {0}".format(remote_file_name)) + self.exec_command("sudo rm -rf {0}".format('/home/ubuntu/shared')) self.exec_command("mkdir -p /home/ubuntu/shared") - self.exec_command("sshfs -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format(controler_ip)) + self.exec_command("sshfs -o IdentityFile={1} -o Ciphers=arcfour -o Compression=no -o reconnect -o idmap=user -o StrictHostKeyChecking=no ubuntu@{0}:/mnt/molnsshared /home/ubuntu/shared".format(controler_ip,remote_file_name)) # Update the Molnsutil package: TODO remove when molnsutil is stable #self.exec_command("cd /usr/local/molnsutil && git pull && sudo python setup.py install") From f2c8376d4295294ecf03a271fd4e031de41e2035 Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 21 Feb 2017 11:33:04 -0800 Subject: [PATCH 096/100] bug fix --- molns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/molns.py b/molns.py index f0523ec..766d1d4 100755 --- a/molns.py +++ b/molns.py @@ -1316,7 +1316,7 @@ def show_instances(cls, args, config): provider_name = provider_obj.name #print "provider_obj.type",provider_obj.type if i.worker_group_id is not None: - name = config.get_object_by_id(i.worker_id, 'WorkerGroup').name + name = config.get_object_by_id(i.worker_group_id, 'WorkerGroup').name itype = 'worker' else: name = config.get_object_by_id(i.controller_id, 'Controller').name From deb39fb0358b2e934c34970263878a50420717de Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 28 Feb 2017 09:02:21 -0800 Subject: [PATCH 097/100] using an updated gillespy --- MolnsLib/installSoftware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index 8c44502..e7becda 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -95,7 +95,7 @@ class InstallSW: "cd /usr/local/ode/ && STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE=/usr/local/ode/ make 1>stdout.log 2>stderr.log", "sudo rm -rf /usr/local/gillespy;sudo mkdir -p /usr/local/gillespy;sudo chown ubuntu /usr/local/gillespy", - "cd /usr/local/ && git clone https://github.com/MOLNs/gillespy.git", + "cd /usr/local/ && git clone https://github.com/briandrawert/gillespy.git", "cd /usr/local/gillespy && sudo STOCHKIT_HOME=/usr/local/StochKit/ STOCHKIT_ODE_HOME=/usr/local/ode/ python setup.py install" ], From 209c4d77d8f30ac4fff0ef2e48a8a969bad4a12f Mon Sep 17 00:00:00 2001 From: Brian Drawert Date: Tue, 28 Feb 2017 09:57:45 -0800 Subject: [PATCH 098/100] Changing name of docker class --- MolnsLib/DockerProxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/DockerProxy.py b/MolnsLib/DockerProxy.py index 122d911..5d0d6b6 100644 --- a/MolnsLib/DockerProxy.py +++ b/MolnsLib/DockerProxy.py @@ -4,7 +4,7 @@ import constants from molns_provider import ProviderBase from constants import Constants -from docker import Client +from docker import APIClient as Client from docker.errors import NotFound, NullResource, APIError From 9c18b6ccbfc4cc55346227ce6079e272e76b83e0 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 1 Mar 2017 11:02:44 -0800 Subject: [PATCH 099/100] refactor to remain compatible with updated docker-py --- MolnsLib/DockerProxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MolnsLib/DockerProxy.py b/MolnsLib/DockerProxy.py index 122d911..5d0d6b6 100644 --- a/MolnsLib/DockerProxy.py +++ b/MolnsLib/DockerProxy.py @@ -4,7 +4,7 @@ import constants from molns_provider import ProviderBase from constants import Constants -from docker import Client +from docker import APIClient as Client from docker.errors import NotFound, NullResource, APIError From 805b071d914673fd9fdbd385f7280139f6f2931c Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Wed, 1 Mar 2017 12:01:22 -0800 Subject: [PATCH 100/100] install new packages to image --- MolnsLib/installSoftware.py | 21 ++++++++++++++++----- MolnsLib/ssh_deploy.py | 11 +++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/MolnsLib/installSoftware.py b/MolnsLib/installSoftware.py index e7becda..a89fc90 100644 --- a/MolnsLib/installSoftware.py +++ b/MolnsLib/installSoftware.py @@ -40,7 +40,7 @@ class InstallSW: "sudo pip install dill cloud pygments", "sudo pip install tornado Jinja2", - # Molnsutil + # Molnsutil develop [ "sudo pip install jsonschema jsonpointer", # EC2/S3 and OpenStack APIs @@ -53,10 +53,21 @@ class InstallSW: "sudo pip install python-keystoneclient", "sudo pip install python-swiftclient", ], - # TODO pip install molnsutil while building image [ - "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", - "cd /usr/local/ && git clone https://github.com/aviral26/molnsutil.git && cd /usr/local/molnsutil && git checkout qsub_support" + "sudo rm -rf /usr/local/molnsutil;sudo mkdir -p /usr/local/molnsutil;sudo chown ubuntu /usr/local/molnsutil", + "cd /usr/local/ && git clone https://github.com/aviral26/molnsutil.git && cd /usr/local/molnsutil && git checkout qsub_support" + ], + + # Molns develop + [ + "sudo rm -rf /usr/local/molns;sudo mkdir -p /usr/local/molns;sudo chown ubuntu /usr/local/molns", + "cd /usr/local/ && git clone https://github.com/aviral26/molns.git && cd /usr/local/molns" + ], + + # Cluster execution + [ + "sudo rm -rf /usr/local/cluster_execution;sudo mkdir -p /usr/local/cluster_execution;sudo chown ubuntu /usr/local/cluster_execution", + "cd /usr/local/ && git clone https://github.com/aviral26/cluster_execution.git" ], # So the workers can mount the controller via SSHfs @@ -108,7 +119,7 @@ class InstallSW: "sudo apt-get install -y gmsh", ], - ["sudo apt-get install docker", "sudo pip install docker-py", "sudo pip install sqlalchemy", + ["sudo apt-get install docker", "sudo pip install docker", "sudo pip install sqlalchemy", "sudo pip install boto", "sudo pip install python-novaclient", "sudo pip install paramiko"], # pyurdme [ "sudo rm -rf /usr/local/pyurdme && sudo mkdir -p /usr/local/pyurdme && sudo chown ubuntu /usr/local/pyurdme", diff --git a/MolnsLib/ssh_deploy.py b/MolnsLib/ssh_deploy.py index 0ee71b1..2167192 100644 --- a/MolnsLib/ssh_deploy.py +++ b/MolnsLib/ssh_deploy.py @@ -588,14 +588,9 @@ def deploy_ipython_controller(self, instance, controller_obj, notebook_password= self.profile)) # TODO remove next three commands after testing. Put them in the image instead. - # Install cluster_execution - self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git") - - # Install molns - self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") - - # Temporary command until new image is created to use latest molnsutil. TODO pip install molnsutil - self.ssh.exec_command("cd /usr/local/molnsutil; git checkout qsub_support; git pull") + # self.ssh.exec_command("git clone https://github.com/aviral26/cluster_execution.git") + # self.ssh.exec_command("git clone https://github.com/aviral26/molns.git") + # self.ssh.exec_command("cd /usr/local/molnsutil; git checkout qsub_support; git pull") self.ssh.exec_command( "sudo iptables -t nat -A PREROUTING -i eth0 -p tcp --dport {0} -j REDIRECT --to-port {1}".format(