diff --git a/bootstrap-scripts/saltmaster-common.sh b/bootstrap-scripts/saltmaster-common.sh index 1f39786..1cfcb6d 100644 --- a/bootstrap-scripts/saltmaster-common.sh +++ b/bootstrap-scripts/saltmaster-common.sh @@ -245,4 +245,15 @@ kafka: EOF fi +cat << EOF >> /srv/salt/platform-salt/pillar/env_parameters.sls +hadoop_components_version: + hbase: $HBASE + hdfs: $HDFS + hive: $HIVE + oozie: $OOZIE + spark: $SPARK + spark2: $SPARK2 + yarn: $YARN +EOF + /tmp/saltmaster-gen-keys.sh diff --git a/cli/backend_base.py b/cli/backend_base.py index 0810c9e..4b13241 100755 --- a/cli/backend_base.py +++ b/cli/backend_base.py @@ -27,6 +27,7 @@ import Queue import StringIO import glob +import xml.etree.ElementTree as ET from threading import Thread @@ -264,6 +265,10 @@ def _get_volume_info(self, node_type, config_file): def _set_up_env_conf(self): self._write_pnda_env_sh(self._cluster) + # Write hadoop components in pnda_env file + if self._pnda_env['hadoop']['HADOOP_DISTRO'] == 'HDP': + self._write_components_version(self._pnda_env['mirrors']['PNDA_MIRROR'], + self._cluster) self._ssh_client.write_ssh_config(self._get_bastion_ip(), self._pnda_env['infrastructure']['OS_USER'], os.path.abspath(self._keyfile)) @@ -278,6 +283,56 @@ def _write_pnda_env_sh(self, cluster): self._pnda_env[section][setting], (list, tuple)) else self._pnda_env[section][setting] pnda_env_sh_file.write('export %s=%s\n' % (setting, val)) + def _write_components_version(self, mirror, cluster): + manifest_file = self._get_hdp_manifest_file(mirror) + components = self._hdp_components_version(manifest_file) + if components: + for component in components: + with open('cli/pnda_env_%s.sh' % cluster, 'a') as pnda_env_sh_file: + pnda_env_sh_file.write('export %s=%s\n' % (component, components[component])) + + def _hdp_components_version(self, manifest_file): + components_version = dict() + response = requests.get(manifest_file) + if response.status_code == 200: + root = ET.fromstring(response.content) + for child in root: + if child.tag == 'manifest': + for attr in child: + components_version[attr.attrib.get('name', 'default')] = attr.attrib.get('version', '') + break + return components_version + + def _get_hdp_manifest_file(self, mirror): + # Fetch salt details from pnda_env.yaml file + if 'PLATFORM_SALT_LOCAL' in self._pnda_env['platform_salt']: + salt_repo = "%s/pillar/services.sls" % (self._pnda_env['platform_salt']['PLATFORM_SALT_LOCAL']) + with open(salt_repo, 'r') as data: + key_value = yaml.load(data) + hdp_version_key = key_value.get('hdp', '') + if hdp_version_key: + hdp_version =hdp_version_key.get('version', '') + if not hdp_version: + CONSOLE.error("Not able to fetch hdp version from platform satl repo") + sys.exit(1) + else: + CONSOLE.error("Not able to fetch hdp version from platform satl repo") + sys.exit(1) + else: + git_repo_url = self._pnda_env['platform_salt']['PLATFORM_GIT_REPO_URI'] + user_name = git_repo_url.split('/')[-2] + salt_repo = "https://raw.githubusercontent.com/%s/platform-salt/%s/pillar/services.sls" %\ + (user_name, self._pnda_env['platform_salt']['PLATFORM_GIT_BRANCH']) + response = requests.get(salt_repo) + if response.status_code == 200: + data = yaml.load(response.text) + hdp_version = data['hdp']['version'] + else: + CONSOLE.error("Not able to fetch hdp version from platform satl repo") + sys.exit(1) + + return "%s%s%s/HDP-%s-MAINT.xml" %(mirror, "/mirror_hdp/HDP/centos7/", hdp_version, hdp_version) + def _bootstrap(self, instance, saltmaster, cluster, flavor, branch, salt_tarball, certs_tarball, error_queue, bootstrap_files=None, bootstrap_commands=None):