From adfe2ef63f1608a9172dc6efb4b1762ba453e107 Mon Sep 17 00:00:00 2001 From: adamlavie Date: Tue, 28 Jun 2016 23:02:07 +0300 Subject: [PATCH 1/5] CFY-5334 add SSL support --- .../manager/scripts/creation_validation.py | 1 - components/manager/scripts/sanity/sanity.py | 64 +++--- components/nginx/scripts/start.py | 10 +- components/restservice/scripts/start.py | 24 +-- components/utils.py | 201 ++++++++++++------ 5 files changed, 173 insertions(+), 127 deletions(-) diff --git a/components/manager/scripts/creation_validation.py b/components/manager/scripts/creation_validation.py index 86b1b8a89..65f2c3e86 100644 --- a/components/manager/scripts/creation_validation.py +++ b/components/manager/scripts/creation_validation.py @@ -10,7 +10,6 @@ import utils # NOQA IMMUTABLE_PROPERTIES = [ - 'security', 'ssh_user' ] diff --git a/components/manager/scripts/sanity/sanity.py b/components/manager/scripts/sanity/sanity.py index db5d3870a..5feb55f2f 100644 --- a/components/manager/scripts/sanity/sanity.py +++ b/components/manager/scripts/sanity/sanity.py @@ -44,7 +44,7 @@ def _upload_app_blueprint(app_tar): app_data = f.read() length = os.path.getsize(app_tar) - headers = utils.create_maintenance_headers() + headers = {} headers['Content-Length'] = length headers['Content-Type'] = 'application/octet-stream' params = urllib.urlencode( @@ -53,9 +53,10 @@ def _upload_app_blueprint(app_tar): endpoint = '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID) url = endpoint + '?' + params - utils.http_request(url, + utils.rest_request(url, data=app_data, - headers=headers) + headers=headers, + method='PUT') def _deploy_app(): @@ -69,13 +70,13 @@ def _deploy_app(): 'blueprint_id': BLUEPRINT_ID, 'inputs': dep_inputs } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - utils.http_request( + utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), data=json.dumps(data), - headers=headers) + headers=headers, + method='PUT') # Waiting for create deployment env to end utils.repetitive( @@ -92,10 +93,9 @@ def _install_sanity_app(): 'deployment_id': DEPLOYMENT_ID, 'workflow_id': 'install' } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - resp = utils.http_request( + resp = utils.rest_request( '{0}/executions'.format(_get_url_prefix()), method='POST', data=json.dumps(data), @@ -112,27 +112,26 @@ def _install_sanity_app(): timeout_msg='Timed out while waiting for ' 'deployment {0} to install'.format(DEPLOYMENT_ID)) - resp_content = resp.readlines() - json_resp = json.loads(resp_content[0]) + json_resp = json.loads(resp.content) return json_resp['id'] def _assert_logs_and_events(execution_id): - headers = utils.create_maintenance_headers() params = urllib.urlencode( dict(execution_id=execution_id, type='cloudify_log')) endpoint = '{0}/events'.format(_get_url_prefix()) url = endpoint + '?' + params - resp = utils.http_request(url, method='GET', headers=headers, timeout=30) + resp = utils.rest_request(url, + method='GET', + timeout=30) if not resp: ctx.abort_operation("Can't connect to elasticsearch") if resp.code != 200: ctx.abort_operation('Failed to retrieve logs/events') - resp_content = resp.readlines() - json_resp = json.loads(resp_content[0]) + json_resp = json.loads(resp.content) if 'items' not in json_resp or not json_resp['items']: ctx.abort_operation('No logs/events received') @@ -165,10 +164,9 @@ def _uninstall_sanity_app(): 'deployment_id': DEPLOYMENT_ID, 'workflow_id': 'uninstall' } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - utils.http_request( + utils.rest_request( '{0}/executions'.format(_get_url_prefix()), method='POST', data=json.dumps(data), @@ -189,12 +187,10 @@ def _uninstall_sanity_app(): def _delete_sanity_deployment(): if not _is_sanity_dep_exist(): return - headers = utils.create_maintenance_headers() - resp = utils.http_request( + resp = utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), - method='DELETE', - headers=headers) + method='DELETE') if resp.code != 200: ctx.abort_operation('Failed deleting ' @@ -205,11 +201,9 @@ def _delete_sanity_deployment(): def _delete_sanity_blueprint(): if not _is_sanity_blueprint_exist(): return - headers = utils.create_maintenance_headers() - resp = utils.http_request( + resp = utils.rest_request( '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID), - method='DELETE', - headers=headers) + method='DELETE') if resp.code != 200: ctx.abort_operation('Failed deleting ' @@ -223,11 +217,9 @@ def _delete_key_file(): def _is_sanity_dep_exist(should_fail=False): - headers = utils.create_maintenance_headers() - res = utils.http_request( + res = utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), method='GET', - headers=headers, should_fail=should_fail) if not res: return False @@ -235,11 +227,9 @@ def _is_sanity_dep_exist(should_fail=False): def _is_sanity_blueprint_exist(should_fail=False): - headers = utils.create_maintenance_headers() - res = utils.http_request( + res = utils.rest_request( '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID), method='GET', - headers=headers, should_fail=should_fail) if not res: return False @@ -276,7 +266,15 @@ def perform_sanity(): perform_sanity() if utils.is_upgrade or utils.is_rollback: + # Restore the snapshot at the end of the workflow. utils.restore_upgrade_snapshot() if utils.is_upgrade: + # To keep the upgrade workflow idempotent, this flag is used to figure + # out if the next upgrade should dispose of old rollback data. utils.set_upgrade_success_in_upgrade_meta() + +if utils.is_rollback: + # remove data created by the upgrade process. + utils.remove(utils.UPGRADE_METADATA_FILE) + utils.remove(utils.ES_UPGRADE_DUMP_PATH) diff --git a/components/nginx/scripts/start.py b/components/nginx/scripts/start.py index 16c9f1782..0b345f69e 100644 --- a/components/nginx/scripts/start.py +++ b/components/nginx/scripts/start.py @@ -25,12 +25,6 @@ def check_response(response): utils.start_service(NGINX_SERVICE_NAME, append_prefix=False) utils.systemd.verify_alive(NGINX_SERVICE_NAME, append_prefix=False) -nginx_url = 'http://127.0.0.1/api/v2.1/blueprints' +nginx_url = '127.0.0.1/api/v2.1/blueprints' -if utils.is_upgrade or utils.is_rollback: - headers = utils.create_maintenance_headers() -else: - headers = utils.get_auth_headers(True) - -utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response, - headers=headers) +utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response) diff --git a/components/restservice/scripts/start.py b/components/restservice/scripts/start.py index 6f4157687..1cfaf6c39 100644 --- a/components/restservice/scripts/start.py +++ b/components/restservice/scripts/start.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import json +import httplib import urllib2 -import urlparse from os.path import join, dirname from cloudify import ctx @@ -23,22 +23,10 @@ def verify_restservice(url): that also requires the storage backend to be up, so if it works, there's a good chance everything is configured correctly. """ - blueprints_url = urlparse.urljoin(url, 'api/v2.1/blueprints') - - headers = utils.get_auth_headers(True) - - if utils.is_upgrade or utils.is_rollback: - # if we're doing an upgrade, we're in maintenance mode - this request - # is safe to perform in maintenance mode, so let's bypass the check - headers = utils.create_maintenance_headers() - else: - headers = utils.get_auth_headers(True) - - req = urllib2.Request(blueprints_url, headers=headers) - + blueprints_url = '{0}/{1}'.format(url, 'api/v2.1/blueprints') try: - response = urllib2.urlopen(req) - except urllib2.URLError as e: + response = utils.rest_request(blueprints_url) + except (urllib2.URLError, httplib.HTTPException) as e: ctx.abort_operation('REST service returned an invalid response: {0}' .format(e)) if response.code == 401: @@ -50,7 +38,7 @@ def verify_restservice(url): .format(response.code)) try: - json.load(response) + json.loads(response.content) except ValueError as e: ctx.abort_operation('REST service returned malformed JSON: {0}' .format(e)) @@ -61,6 +49,6 @@ def verify_restservice(url): utils.systemd.verify_alive(REST_SERVICE_NAME) -restservice_url = 'http://{0}:{1}'.format('127.0.0.1', 8100) +restservice_url = '127.0.0.1' utils.verify_service_http(REST_SERVICE_NAME, restservice_url) verify_restservice(restservice_url) diff --git a/components/utils.py b/components/utils.py index 3f7f9fe18..999fd541a 100644 --- a/components/utils.py +++ b/components/utils.py @@ -11,10 +11,12 @@ import socket import urllib import urllib2 +import httplib import hashlib import tempfile import subprocess from functools import wraps +from urlparse import urlparse from time import sleep, gmtime, strftime from distutils.version import LooseVersion @@ -1099,7 +1101,41 @@ def start_service(service_name, append_prefix=True, ignore_restart_fail=False): systemd.start(service_name, append_prefix=append_prefix) -def http_request(url, data=None, method='PUT', +def rest_request(url, **request_kwargs): + headers = request_kwargs.get('headers', {}) + auth_headers = get_auth_headers() + headers.update(auth_headers) + if is_upgrade or is_rollback: + headers.update({'X-BYPASS-MAINTENANCE': 'True'}) + request_kwargs['headers'] = headers + + manager_config = _get_curr_manager_config() + security = manager_config['security'] + ssl_enabled = security.get('ssl', {}).get('enabled') + + if ssl_enabled: + if not url.startswith('https'): + url = 'https://{0}'.format(url) + server_crt, server_key = _get_rest_key_and_crt() + request_kwargs.update({'server_crt': server_crt, + 'server_key': server_key}) + res = https_request(url, **request_kwargs) + return RestServiceResponse(res.status, res.read()) + else: + if not url.startswith('http'): + url = 'http://{0}'.format(url) + res = http_request(url, **request_kwargs) + return RestServiceResponse(res.code, res.read()) + + +class RestServiceResponse(object): + + def __init__(self, code, content): + self.code = code + self.content = content + + +def http_request(url, data=None, method='GET', headers=None, timeout=None, should_fail=False): headers = headers or {} request = urllib2.Request(url, data=data, headers=headers) @@ -1112,22 +1148,55 @@ def http_request(url, data=None, method='PUT', if not should_fail: ctx.logger.error('Failed to {0} {1} (reason: {2})'.format( method, url, e.reason)) + raise e + + +def https_request(url, data=None, method='GET', headers=None, timeout=None, + should_fail=False, server_crt=None, server_key=None): + headers = headers or {} + parsed_uri = urlparse(url) + host = parsed_uri.netloc + if ':' not in host: + host += ':443' + uri = parsed_uri.path + if parsed_uri.query: + uri += '?{0}'.format(parsed_uri.query) + ctx.logger.info('URL: {0}, URI: {1}, method {2}, headers {3} server.cert ' + '{4} server.key {5}'.format(url, uri, method, + headers, server_crt, + server_key)) + con = httplib.HTTPSConnection(host=host, + cert_file=server_crt, + key_file=server_key, + timeout=timeout) + try: + con.request(method, uri, body=data, headers=headers) + return con.getresponse() + except httplib.HTTPException as e: + if not should_fail: + ctx.logger.error('Failed to {0} {1} (reason: {2})'.format( + method, url, e.reason)) + raise e + + +def _get_rest_key_and_crt(): + nginx_resources = resource_factory.get_resources_dir('nginx') + server_key = os.path.join(nginx_resources, 'server.key') + server_crt = os.path.join(nginx_resources, 'server.crt') + return server_crt, server_key def wait_for_workflow( deployment_id, workflow_id, - url_prefix='http://localhost/api/{0}'.format(REST_VERSION)): - headers = create_maintenance_headers() + url_prefix='localhost/api/{0}'.format(REST_VERSION)): params = urllib.urlencode(dict(deployment_id=deployment_id)) endpoint = '{0}/executions'.format(url_prefix) url = endpoint + '?' + params - res = http_request( - url, - method='GET', - headers=headers) - res_content = res.readlines() - json_res = json.loads(res_content[0]) + + res = rest_request(url, method='GET') + json_res = json.loads(res.content) + for execution in json_res['items']: if execution['workflow_id'] == workflow_id: execution_status = execution['status'] @@ -1139,32 +1208,31 @@ def wait_for_workflow( return False -def _wait_for_execution(execution_id, headers): +def _wait_for_execution(execution_id): poll_interval = 2 while True: - res = _list_executions_with_retries(headers, execution_id) - content = json.loads(res.readlines()[0]) + res = _list_executions_with_retries(execution_id) + content = json.loads(res.content) execution_item = content['items'][0] execution_status = execution_item['status'] if execution_status == 'terminated': - return True + return elif execution_status == 'failed': ctx.abort_operation('Execution with id {0} failed'. format(execution_id)) sleep(poll_interval) -def _list_executions_with_retries(headers, execution_id, retries=6): +def _list_executions_with_retries(execution_id, retries=6): count = 0 err = 'Failed listing existing executions.' - url = 'http://localhost/api/{0}/executions?' \ - '_include_system_workflows=true&id={1}'.format(REST_VERSION, - execution_id) + url = 'localhost/api/{0}/executions?_include_system_workflows=true&id={1}'\ + .format(REST_VERSION, execution_id) while count != retries: - res = http_request(url, method='GET', headers=headers) + res = rest_request(url, method='GET') if res.code != 200: err = 'Failed listing existing executions. Message: {0}' \ - .format(res.readlines()) + .format(res.content) ctx.logger.error(err) sleep(2) else: @@ -1172,20 +1240,10 @@ def _list_executions_with_retries(headers, execution_id, retries=6): ctx.abort_operation(err) -def create_maintenance_headers(upgrade_props=True): - headers = {'X-BYPASS-MAINTENANCE': 'True'} - auth_props = get_auth_headers(upgrade_props) - headers.update(auth_props) - return headers - - -def get_auth_headers(upgrade_props): +def get_auth_headers(): headers = {} - if upgrade_props: - config = ctx_factory.get('manager-config') - else: - config = ctx_factory.load_rollback_props('manager-config') - security = config['security'] + manager_config = _get_curr_manager_config() + security = manager_config['security'] security_enabled = security['enabled'] if security_enabled: username = security.get('admin_username') @@ -1196,28 +1254,36 @@ def get_auth_headers(upgrade_props): return headers +def _get_curr_manager_config(): + nginx_upgraded = os.path.isdir( + ctx_factory.get_rollback_properties_dir('nginx')) + if nginx_upgraded: + config = ctx_factory.get('manager-config') + else: + config = ctx_factory.load_rollback_props('manager-config') or \ + ctx_factory.get('manager-config') + return config + + def create_upgrade_snapshot(): if _get_upgrade_data().get('snapshot_id'): ctx.logger.debug('Upgrade snapshot already created.') return snapshot_id = _generate_upgrade_snapshot_id() - url = 'http://localhost/api/{0}/snapshots/{1}'.format(REST_VERSION, - snapshot_id) + url = 'localhost/api/{0}/snapshots/{1}'.format(REST_VERSION, snapshot_id) data = json.dumps({'include_metrics': 'true', 'include_credentials': 'true'}) - headers = create_maintenance_headers(upgrade_props=False) - req_headers = headers.copy() - req_headers.update({'Content-Type': 'application/json'}) + req_headers = {'Content-Type': 'application/json'} ctx.logger.info('Creating snapshot with ID {0}' .format(snapshot_id)) - res = http_request(url, data=data, method='PUT', headers=req_headers) + res = rest_request(url, data=data, method='PUT', headers=req_headers) if res.code != 201: err = 'Failed creating snapshot {0}. Message: {1}'\ - .format(snapshot_id, res.readlines()) + .format(snapshot_id, res.content) ctx.logger.error(err) ctx.abort_operation(err) - execution_id = json.loads(res.readlines()[0])['id'] - _wait_for_execution(execution_id, headers) + execution_id = json.loads(res.content)['id'] + _wait_for_execution(execution_id) ctx.logger.info('Snapshot with ID {0} created successfully' .format(snapshot_id)) ctx.logger.info('Setting snapshot info to upgrade metadata in {0}'. @@ -1227,37 +1293,35 @@ def create_upgrade_snapshot(): def restore_upgrade_snapshot(): snapshot_id = _get_upgrade_data()['snapshot_id'] - url = 'http://localhost/api/{0}/snapshots/{1}/restore'.format(REST_VERSION, - snapshot_id) + url = 'localhost/api/{0}/snapshots/{1}/restore'.format(REST_VERSION, + snapshot_id) data = json.dumps({'recreate_deployments_envs': 'false', 'force': 'true'}) - headers = create_maintenance_headers(upgrade_props=True) - req_headers = headers.copy() - req_headers.update({'Content-Type': 'application/json'}) + req_headers = {'Content-Type': 'application/json'} ctx.logger.info('Restoring snapshot with ID {0}'.format(snapshot_id)) - res = http_request(url, data=data, method='POST', headers=req_headers) + res = rest_request(url, data=data, method='POST', headers=req_headers) if res.code != 200: err = 'Failed restoring snapshot {0}. Message: {1}' \ - .format(snapshot_id, res.readlines()) + .format(snapshot_id, res.content) ctx.logger.error(err) ctx.abort_operation(err) - execution_id = json.loads(res.readlines()[0])['id'] - _wait_for_execution(execution_id, headers) + execution_id = json.loads(res.content)['id'] + _wait_for_execution(execution_id) ctx.logger.info('Snapshot with ID {0} restored successfully' .format(snapshot_id)) def _generate_upgrade_snapshot_id(): - url = 'http://localhost/api/{0}/version'.format(REST_VERSION) - auth_headers = get_auth_headers(upgrade_props=False) - res = http_request(url, method='GET', headers=auth_headers) + url = 'localhost/api/{0}/version'.format(REST_VERSION) + auth_headers = get_auth_headers() + res = rest_request(url, method='GET', headers=auth_headers) if res.code != 200: err = 'Failed extracting current manager version. Message: {0}' \ - .format(res.readlines()) + .format(res.content) ctx.logger.error(err) ctx.abort_operation(err) curr_time = strftime("%Y-%m-%d_%H:%M:%S", gmtime()) - version_data = json.loads(res.read()) + version_data = json.loads(res.content) snapshot_upgrade_name = 'upgrade_snapshot_{0}_build_{1}_{2}' \ .format(version_data['version'], version_data['build'], @@ -1284,11 +1348,14 @@ def _get_upgrade_data(): @retry((IOError, ValueError)) -def check_http_response(url, predicate=None, **request_kwargs): - req = urllib2.Request(url, **request_kwargs) +def check_http_response(url, service_name, predicate=None, + **request_kwargs): try: - response = urllib2.urlopen(req) - except urllib2.HTTPError as e: + if service_name in ('nginx', 'restservice'): + response = rest_request(url, **request_kwargs) + else: + response = http_request(url, **request_kwargs) + except (urllib2.HTTPError, httplib.HTTPException) as e: # HTTPError can also be used as a non-200 response. Pass this # through to the predicate function, so it can decide if a # non-200 response is fine or not. @@ -1301,7 +1368,7 @@ def check_http_response(url, predicate=None, **request_kwargs): def verify_service_http(service_name, url, *args, **kwargs): try: - return check_http_response(url, *args, **kwargs) + return check_http_response(url, service_name, *args, **kwargs) except (IOError, ValueError) as e: ctx.abort_operation('{0} error: {1}: {2}'.format(service_name, url, e)) @@ -1372,19 +1439,19 @@ def verify_immutable_properties(service_name, properties): def _is_version_greater_than_curr(new_version): - version_url = 'http://localhost/api/{0}/version'.format(REST_VERSION) - version_res = http_request(version_url, method='GET') + version_url = 'localhost/api/{0}/version'.format(REST_VERSION) + version_res = rest_request(version_url, method='GET') if version_res.code != 200: ctx.abort_operation('Failed retrieving manager version') - curr_version = json.loads(version_res.readlines()[0])['version'] + curr_version = json.loads(version_res.content)['version'] ctx.logger.info('Current manager version is {0}.'.format(curr_version)) return LooseVersion(new_version) > LooseVersion(curr_version) -# rollback resources will be removed only if the last upgrade passed -# successfully and the 'upgrade to' version is greater than the current version -# # This function MUST be invoked by the first node and before upgrade snapshot -# is created. +# rollback resources will be removed upon upgrade execution only if the last +# upgrade passed successfully and the 'upgrade to' version is greater than +# the current version. This function MUST be invoked by the first node and +# before upgrade snapshot is created. def clean_rollback_resources_if_necessary(): if not is_upgrade: return From 5f7062f7537a805e8a20e68d5ce894098e7e4c7d Mon Sep 17 00:00:00 2001 From: adamlavie Date: Tue, 28 Jun 2016 23:02:07 +0300 Subject: [PATCH 2/5] CFY-5334 add SSL support --- .../manager/scripts/creation_validation.py | 1 - components/manager/scripts/sanity/sanity.py | 64 +++--- components/nginx/scripts/start.py | 10 +- components/restservice/scripts/start.py | 24 +- components/utils.py | 208 ++++++++++++------ 5 files changed, 176 insertions(+), 131 deletions(-) diff --git a/components/manager/scripts/creation_validation.py b/components/manager/scripts/creation_validation.py index 86b1b8a89..65f2c3e86 100644 --- a/components/manager/scripts/creation_validation.py +++ b/components/manager/scripts/creation_validation.py @@ -10,7 +10,6 @@ import utils # NOQA IMMUTABLE_PROPERTIES = [ - 'security', 'ssh_user' ] diff --git a/components/manager/scripts/sanity/sanity.py b/components/manager/scripts/sanity/sanity.py index db5d3870a..5feb55f2f 100644 --- a/components/manager/scripts/sanity/sanity.py +++ b/components/manager/scripts/sanity/sanity.py @@ -44,7 +44,7 @@ def _upload_app_blueprint(app_tar): app_data = f.read() length = os.path.getsize(app_tar) - headers = utils.create_maintenance_headers() + headers = {} headers['Content-Length'] = length headers['Content-Type'] = 'application/octet-stream' params = urllib.urlencode( @@ -53,9 +53,10 @@ def _upload_app_blueprint(app_tar): endpoint = '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID) url = endpoint + '?' + params - utils.http_request(url, + utils.rest_request(url, data=app_data, - headers=headers) + headers=headers, + method='PUT') def _deploy_app(): @@ -69,13 +70,13 @@ def _deploy_app(): 'blueprint_id': BLUEPRINT_ID, 'inputs': dep_inputs } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - utils.http_request( + utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), data=json.dumps(data), - headers=headers) + headers=headers, + method='PUT') # Waiting for create deployment env to end utils.repetitive( @@ -92,10 +93,9 @@ def _install_sanity_app(): 'deployment_id': DEPLOYMENT_ID, 'workflow_id': 'install' } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - resp = utils.http_request( + resp = utils.rest_request( '{0}/executions'.format(_get_url_prefix()), method='POST', data=json.dumps(data), @@ -112,27 +112,26 @@ def _install_sanity_app(): timeout_msg='Timed out while waiting for ' 'deployment {0} to install'.format(DEPLOYMENT_ID)) - resp_content = resp.readlines() - json_resp = json.loads(resp_content[0]) + json_resp = json.loads(resp.content) return json_resp['id'] def _assert_logs_and_events(execution_id): - headers = utils.create_maintenance_headers() params = urllib.urlencode( dict(execution_id=execution_id, type='cloudify_log')) endpoint = '{0}/events'.format(_get_url_prefix()) url = endpoint + '?' + params - resp = utils.http_request(url, method='GET', headers=headers, timeout=30) + resp = utils.rest_request(url, + method='GET', + timeout=30) if not resp: ctx.abort_operation("Can't connect to elasticsearch") if resp.code != 200: ctx.abort_operation('Failed to retrieve logs/events') - resp_content = resp.readlines() - json_resp = json.loads(resp_content[0]) + json_resp = json.loads(resp.content) if 'items' not in json_resp or not json_resp['items']: ctx.abort_operation('No logs/events received') @@ -165,10 +164,9 @@ def _uninstall_sanity_app(): 'deployment_id': DEPLOYMENT_ID, 'workflow_id': 'uninstall' } - headers = utils.create_maintenance_headers() - headers.update({'content-type': 'application/json'}) + headers = {'content-type': 'application/json'} - utils.http_request( + utils.rest_request( '{0}/executions'.format(_get_url_prefix()), method='POST', data=json.dumps(data), @@ -189,12 +187,10 @@ def _uninstall_sanity_app(): def _delete_sanity_deployment(): if not _is_sanity_dep_exist(): return - headers = utils.create_maintenance_headers() - resp = utils.http_request( + resp = utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), - method='DELETE', - headers=headers) + method='DELETE') if resp.code != 200: ctx.abort_operation('Failed deleting ' @@ -205,11 +201,9 @@ def _delete_sanity_deployment(): def _delete_sanity_blueprint(): if not _is_sanity_blueprint_exist(): return - headers = utils.create_maintenance_headers() - resp = utils.http_request( + resp = utils.rest_request( '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID), - method='DELETE', - headers=headers) + method='DELETE') if resp.code != 200: ctx.abort_operation('Failed deleting ' @@ -223,11 +217,9 @@ def _delete_key_file(): def _is_sanity_dep_exist(should_fail=False): - headers = utils.create_maintenance_headers() - res = utils.http_request( + res = utils.rest_request( '{0}/deployments/{1}'.format(_get_url_prefix(), DEPLOYMENT_ID), method='GET', - headers=headers, should_fail=should_fail) if not res: return False @@ -235,11 +227,9 @@ def _is_sanity_dep_exist(should_fail=False): def _is_sanity_blueprint_exist(should_fail=False): - headers = utils.create_maintenance_headers() - res = utils.http_request( + res = utils.rest_request( '{0}/blueprints/{1}'.format(_get_url_prefix(), BLUEPRINT_ID), method='GET', - headers=headers, should_fail=should_fail) if not res: return False @@ -276,7 +266,15 @@ def perform_sanity(): perform_sanity() if utils.is_upgrade or utils.is_rollback: + # Restore the snapshot at the end of the workflow. utils.restore_upgrade_snapshot() if utils.is_upgrade: + # To keep the upgrade workflow idempotent, this flag is used to figure + # out if the next upgrade should dispose of old rollback data. utils.set_upgrade_success_in_upgrade_meta() + +if utils.is_rollback: + # remove data created by the upgrade process. + utils.remove(utils.UPGRADE_METADATA_FILE) + utils.remove(utils.ES_UPGRADE_DUMP_PATH) diff --git a/components/nginx/scripts/start.py b/components/nginx/scripts/start.py index cb35d2044..0b345f69e 100644 --- a/components/nginx/scripts/start.py +++ b/components/nginx/scripts/start.py @@ -25,12 +25,6 @@ def check_response(response): utils.start_service(NGINX_SERVICE_NAME, append_prefix=False) utils.systemd.verify_alive(NGINX_SERVICE_NAME, append_prefix=False) -nginx_url = '{0}://127.0.0.1/api/v2.1/version'.format( - ctx.instance.runtime_properties['rest_protocol']) +nginx_url = '127.0.0.1/api/v2.1/blueprints' -headers = {} -if utils.is_upgrade or utils.is_rollback: - headers = utils.create_maintenance_headers() - -utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response, - headers=headers) +utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response) diff --git a/components/restservice/scripts/start.py b/components/restservice/scripts/start.py index 6f4157687..1cfaf6c39 100644 --- a/components/restservice/scripts/start.py +++ b/components/restservice/scripts/start.py @@ -1,8 +1,8 @@ #!/usr/bin/env python import json +import httplib import urllib2 -import urlparse from os.path import join, dirname from cloudify import ctx @@ -23,22 +23,10 @@ def verify_restservice(url): that also requires the storage backend to be up, so if it works, there's a good chance everything is configured correctly. """ - blueprints_url = urlparse.urljoin(url, 'api/v2.1/blueprints') - - headers = utils.get_auth_headers(True) - - if utils.is_upgrade or utils.is_rollback: - # if we're doing an upgrade, we're in maintenance mode - this request - # is safe to perform in maintenance mode, so let's bypass the check - headers = utils.create_maintenance_headers() - else: - headers = utils.get_auth_headers(True) - - req = urllib2.Request(blueprints_url, headers=headers) - + blueprints_url = '{0}/{1}'.format(url, 'api/v2.1/blueprints') try: - response = urllib2.urlopen(req) - except urllib2.URLError as e: + response = utils.rest_request(blueprints_url) + except (urllib2.URLError, httplib.HTTPException) as e: ctx.abort_operation('REST service returned an invalid response: {0}' .format(e)) if response.code == 401: @@ -50,7 +38,7 @@ def verify_restservice(url): .format(response.code)) try: - json.load(response) + json.loads(response.content) except ValueError as e: ctx.abort_operation('REST service returned malformed JSON: {0}' .format(e)) @@ -61,6 +49,6 @@ def verify_restservice(url): utils.systemd.verify_alive(REST_SERVICE_NAME) -restservice_url = 'http://{0}:{1}'.format('127.0.0.1', 8100) +restservice_url = '127.0.0.1' utils.verify_service_http(REST_SERVICE_NAME, restservice_url) verify_restservice(restservice_url) diff --git a/components/utils.py b/components/utils.py index e786c4555..c07b87eb5 100644 --- a/components/utils.py +++ b/components/utils.py @@ -11,10 +11,12 @@ import socket import urllib import urllib2 +import httplib import hashlib import tempfile import subprocess from functools import wraps +from urlparse import urlparse from time import sleep, gmtime, strftime from distutils.version import LooseVersion @@ -1211,7 +1213,41 @@ def start_service(service_name, append_prefix=True, ignore_restart_fail=False): systemd.start(service_name, append_prefix=append_prefix) -def http_request(url, data=None, method='PUT', +def rest_request(url, **request_kwargs): + headers = request_kwargs.get('headers', {}) + auth_headers = get_auth_headers() + headers.update(auth_headers) + if is_upgrade or is_rollback: + headers.update({'X-BYPASS-MAINTENANCE': 'True'}) + request_kwargs['headers'] = headers + + manager_config = _get_curr_manager_config() + security = manager_config['security'] + ssl_enabled = security.get('ssl', {}).get('enabled') + + if ssl_enabled: + if not url.startswith('https'): + url = 'https://{0}'.format(url) + server_crt, server_key = _get_rest_key_and_crt() + request_kwargs.update({'server_crt': server_crt, + 'server_key': server_key}) + res = https_request(url, **request_kwargs) + return RestServiceResponse(res.status, res.read()) + else: + if not url.startswith('http'): + url = 'http://{0}'.format(url) + res = http_request(url, **request_kwargs) + return RestServiceResponse(res.code, res.read()) + + +class RestServiceResponse(object): + + def __init__(self, code, content): + self.code = code + self.content = content + + +def http_request(url, data=None, method='GET', headers=None, timeout=None, should_fail=False): headers = headers or {} request = urllib2.Request(url, data=data, headers=headers) @@ -1224,22 +1260,55 @@ def http_request(url, data=None, method='PUT', if not should_fail: ctx.logger.error('Failed to {0} {1} (reason: {2})'.format( method, url, e.reason)) + raise e + + +def https_request(url, data=None, method='GET', headers=None, timeout=None, + should_fail=False, server_crt=None, server_key=None): + headers = headers or {} + parsed_uri = urlparse(url) + host = parsed_uri.netloc + if ':' not in host: + host += ':443' + uri = parsed_uri.path + if parsed_uri.query: + uri += '?{0}'.format(parsed_uri.query) + ctx.logger.info('URL: {0}, URI: {1}, method {2}, headers {3} server.cert ' + '{4} server.key {5}'.format(url, uri, method, + headers, server_crt, + server_key)) + con = httplib.HTTPSConnection(host=host, + cert_file=server_crt, + key_file=server_key, + timeout=timeout) + try: + con.request(method, uri, body=data, headers=headers) + return con.getresponse() + except httplib.HTTPException as e: + if not should_fail: + ctx.logger.error('Failed to {0} {1} (reason: {2})'.format( + method, url, e.reason)) + raise e + + +def _get_rest_key_and_crt(): + nginx_resources = resource_factory.get_resources_dir('nginx') + server_key = os.path.join(nginx_resources, 'server.key') + server_crt = os.path.join(nginx_resources, 'server.crt') + return server_crt, server_key def wait_for_workflow( deployment_id, workflow_id, - url_prefix='http://localhost/api/{0}'.format(REST_VERSION)): - headers = create_maintenance_headers() + url_prefix='localhost/api/{0}'.format(REST_VERSION)): params = urllib.urlencode(dict(deployment_id=deployment_id)) endpoint = '{0}/executions'.format(url_prefix) url = endpoint + '?' + params - res = http_request( - url, - method='GET', - headers=headers) - res_content = res.readlines() - json_res = json.loads(res_content[0]) + + res = rest_request(url, method='GET') + json_res = json.loads(res.content) + for execution in json_res['items']: if execution['workflow_id'] == workflow_id: execution_status = execution['status'] @@ -1251,32 +1320,31 @@ def wait_for_workflow( return False -def _wait_for_execution(execution_id, headers): +def _wait_for_execution(execution_id): poll_interval = 2 while True: - res = _list_executions_with_retries(headers, execution_id) - content = json.loads(res.readlines()[0]) + res = _list_executions_with_retries(execution_id) + content = json.loads(res.content) execution_item = content['items'][0] execution_status = execution_item['status'] if execution_status == 'terminated': - return True + return elif execution_status == 'failed': ctx.abort_operation('Execution with id {0} failed'. format(execution_id)) sleep(poll_interval) -def _list_executions_with_retries(headers, execution_id, retries=6): +def _list_executions_with_retries(execution_id, retries=6): count = 0 err = 'Failed listing existing executions.' - url = 'http://localhost/api/{0}/executions?' \ - '_include_system_workflows=true&id={1}'.format(REST_VERSION, - execution_id) + url = 'localhost/api/{0}/executions?_include_system_workflows=true&id={1}'\ + .format(REST_VERSION, execution_id) while count != retries: - res = http_request(url, method='GET', headers=headers) + res = rest_request(url, method='GET') if res.code != 200: err = 'Failed listing existing executions. Message: {0}' \ - .format(res.readlines()) + .format(res.content) ctx.logger.error(err) sleep(2) else: @@ -1284,53 +1352,50 @@ def _list_executions_with_retries(headers, execution_id, retries=6): ctx.abort_operation(err) -def create_maintenance_headers(upgrade_props=True): - headers = {'X-BYPASS-MAINTENANCE': 'True'} - auth_props = get_auth_headers(upgrade_props) - headers.update(auth_props) - return headers - - -def get_auth_headers(upgrade_props): +def get_auth_headers(): headers = {} - if upgrade_props: - manager_config = ctx_factory.get('manager-config') - else: - manager_config = ctx_factory.load_rollback_props('manager-config') - - security_enabled = manager_config['security']['enabled'] - agent_config = manager_config['cloudify']['cloudify_agent'] + manager_config = _get_curr_manager_config() + security = manager_config['security'] + security_enabled = security['enabled'] if security_enabled: - username = agent_config.get('rest_username') - password = agent_config.get('rest_password') + username = security.get('rest_username') + password = security.get('rest_password') headers.update({'Authorization': 'Basic ' + base64.b64encode('{0}:{1}'.format( username, password))}) return headers +def _get_curr_manager_config(): + nginx_upgraded = os.path.isdir( + ctx_factory.get_rollback_properties_dir('nginx')) + if nginx_upgraded: + config = ctx_factory.get('manager-config') + else: + config = ctx_factory.load_rollback_props('manager-config') or \ + ctx_factory.get('manager-config') + return config + + def create_upgrade_snapshot(): if _get_upgrade_data().get('snapshot_id'): ctx.logger.debug('Upgrade snapshot already created.') return snapshot_id = _generate_upgrade_snapshot_id() - url = 'http://localhost/api/{0}/snapshots/{1}'.format(REST_VERSION, - snapshot_id) + url = 'localhost/api/{0}/snapshots/{1}'.format(REST_VERSION, snapshot_id) data = json.dumps({'include_metrics': 'true', 'include_credentials': 'true'}) - headers = create_maintenance_headers(upgrade_props=False) - req_headers = headers.copy() - req_headers.update({'Content-Type': 'application/json'}) + req_headers = {'Content-Type': 'application/json'} ctx.logger.info('Creating snapshot with ID {0}' .format(snapshot_id)) - res = http_request(url, data=data, method='PUT', headers=req_headers) + res = rest_request(url, data=data, method='PUT', headers=req_headers) if res.code != 201: err = 'Failed creating snapshot {0}. Message: {1}'\ - .format(snapshot_id, res.readlines()) + .format(snapshot_id, res.content) ctx.logger.error(err) ctx.abort_operation(err) - execution_id = json.loads(res.readlines()[0])['id'] - _wait_for_execution(execution_id, headers) + execution_id = json.loads(res.content)['id'] + _wait_for_execution(execution_id) ctx.logger.info('Snapshot with ID {0} created successfully' .format(snapshot_id)) ctx.logger.info('Setting snapshot info to upgrade metadata in {0}'. @@ -1340,37 +1405,35 @@ def create_upgrade_snapshot(): def restore_upgrade_snapshot(): snapshot_id = _get_upgrade_data()['snapshot_id'] - url = 'http://localhost/api/{0}/snapshots/{1}/restore'.format(REST_VERSION, - snapshot_id) + url = 'localhost/api/{0}/snapshots/{1}/restore'.format(REST_VERSION, + snapshot_id) data = json.dumps({'recreate_deployments_envs': 'false', 'force': 'true'}) - headers = create_maintenance_headers(upgrade_props=True) - req_headers = headers.copy() - req_headers.update({'Content-Type': 'application/json'}) + req_headers = {'Content-Type': 'application/json'} ctx.logger.info('Restoring snapshot with ID {0}'.format(snapshot_id)) - res = http_request(url, data=data, method='POST', headers=req_headers) + res = rest_request(url, data=data, method='POST', headers=req_headers) if res.code != 200: err = 'Failed restoring snapshot {0}. Message: {1}' \ - .format(snapshot_id, res.readlines()) + .format(snapshot_id, res.content) ctx.logger.error(err) ctx.abort_operation(err) - execution_id = json.loads(res.readlines()[0])['id'] - _wait_for_execution(execution_id, headers) + execution_id = json.loads(res.content)['id'] + _wait_for_execution(execution_id) ctx.logger.info('Snapshot with ID {0} restored successfully' .format(snapshot_id)) def _generate_upgrade_snapshot_id(): - url = 'http://localhost/api/{0}/version'.format(REST_VERSION) - auth_headers = get_auth_headers(upgrade_props=False) - res = http_request(url, method='GET', headers=auth_headers) + url = 'localhost/api/{0}/version'.format(REST_VERSION) + auth_headers = get_auth_headers() + res = rest_request(url, method='GET', headers=auth_headers) if res.code != 200: err = 'Failed extracting current manager version. Message: {0}' \ - .format(res.readlines()) + .format(res.content) ctx.logger.error(err) ctx.abort_operation(err) curr_time = strftime("%Y-%m-%d_%H:%M:%S", gmtime()) - version_data = json.loads(res.read()) + version_data = json.loads(res.content) snapshot_upgrade_name = 'upgrade_snapshot_{0}_build_{1}_{2}' \ .format(version_data['version'], version_data['build'], @@ -1397,11 +1460,14 @@ def _get_upgrade_data(): @retry((IOError, ValueError)) -def check_http_response(url, predicate=None, **request_kwargs): - req = urllib2.Request(url, **request_kwargs) +def check_http_response(url, service_name, predicate=None, + **request_kwargs): try: - response = urllib2.urlopen(req) - except urllib2.HTTPError as e: + if service_name in ('nginx', 'restservice'): + response = rest_request(url, **request_kwargs) + else: + response = http_request(url, **request_kwargs) + except (urllib2.HTTPError, httplib.HTTPException) as e: # HTTPError can also be used as a non-200 response. Pass this # through to the predicate function, so it can decide if a # non-200 response is fine or not. @@ -1414,7 +1480,7 @@ def check_http_response(url, predicate=None, **request_kwargs): def verify_service_http(service_name, url, *args, **kwargs): try: - return check_http_response(url, *args, **kwargs) + return check_http_response(url, service_name, *args, **kwargs) except (IOError, ValueError) as e: ctx.abort_operation('{0} error: {1}: {2}'.format(service_name, url, e)) @@ -1485,19 +1551,19 @@ def verify_immutable_properties(service_name, properties): def _is_version_greater_than_curr(new_version): - version_url = 'http://localhost/api/{0}/version'.format(REST_VERSION) - version_res = http_request(version_url, method='GET') + version_url = 'localhost/api/{0}/version'.format(REST_VERSION) + version_res = rest_request(version_url, method='GET') if version_res.code != 200: ctx.abort_operation('Failed retrieving manager version') - curr_version = json.loads(version_res.readlines()[0])['version'] + curr_version = json.loads(version_res.content)['version'] ctx.logger.info('Current manager version is {0}.'.format(curr_version)) return LooseVersion(new_version) > LooseVersion(curr_version) -# rollback resources will be removed only if the last upgrade passed -# successfully and the 'upgrade to' version is greater than the current version -# # This function MUST be invoked by the first node and before upgrade snapshot -# is created. +# rollback resources will be removed upon upgrade execution only if the last +# upgrade passed successfully and the 'upgrade to' version is greater than +# the current version. This function MUST be invoked by the first node and +# before upgrade snapshot is created. def clean_rollback_resources_if_necessary(): if not is_upgrade: return From 76ca7968d43d9522aab1ab8639959e2b0be6f52b Mon Sep 17 00:00:00 2001 From: adamlavie Date: Tue, 12 Jul 2016 13:02:47 +0300 Subject: [PATCH 3/5] CFY-5334 add SSL support --- components/manager/scripts/configure_manager.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/components/manager/scripts/configure_manager.py b/components/manager/scripts/configure_manager.py index 0822cea94..1ea74dbeb 100644 --- a/components/manager/scripts/configure_manager.py +++ b/components/manager/scripts/configure_manager.py @@ -54,14 +54,15 @@ def _configure_security_properties(): if security_enabled: # agent access-control settings - agents_rest_username = agent_config['rest_username'] - agents_rest_password = agent_config['rest_password'] - ctx.instance.runtime_properties['agents_rest_username'] = \ - agents_rest_username - ctx.instance.runtime_properties['agents_rest_password'] = \ - agents_rest_password - ctx.logger.info('agents_rest_username: {0}'. - format(agents_rest_username)) + agents_rest_username = agent_config.get('rest_username') + agents_rest_password = agent_config.get('rest_password') + if agents_rest_username and agents_rest_password: + ctx.instance.runtime_properties['agents_rest_username'] = \ + agents_rest_username + ctx.instance.runtime_properties['agents_rest_password'] = \ + agents_rest_password + ctx.logger.info('agents_rest_username: {0}'. + format(agents_rest_username)) if security_enabled and ssl_enabled: # manager SSL settings From fb5350ad24d2f33226b50253c11f10e60eefac08 Mon Sep 17 00:00:00 2001 From: adamlavie Date: Wed, 20 Jul 2016 17:21:54 +0300 Subject: [PATCH 4/5] CFY-5334 ssl support for manager upgrade --- components/elasticsearch/scripts/create.py | 2 +- .../manager/scripts/configure_manager.py | 4 + components/manager/scripts/create.py | 3 - components/manager/scripts/sanity/sanity.py | 8 +- components/nginx/scripts/preconfigure.py | 13 ++- components/nginx/scripts/start.py | 3 +- .../restservice/scripts/preconfigure.py | 2 + components/restservice/scripts/start.py | 2 +- components/riemann/scripts/preconfigure.py | 5 - components/utils.py | 98 +++++++++++-------- openstack-manager-blueprint.yaml | 2 +- simple-manager-blueprint.yaml | 2 +- types/manager-types.yaml | 6 +- 13 files changed, 87 insertions(+), 63 deletions(-) diff --git a/components/elasticsearch/scripts/create.py b/components/elasticsearch/scripts/create.py index 519e28b0e..066c003b9 100644 --- a/components/elasticsearch/scripts/create.py +++ b/components/elasticsearch/scripts/create.py @@ -314,7 +314,7 @@ def main(): es_endpoint_ip = ctx_properties['es_endpoint_ip'] es_endpoint_port = ctx_properties['es_endpoint_port'] - if utils.is_upgrade: + if utils.is_upgrade or utils.is_rollback: dump_upgrade_data() if not es_endpoint_ip: diff --git a/components/manager/scripts/configure_manager.py b/components/manager/scripts/configure_manager.py index 1ea74dbeb..4cf71f045 100644 --- a/components/manager/scripts/configure_manager.py +++ b/components/manager/scripts/configure_manager.py @@ -26,6 +26,10 @@ ) import utils # NOQA +# This MUST be invoked by the first node, before upgrade snapshot is created. +rest_host = ctx.instance.runtime_properties['internal_rest_host'] +utils.clean_rollback_resources_if_necessary(rest_host) + NODE_NAME = 'manager-config' diff --git a/components/manager/scripts/create.py b/components/manager/scripts/create.py index ba17e060f..798323d75 100644 --- a/components/manager/scripts/create.py +++ b/components/manager/scripts/create.py @@ -13,9 +13,6 @@ NODE_NAME = 'manager-resources' -# This MUST be invoked by the first node, before upgrade snapshot is created. -utils.clean_rollback_resources_if_necessary() - ctx_properties = utils.ctx_factory.create(NODE_NAME) diff --git a/components/manager/scripts/sanity/sanity.py b/components/manager/scripts/sanity/sanity.py index 5feb55f2f..9800eb9f3 100644 --- a/components/manager/scripts/sanity/sanity.py +++ b/components/manager/scripts/sanity/sanity.py @@ -83,7 +83,7 @@ def _deploy_app(): utils.wait_for_workflow, deployment_id=DEPLOYMENT_ID, workflow_id='create_deployment_environment', - url_prefix=_get_url_prefix(), + rest_host=manager_ip, timeout_msg='Timed out while waiting for ' 'deployment {0} to be created'.format(DEPLOYMENT_ID)) @@ -108,7 +108,7 @@ def _install_sanity_app(): interval=30, deployment_id=DEPLOYMENT_ID, workflow_id='install', - url_prefix=_get_url_prefix(), + rest_host=manager_ip, timeout_msg='Timed out while waiting for ' 'deployment {0} to install'.format(DEPLOYMENT_ID)) @@ -179,7 +179,7 @@ def _uninstall_sanity_app(): interval=30, deployment_id=DEPLOYMENT_ID, workflow_id='uninstall', - url_prefix=_get_url_prefix(), + rest_host=manager_ip, timeout_msg='Timed out while waiting for ' 'deployment {0} to uninstall.'.format(DEPLOYMENT_ID)) @@ -267,7 +267,7 @@ def perform_sanity(): if utils.is_upgrade or utils.is_rollback: # Restore the snapshot at the end of the workflow. - utils.restore_upgrade_snapshot() + utils.restore_upgrade_snapshot(manager_ip) if utils.is_upgrade: # To keep the upgrade workflow idempotent, this flag is used to figure diff --git a/components/nginx/scripts/preconfigure.py b/components/nginx/scripts/preconfigure.py index d09c60c57..265d24acb 100644 --- a/components/nginx/scripts/preconfigure.py +++ b/components/nginx/scripts/preconfigure.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import os from os.path import join, dirname from cloudify import ctx @@ -13,20 +14,24 @@ EXTERNAL_REST_CERT_PATH = '/root/cloudify/ssl/external_rest_host.crt' NGINX_SERVICE_NAME = 'nginx' -ctx_properties = {'service_name': NGINX_SERVICE_NAME} def preconfigure_nginx(): target_runtime_props = ctx.target.instance.runtime_properties + source_runtime_props = ctx.source.instance.runtime_properties + # this is used by nginx's default.conf to select the relevant configuration rest_protocol = target_runtime_props['rest_protocol'] - + rest_host = target_runtime_props['internal_rest_host'] # TODO: NEED TO IMPLEMENT THIS IN CTX UTILS - ctx.source.instance.runtime_properties['rest_protocol'] = rest_protocol + source_runtime_props['rest_protocol'] = rest_protocol + ctx.logger.info('setting rest host to {}'.format(rest_host)) + source_runtime_props['rest_host'] = rest_host if rest_protocol == 'https': + utils.deploy_rest_certificates( - internal_rest_host=target_runtime_props['internal_rest_host'], + internal_rest_host=rest_host, external_rest_host=target_runtime_props['external_rest_host']) # get rest public certificate for output later diff --git a/components/nginx/scripts/start.py b/components/nginx/scripts/start.py index 0b345f69e..fb195dc8a 100644 --- a/components/nginx/scripts/start.py +++ b/components/nginx/scripts/start.py @@ -25,6 +25,7 @@ def check_response(response): utils.start_service(NGINX_SERVICE_NAME, append_prefix=False) utils.systemd.verify_alive(NGINX_SERVICE_NAME, append_prefix=False) -nginx_url = '127.0.0.1/api/v2.1/blueprints' +rest_host = ctx.instance.runtime_properties['rest_host'] +nginx_url = '{0}/api/v2.1/blueprints'.format(rest_host) utils.verify_service_http(NGINX_SERVICE_NAME, nginx_url, check_response) diff --git a/components/restservice/scripts/preconfigure.py b/components/restservice/scripts/preconfigure.py index 5abd595ae..4a8e394f9 100644 --- a/components/restservice/scripts/preconfigure.py +++ b/components/restservice/scripts/preconfigure.py @@ -10,5 +10,7 @@ def preconfigure_restservice(): ctx.logger.info('security_config is: {0}'.format(security_config)) ctx.source.instance.runtime_properties['security_configuration'] = \ security_config + ctx.source.instance.runtime_properties['rest_host'] = \ + ctx.target.instance.runtime_properties['internal_rest_host'] preconfigure_restservice() diff --git a/components/restservice/scripts/start.py b/components/restservice/scripts/start.py index 1cfaf6c39..d12d6ca4a 100644 --- a/components/restservice/scripts/start.py +++ b/components/restservice/scripts/start.py @@ -49,6 +49,6 @@ def verify_restservice(url): utils.systemd.verify_alive(REST_SERVICE_NAME) -restservice_url = '127.0.0.1' +restservice_url = ctx.instance.runtime_properties['rest_host'] utils.verify_service_http(REST_SERVICE_NAME, restservice_url) verify_restservice(restservice_url) diff --git a/components/riemann/scripts/preconfigure.py b/components/riemann/scripts/preconfigure.py index a64fcbef6..b91bad21e 100644 --- a/components/riemann/scripts/preconfigure.py +++ b/components/riemann/scripts/preconfigure.py @@ -4,11 +4,6 @@ from cloudify import ctx -ctx.download_resource( - join('components', 'utils.py'), - join(dirname(__file__), 'utils.py')) -import utils # NOQA - ctx.source.instance.runtime_properties['rest_host'] = \ ctx.target.instance.runtime_properties['internal_rest_host'] diff --git a/components/utils.py b/components/utils.py index c07b87eb5..2d9d5613e 100644 --- a/components/utils.py +++ b/components/utils.py @@ -240,6 +240,16 @@ def deploy_ssl_cert_and_key(cert_filename, key_filename, cn): 'key "{1}" for CN "{2}"...'. format(cert_filename, key_filename, cn)) _generate_ssl_cert(cert_target_path, key_target_path, cn) + deploy_blueprint_resource('', + cert_target_path, + NGINX_SERVICE_NAME, + user_resource=True, + load_ctx=False) + deploy_blueprint_resource('', + key_target_path, + NGINX_SERVICE_NAME, + user_resource=True, + load_ctx=False) else: raise @@ -1045,7 +1055,7 @@ def create(self, source, destination, service_name, user_resource=False, @staticmethod def _is_download_required(local_resource_path, is_render): result = False - if not os.path.isfile(local_resource_path): + if not os.path.isfile(local_resource_path) and not is_rollback: result = True # rendered resources should be re-rendered if in upgrade. if is_render and is_upgrade: @@ -1059,15 +1069,13 @@ def _get_dest_by_resources_json(self, service_name, resource_name): def _download_user_resource(self, source, dest, resource_name, service_name, render=True, load_ctx=True): if is_upgrade: - install_props = self._get_rollback_resources_json(service_name) - existing_resource_path = install_props.get(resource_name, '') - if os.path.isfile(existing_resource_path): + rollback_props = self._get_rollback_resources_json(service_name) + existing_resource_path = rollback_props.get(resource_name, '') + if is_file(existing_resource_path): ctx.logger.info('Using existing resource for {0}' .format(resource_name)) # update the resource file we hold that might have changed - install_resource = self._get_local_file_path( - service_name, resource_name) - copy(existing_resource_path, install_resource) + copy(existing_resource_path, dest) else: ctx.logger.info('User resource {0} not found on {1}' .format(resource_name, dest)) @@ -1204,6 +1212,14 @@ def _get_rollback_resources_json(self, service_name): ctx_factory = CtxPropertyFactory() +def is_file(path): + try: + sudo('cat {0}'.format(path)) + return True + except subprocess.CalledProcessError as e: + return False + + def start_service(service_name, append_prefix=True, ignore_restart_fail=False): if is_upgrade or is_rollback: systemd.restart(service_name, @@ -1215,7 +1231,7 @@ def start_service(service_name, append_prefix=True, ignore_restart_fail=False): def rest_request(url, **request_kwargs): headers = request_kwargs.get('headers', {}) - auth_headers = get_auth_headers() + auth_headers = _get_auth_headers() headers.update(auth_headers) if is_upgrade or is_rollback: headers.update({'X-BYPASS-MAINTENANCE': 'True'}) @@ -1274,9 +1290,9 @@ def https_request(url, data=None, method='GET', headers=None, timeout=None, if parsed_uri.query: uri += '?{0}'.format(parsed_uri.query) ctx.logger.info('URL: {0}, URI: {1}, method {2}, headers {3} server.cert ' - '{4} server.key {5}'.format(url, uri, method, - headers, server_crt, - server_key)) + '{4} server.key {5}, host {6}'.format(url, uri, method, + headers, server_crt, + server_key, host)) con = httplib.HTTPSConnection(host=host, cert_file=server_crt, key_file=server_key, @@ -1298,10 +1314,10 @@ def _get_rest_key_and_crt(): return server_crt, server_key -def wait_for_workflow( - deployment_id, - workflow_id, - url_prefix='localhost/api/{0}'.format(REST_VERSION)): +def wait_for_workflow(rest_host, + deployment_id, + workflow_id): + url_prefix = '{0}/api/{1}'.format(rest_host, REST_VERSION) params = urllib.urlencode(dict(deployment_id=deployment_id)) endpoint = '{0}/executions'.format(url_prefix) url = endpoint + '?' + params @@ -1320,10 +1336,10 @@ def wait_for_workflow( return False -def _wait_for_execution(execution_id): +def _wait_for_execution(rest_host, execution_id): poll_interval = 2 while True: - res = _list_executions_with_retries(execution_id) + res = _list_executions_with_retries(rest_host, execution_id) content = json.loads(res.content) execution_item = content['items'][0] execution_status = execution_item['status'] @@ -1335,11 +1351,11 @@ def _wait_for_execution(execution_id): sleep(poll_interval) -def _list_executions_with_retries(execution_id, retries=6): +def _list_executions_with_retries(rest_host, execution_id, retries=6): count = 0 err = 'Failed listing existing executions.' - url = 'localhost/api/{0}/executions?_include_system_workflows=true&id={1}'\ - .format(REST_VERSION, execution_id) + url = '{0}/api/{1}/executions?_include_system_workflows=true&id={2}'\ + .format(rest_host, REST_VERSION, execution_id) while count != retries: res = rest_request(url, method='GET') if res.code != 200: @@ -1352,14 +1368,17 @@ def _list_executions_with_retries(execution_id, retries=6): ctx.abort_operation(err) -def get_auth_headers(): +def _get_auth_headers(): headers = {} manager_config = _get_curr_manager_config() security = manager_config['security'] security_enabled = security['enabled'] if security_enabled: - username = security.get('rest_username') - password = security.get('rest_password') + # property names changed between 3.4 ==> 3.4.1 + username = security.get('rest_username', + security.get('admin_username')) + password = security.get('rest_password', + security.get('admin_password')) headers.update({'Authorization': 'Basic ' + base64.b64encode('{0}:{1}'.format( username, password))}) @@ -1377,12 +1396,13 @@ def _get_curr_manager_config(): return config -def create_upgrade_snapshot(): +def create_upgrade_snapshot(rest_host): if _get_upgrade_data().get('snapshot_id'): ctx.logger.debug('Upgrade snapshot already created.') return - snapshot_id = _generate_upgrade_snapshot_id() - url = 'localhost/api/{0}/snapshots/{1}'.format(REST_VERSION, snapshot_id) + snapshot_id = _generate_upgrade_snapshot_id(rest_host) + url = '{0}/api/{1}/snapshots/{2}'\ + .format(rest_host, REST_VERSION, snapshot_id) data = json.dumps({'include_metrics': 'true', 'include_credentials': 'true'}) req_headers = {'Content-Type': 'application/json'} @@ -1395,7 +1415,7 @@ def create_upgrade_snapshot(): ctx.logger.error(err) ctx.abort_operation(err) execution_id = json.loads(res.content)['id'] - _wait_for_execution(execution_id) + _wait_for_execution(rest_host, execution_id) ctx.logger.info('Snapshot with ID {0} created successfully' .format(snapshot_id)) ctx.logger.info('Setting snapshot info to upgrade metadata in {0}'. @@ -1403,10 +1423,11 @@ def create_upgrade_snapshot(): _set_upgrade_data(snapshot_id=snapshot_id) -def restore_upgrade_snapshot(): +def restore_upgrade_snapshot(rest_host): snapshot_id = _get_upgrade_data()['snapshot_id'] - url = 'localhost/api/{0}/snapshots/{1}/restore'.format(REST_VERSION, - snapshot_id) + url = '{0}/api/{1}/snapshots/{2}/restore'.format(rest_host, + REST_VERSION, + snapshot_id) data = json.dumps({'recreate_deployments_envs': 'false', 'force': 'true'}) req_headers = {'Content-Type': 'application/json'} @@ -1418,15 +1439,14 @@ def restore_upgrade_snapshot(): ctx.logger.error(err) ctx.abort_operation(err) execution_id = json.loads(res.content)['id'] - _wait_for_execution(execution_id) + _wait_for_execution(rest_host, execution_id) ctx.logger.info('Snapshot with ID {0} restored successfully' .format(snapshot_id)) -def _generate_upgrade_snapshot_id(): - url = 'localhost/api/{0}/version'.format(REST_VERSION) - auth_headers = get_auth_headers() - res = rest_request(url, method='GET', headers=auth_headers) +def _generate_upgrade_snapshot_id(rest_host): + url = '{0}/api/{1}/version'.format(rest_host, REST_VERSION) + res = rest_request(url, method='GET') if res.code != 200: err = 'Failed extracting current manager version. Message: {0}' \ .format(res.content) @@ -1550,8 +1570,8 @@ def verify_immutable_properties(service_name, properties): service_name, ','.join(descr_parts))) -def _is_version_greater_than_curr(new_version): - version_url = 'localhost/api/{0}/version'.format(REST_VERSION) +def _is_version_greater_than_curr(rest_host, new_version): + version_url = '{0}/api/{1}/version'.format(rest_host, REST_VERSION) version_res = rest_request(version_url, method='GET') if version_res.code != 200: ctx.abort_operation('Failed retrieving manager version') @@ -1564,11 +1584,11 @@ def _is_version_greater_than_curr(new_version): # upgrade passed successfully and the 'upgrade to' version is greater than # the current version. This function MUST be invoked by the first node and # before upgrade snapshot is created. -def clean_rollback_resources_if_necessary(): +def clean_rollback_resources_if_necessary(rest_host): if not is_upgrade: return new_version = ctx.node.properties['manager_version'] - is_upgrade_version = _is_version_greater_than_curr(new_version) + is_upgrade_version = _is_version_greater_than_curr(rest_host, new_version) # The 'upgrade_success' flag will only be set if the previous upgrade # execution ended successfully latest_workflow_result = _get_upgrade_data().get('upgrade_success') diff --git a/openstack-manager-blueprint.yaml b/openstack-manager-blueprint.yaml index 33a044605..149db23b1 100644 --- a/openstack-manager-blueprint.yaml +++ b/openstack-manager-blueprint.yaml @@ -266,7 +266,7 @@ node_templates: target: manager_host target_interfaces: cloudify.interfaces.relationship_lifecycle: - postconfigure: + preconfigure: implementation: components/manager/scripts/set_manager_ips.py inputs: public_ip: { get_attribute: [manager_server_ip, floating_ip_address] } diff --git a/simple-manager-blueprint.yaml b/simple-manager-blueprint.yaml index 57b4a5791..6dd501297 100644 --- a/simple-manager-blueprint.yaml +++ b/simple-manager-blueprint.yaml @@ -171,7 +171,7 @@ node_templates: target: manager_host target_interfaces: cloudify.interfaces.relationship_lifecycle: - postconfigure: + preconfigure: implementation: components/manager/scripts/set_manager_ips.py inputs: public_ip: { get_property: [manager_host, public_ip] } diff --git a/types/manager-types.yaml b/types/manager-types.yaml index f3d8b1ec1..29f0bb7d0 100644 --- a/types/manager-types.yaml +++ b/types/manager-types.yaml @@ -25,8 +25,6 @@ node_types: default: { get_input: minimum_required_available_disk_space_in_gb } allowed_heap_size_gap_in_mb: default: { get_input: allowed_heap_size_gap_in_mb } - manager_version: - default: 3.4.0 interfaces: cloudify.interfaces.lifecycle: create: @@ -67,6 +65,8 @@ node_types: default: /root/.ssh/agent_key.pem public_ip: default: '' + manager_version: + default: 3.4.1 cloudify: description: > Configuration for Cloudify Manager @@ -990,7 +990,7 @@ node_types: manager_user: default: { get_input: ssh_user } manager_ip: - default: { get_attribute: [manager_host, public_ip] } + default: { get_attribute: [manager_configuration, internal_rest_host] } hide_output: *hide_output fabric_env: *simple_fabric_env From 11bc32eb6bf5e058c0321000371aad5b5ee65fdd Mon Sep 17 00:00:00 2001 From: adamlavie Date: Wed, 20 Jul 2016 17:24:14 +0300 Subject: [PATCH 5/5] CFY-5334 upgrade with ssl --- components/nginx/scripts/preconfigure.py | 1 - components/riemann/scripts/preconfigure.py | 2 -- components/utils.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/components/nginx/scripts/preconfigure.py b/components/nginx/scripts/preconfigure.py index 265d24acb..cf82d639f 100644 --- a/components/nginx/scripts/preconfigure.py +++ b/components/nginx/scripts/preconfigure.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import os from os.path import join, dirname from cloudify import ctx diff --git a/components/riemann/scripts/preconfigure.py b/components/riemann/scripts/preconfigure.py index b91bad21e..4ed485486 100644 --- a/components/riemann/scripts/preconfigure.py +++ b/components/riemann/scripts/preconfigure.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from os.path import join, dirname - from cloudify import ctx diff --git a/components/utils.py b/components/utils.py index 2d9d5613e..7706e8f43 100644 --- a/components/utils.py +++ b/components/utils.py @@ -1216,7 +1216,7 @@ def is_file(path): try: sudo('cat {0}'.format(path)) return True - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: return False