diff --git a/.github/workflows/jira.yaml b/.github/workflows/jira.yaml new file mode 100644 index 0000000..af57a5b --- /dev/null +++ b/.github/workflows/jira.yaml @@ -0,0 +1,26 @@ +name: Sync GitHub issues to Jira +on: [issues, issue_comment] + +jobs: + sync-issues: + name: Sync issues to Jira + runs-on: ubuntu-latest + steps: + - uses: ikethecoder/sync-issues-github-jira@dev + with: + webhook-url: ${{ secrets.JIRA_WEBHOOK_URL }} + + cleanup-runs: + name: Delete workflow runs + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Delete workflow runs + uses: Mattraks/delete-workflow-runs@v2 + with: + token: ${{ github.token }} + repository: ${{ github.repository }} + retain_days: 1 + keep_minimum_runs: 5 + delete_workflow_pattern: jira.yaml diff --git a/microservices/gatewayApi/Dockerfile b/microservices/gatewayApi/Dockerfile index 4418a98..ab2be49 100644 --- a/microservices/gatewayApi/Dockerfile +++ b/microservices/gatewayApi/Dockerfile @@ -21,10 +21,16 @@ RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl - #COPY --from=build /deck/deck /usr/local/bin +# gwa api v1/v2 RUN curl -sL https://github.com/kong/deck/releases/download/v1.5.0/deck_1.5.0_linux_amd64.tar.gz -o deck.tar.gz && \ tar -xf deck.tar.gz -C /tmp && \ cp /tmp/deck /usr/local/bin/ +# gwa api v3 +RUN curl -sL https://github.com/Kong/deck/releases/download/v1.27.1/deck_1.27.1_linux_amd64.tar.gz -o deck.tar.gz && \ + tar -xf deck.tar.gz -C /tmp && \ + cp /tmp/deck /usr/local/bin/deck127 + COPY requirements.txt requirements.txt RUN pip install -r requirements.txt diff --git a/microservices/gatewayApi/app.py b/microservices/gatewayApi/app.py index a79f293..57bb195 100644 --- a/microservices/gatewayApi/app.py +++ b/microservices/gatewayApi/app.py @@ -15,6 +15,7 @@ import v1.v1 as v1 import v2.v2 as v2 +import v3.v3 as v3 def create_app(test_config=None): @@ -43,6 +44,7 @@ def create_app(test_config=None): ##Routes## v1.Register(app) v2.Register(app) + v3.Register(app) Compress(app) @app.before_request diff --git a/microservices/gatewayApi/clients/portal.py b/microservices/gatewayApi/clients/portal.py index 438d81b..e9ee7e1 100644 --- a/microservices/gatewayApi/clients/portal.py +++ b/microservices/gatewayApi/clients/portal.py @@ -40,9 +40,9 @@ def record_gateway_event(event_id, action, result, namespace, message="", blob=" entity = 'gateway configuration' actor = "Unknown Actor" - if "clientId" in g.principal: + if 'principal' in g and "clientId" in g.principal: actor = g.principal["clientId"] - if "name" in g.principal: + if 'principal' in g and "name" in g.principal: actor = g.principal["name"] payload = { diff --git a/microservices/gatewayApi/requirements.txt b/microservices/gatewayApi/requirements.txt index c6814ad..f90ed21 100644 --- a/microservices/gatewayApi/requirements.txt +++ b/microservices/gatewayApi/requirements.txt @@ -1,3 +1,4 @@ +werkzeug==2.2.2 ply==3.10 cryptography==38.0.4 authlib==0.15.3 @@ -21,4 +22,4 @@ pytest-cov==4.0.0 pytest-mock==3.10.0 pycodestyle==2.10.0 pylint==1.7.2 -flask-jwt-simple==0.0.3 \ No newline at end of file +flask-jwt-simple==0.0.3 diff --git a/microservices/gatewayApi/swagger.py b/microservices/gatewayApi/swagger.py index 9168502..3dcd94b 100644 --- a/microservices/gatewayApi/swagger.py +++ b/microservices/gatewayApi/swagger.py @@ -37,7 +37,6 @@ def openapi_spec(ver:str): return Response(open("%s/%s.yaml" % (conf.data['workingFolder'], ver)).read(), mimetype='application/x-yaml') #app.add_url_rule("/%s/docs/openapi.yaml" % version, openapi_spec) - app.register_blueprint(swaggerui_blueprint) for version in versions: ## Template the spec and write it to a temporary location @@ -52,10 +51,12 @@ def openapi_spec(ver:str): authorization_url = discovery["authorization_endpoint"], accesstoken_url = discovery["token_endpoint"] )) - log.info("Configured /%s/docs" % version) + app.register_blueprint(swaggerui_blueprint) + log.info("Swagger UI registered") + except: traceback.print_exc(file=sys.stdout) log.error("Failed to do OIDC Discovery for %s, sleeping 5 seconds and trying again." % version) diff --git a/microservices/gatewayApi/v3/__init__.py b/microservices/gatewayApi/v3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/microservices/gatewayApi/v3/auth/__init__.py b/microservices/gatewayApi/v3/auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/microservices/gatewayApi/v3/auth/auth.py b/microservices/gatewayApi/v3/auth/auth.py new file mode 100644 index 0000000..f1f4498 --- /dev/null +++ b/microservices/gatewayApi/v3/auth/auth.py @@ -0,0 +1,3 @@ +from auth.auth import admin_jwt +from auth.authz import enforce_authorization, enforce_role_authorization, users_group_root, admins_group_root, ns_claim +from auth.uma import uma_enforce \ No newline at end of file diff --git a/microservices/gatewayApi/v3/routes/__init__.py b/microservices/gatewayApi/v3/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/microservices/gatewayApi/v3/routes/gateway.py b/microservices/gatewayApi/v3/routes/gateway.py new file mode 100644 index 0000000..4eadd8d --- /dev/null +++ b/microservices/gatewayApi/v3/routes/gateway.py @@ -0,0 +1,591 @@ +import os +import shutil +import sys +import http +import traceback +from urllib.parse import urlparse +from subprocess import Popen, PIPE, STDOUT +import uuid +import logging +import json +import requests +import yaml +from werkzeug.exceptions import HTTPException, NotFound +from flask import Blueprint, config, jsonify, request, Response, make_response, abort, g, current_app as app +from io import TextIOWrapper +from clients.ocp_routes import get_host_list + +from v3.auth.auth import admin_jwt, uma_enforce + +from v2.services.namespaces import NamespaceService + +from clients.portal import record_gateway_event +from clients.kong import get_routes, register_kong_certs +from clients.ocp_networksecuritypolicy import get_ocp_service_namespaces, check_nsp, apply_nsp, delete_nsp +from clients.ocp_routes import prepare_apply_routes, prepare_delete_routes, apply_routes, delete_routes +from clients.ocp_gateway_secret import prep_submitted_config, prep_and_apply_secret, write_submitted_config + +from utils.validators import host_valid +from utils.transforms import plugins_transformations +from utils.masking import mask + +gw = Blueprint('gwa_v3', 'gateway') +local_environment = os.environ.get("LOCAL_ENVIRONMENT", default=False) + + +def abort_early(event_id, action, namespace, response): + record_gateway_event(event_id, action, 'failed', namespace, json.dumps(response.get_json())) + abort(make_response(response, 400)) + + + +@gw.route('', + methods=['PUT'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'GatewayConfig.Publish') +def write_config(namespace: str) -> object: + """ + (Over)write + :return: JSON of success message or error message + """ + + event_id = str(uuid.uuid4()) + + log = app.logger + + outFolder = namespace + + ns_svc = NamespaceService() + ns_attributes = ns_svc.get_namespace_attributes(namespace) + + dp = get_data_plane(ns_attributes) + + # Build a list of existing hosts that are outside this namespace + # They become reserved and any conflict will return an error + reserved_hosts = [] + all_routes = get_routes() + tag_match = "ns.%s" % namespace + for route in all_routes: + if tag_match not in route['tags'] and 'hosts' in route: + for host in route['hosts']: + reserved_hosts.append(host) + reserved_hosts = list(set(reserved_hosts)) + + + dfile = None + select_tag_qualifier = None + + if 'configFile' in request.files and not request.files['configFile'].filename == '': + log.debug("[%s] %s", namespace, request.files['configFile']) + dfile = request.files['configFile'] + dry_run = request.values['dryRun'] + if "qualifier" in request.values: + select_tag_qualifier = request.values['qualifier'] + elif request.content_type.startswith("application/json") and not request.json['configFile'] in [None, '']: + dfile = request.json['configFile'] + dry_run = request.json['dryRun'] + if "qualifier" in request.json: + select_tag_qualifier = request.json['qualifier'] + else: + log.error("Missing input") + log.error("%s", request.get_data()) + log.error(request.form) + log.error(request.content_type) + log.error(request.headers) + abort_early(event_id, 'publish', namespace, jsonify(error="Missing input")) + + cmd = "sync" + if dry_run == 'true' or dry_run is True: + cmd = "diff" + + if cmd == 'sync': + record_gateway_event(event_id, 'publish', 'received', namespace) + + tempFolder = "%s/%s/%s" % ('/tmp', uuid.uuid4(), outFolder) + os.makedirs(tempFolder, exist_ok=False) + + # dfile.save("%s/%s" % (tempFolder, 'config.yaml')) + + # log.debug("Saved to %s" % tempFolder) + yaml_documents = load_yaml_files(dfile) + + if len(yaml_documents) == 0: + log.error("%s - %s" % (namespace, "Empty Configuration Passed")) + abort_early(event_id, 'publish', namespace, jsonify(error="Empty Configuration Passed")) + + selectTag = "ns.%s" % namespace + ns_qualifier = None + if select_tag_qualifier is not None and select_tag_qualifier != "" and "." not in select_tag_qualifier: + ns_qualifier = "%s.%s" % (selectTag, select_tag_qualifier) + + orig_config = prep_submitted_config(clone_yaml_files(yaml_documents)) + + update_routes_flag = False + + for index, gw_config in enumerate(yaml_documents): + log.info("[%s] Parsing file %s" % (namespace, index)) + + if gw_config is None: + continue + + ####################### + # Enrichments + ####################### + + # Transformation route hosts if in non-prod environment (HOST_TRANSFORM_ENABLED) + host_transformation(namespace, dp, gw_config) + + # If there is a tag with a pipeline qualifier (i.e./ ns..dev) + # then add to tags automatically the tag: ns. + object_count = tags_transformation(namespace, gw_config) + + # + # Enrich the rate-limiting plugin with the appropriate Redis details + plugins_transformations(namespace, gw_config) + + with open("%s/%s" % (tempFolder, 'config-%02d.yaml' % index), 'w') as file: + yaml.dump(gw_config, file) + + ####################### + # Validations + ####################### + + # Validate that the every object is tagged with the namespace + try: + validate_base_entities(gw_config, ns_attributes) + validate_tags(gw_config, selectTag) + except Exception as ex: + traceback.print_exc() + log.error("%s - %s" % (namespace, " Tag Validation Errors: %s" % ex)) + abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex)) + + # Validate that hosts are valid + try: + validate_hosts(gw_config, reserved_hosts, ns_attributes) + except Exception as ex: + traceback.print_exc() + log.error("%s - %s" % (namespace, " Host Validation Errors: %s" % ex)) + abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex)) + + # Validate upstream URLs are valid + try: + protected_kube_namespaces = json.loads(app.config['protectedKubeNamespaces']) + validate_upstream(gw_config, ns_attributes, protected_kube_namespaces) + except Exception as ex: + traceback.print_exc() + log.error("%s - %s" % (namespace, " Upstream Validation Errors: %s" % ex)) + abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % ex)) + + # Validation #3 + # Validate that certain plugins are configured (such as the gwa_gov_endpoint) at the right level + + # Validate based on DNS 952 + + nsq = traverse_get_ns_qualifier(gw_config, selectTag) + if nsq is not None: + if ns_qualifier is not None and nsq != ns_qualifier: + abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % + ("Conflicting ns qualifiers (%s != %s)" % (ns_qualifier, nsq)))) + ns_qualifier = nsq + log.info("[%s] CHANGING ns_qualifier %s" % (namespace, ns_qualifier)) + elif ns_qualifier is not None and object_count > 0: + abort_early(event_id, 'publish', namespace, jsonify(error="Validation Errors:\n%s" % + ("Specified qualifier (%s) does not match tags in configuration (%s)" % (ns_qualifier, selectTag)))) + + if update_routes_check(gw_config): + update_routes_flag = True + + if ns_qualifier is not None: + selectTag = ns_qualifier + + # Call the 'deck' command + deck_cli = "deck127" + log.info("[%s] %s action using %s" % (namespace, cmd, selectTag)) + + args = [ + deck_cli, "validate", "--config", "/tmp/deck.yaml", "--state", tempFolder + ] + log.debug("[%s] Running %s" % (namespace, args)) + deck_validate = Popen(args, stdout=PIPE, stderr=STDOUT) + out, err = deck_validate.communicate() + + if deck_validate.returncode != 0: + log.warn("[%s] - %s" % (namespace, out.decode('utf-8'))) + abort_early(event_id, 'validate', namespace, jsonify( + error="Validation Failed.", results=mask(out.decode('utf-8')))) + + args = [ + deck_cli, cmd, "--config", "/tmp/deck.yaml", "--skip-consumers", "--select-tag", selectTag, "--state", tempFolder + ] + log.debug("[%s] Running %s" % (namespace, args)) + deck_run = Popen(args, stdout=PIPE, stderr=STDOUT) + out, err = deck_run.communicate() + if deck_run.returncode != 0: + cleanup(tempFolder) + log.warn("[%s] - %s" % (namespace, out.decode('utf-8'))) + abort_early(event_id, 'publish', namespace, jsonify(error="Sync Failed.", results=mask(out.decode('utf-8')))) + # skip creation of routes in local development environment + elif cmd == "sync" and not local_environment: + try: + if update_routes_flag: + host_list = get_host_list(tempFolder) + session = requests.Session() + session.headers.update({"Content-Type": "application/json"}) + route_payload = { + "hosts": host_list, + "select_tag": selectTag, + "ns_attributes": ns_attributes.getAttrs() + } + rqst_url = app.config['data_planes'][dp]["kube-api"] + log.debug("[%s] - Initiating request to kube API" % (dp)) + res = session.put(rqst_url + "/namespaces/%s/routes" % namespace, json=route_payload, auth=( + app.config['kubeApiCreds']['kubeApiUser'], app.config['kubeApiCreds']['kubeApiPass'])) + log.debug("[%s] - The kube API responded with %s" % (dp, res.status_code)) + if res.status_code != 201: + log.debug("[%s] - The kube API could not process the request" % (dp)) + raise Exception("[%s] - Failed to apply routes: %s" % (dp, str(res.text))) + session.close() + + if has_namespace_local_host_permission(ns_attributes): + session = requests.Session() + session.headers.update({"Content-Type": "application/json"}) + rqst_url = app.config['data_planes'][dp]["kube-api"] + log.debug("[%s] - Initiating request to kube API for Certs" % (dp)) + res = session.get(rqst_url + "/namespaces/%s/local_tls" % namespace, auth=( + app.config['kubeApiCreds']['kubeApiUser'], app.config['kubeApiCreds']['kubeApiPass'])) + log.debug("[%s] - The kube API responded with %s" % (dp, res.status_code)) + if res.status_code != 200: + log.debug("[%s] - The kube API could not process the request" % (dp)) + raise Exception("[%s] - Failed to get certs: %s" % (dp, str(res.text))) + cert_data = res.json() + session.close() + + register_kong_certs(namespace, cert_data) + + except HTTPException as ex: + traceback.print_exc() + log.error("[%s] Error updating custom routes. %s" % (namespace, ex)) + abort_early(event_id, 'publish', namespace, jsonify(error="Partially failed.")) + except: + traceback.print_exc() + log.error("[%s] Error updating custom routes. %s" % (namespace, sys.exc_info()[0])) + abort_early(event_id, 'publish', namespace, jsonify(error="Partially failed.")) + + cleanup(tempFolder) + + log.debug("[%s] The exit code was: %d" % (namespace, deck_run.returncode)) + + message = "Sync successful." + if cmd == 'diff': + message = "Dry-run. No changes applied." + + if cmd == 'sync': + record_gateway_event(event_id, 'published', 'completed', namespace, blob=orig_config) + return make_response(jsonify(message=message, results=mask(out.decode('utf-8')))) + + +def cleanup(dir_path): + log = app.logger + try: + shutil.rmtree(dir_path) + log.debug("Deleted folder %s" % dir_path) + except OSError as e: + log.error("Error: %s : %s" % (dir_path, e.strerror)) + +def validate_base_entities(yaml, ns_attributes): + traversables = ['_format_version', '_plugin_configs', 'services', 'upstreams', 'certificates', 'caCertificates'] + + allow_protected_ns = ns_attributes.get('perm-protected-ns', ['deny'])[0] == 'allow' + if allow_protected_ns: + traversables.append('plugins') + + for k in yaml: + if k not in traversables: + raise Exception("Invalid base entity %s" % k) + +def validate_tags(yaml, required_tag): + # throw an exception if there are invalid tags + errors = [] + qualifiers = [] + + if traverse_has_ns_qualifier(yaml, required_tag) and traverse_has_ns_tag_only(yaml, required_tag): + errors.append( + "Tags for the namespace can not have a mix of 'ns.' and 'ns..'. Rejecting request.") + + traverse("", errors, yaml, required_tag, qualifiers) + if len(qualifiers) > 1: + errors.append("Too many different qualified namespaces (%s). Rejecting request." % qualifiers) + + if len(errors) != 0: + raise Exception('\n'.join(errors)) + +def traverse(source, errors, yaml, required_tag, qualifiers): + traversables = ['services', 'routes', 'plugins', 'upstreams', 'consumers', 'certificates', 'caCertificates'] + for k in yaml: + if k in traversables: + for index, item in enumerate(yaml[k]): + if 'tags' in item: + if required_tag not in item['tags']: + errors.append("%s.%s.%s missing required tag %s" % (source, k, item['name'], required_tag)) + for tag in item['tags']: + # if the required_tag is "abc" and the tag starts with "ns." + # then ns.abc and ns.abc.dev are valid, but anything else is an error + if tag.startswith("ns.") and tag != required_tag and not tag.startswith("%s." % required_tag): + errors.append("%s.%s.%s invalid ns tag %s" % (source, k, item['name'], tag)) + if tag.startswith("%s." % required_tag) and tag not in qualifiers: + qualifiers.append(tag) + else: + errors.append("%s.%s.%s no tags found" % (source, k, item['name'])) + nm = "[%d]" % index + if 'name' in item: + nm = item['name'] + traverse("%s.%s.%s" % (source, k, nm), errors, item, required_tag, qualifiers) + + +def host_transformation(namespace, data_plane, yaml): + log = app.logger + + transforms = 0 + if 'services' in yaml: + for service in yaml['services']: + if 'routes' in service: + for route in service['routes']: + if 'hosts' in route: + new_hosts = [] + for host in route['hosts']: + if is_host_local(host): + new_hosts.append(transform_local_host(data_plane, host)) + elif is_host_transform_enabled(): + new_hosts.append(transform_host(host)) + transforms = transforms + 1 + else: + new_hosts.append(host) + route['hosts'] = new_hosts + log.debug("[%s] Host transformations %d" % (namespace, transforms)) + +def is_host_local (host): + return host.endswith(".cluster.local") + +def has_namespace_local_host_permission (ns_attributes): + for domain in ns_attributes.get('perm-domains', ['.api.gov.bc.ca']): + if is_host_local(domain): + return True + return False + +# Validate transformed host: ..svc.cluster.local +def validate_local_host(host): + if is_host_local(host): + if len(host.split('.')) != 5: + return False + return True + +def transform_local_host(data_plane, host): + suffix_len = len(".cluster.local") + kube_ns = app.config['data_planes'][data_plane]["kube-ns"] + name_part = host[:-suffix_len] + return "gw-%s.%s.svc.cluster.local" % (name_part, kube_ns) + +def transform_host(host): + if is_host_local(host): + return host + elif is_host_transform_enabled(): + conf = app.config['hostTransformation'] + return "%s%s" % (host.replace('.', '-'), conf['baseUrl']) + else: + return host + +def validate_upstream(yaml, ns_attributes, protected_kube_namespaces): + errors = [] + + allow_protected_ns = ns_attributes.get('perm-protected-ns', ['deny'])[0] == 'allow' + + # A host must not contain a list of protected + if 'services' in yaml: + for service in yaml['services']: + if 'url' in service: + try: + u = urlparse(service["url"]) + if u.hostname is None: + errors.append("service upstream has invalid url specified (e1)") + else: + validate_upstream_host(u.hostname, errors, allow_protected_ns, protected_kube_namespaces) + except Exception as e: + errors.append("service upstream has invalid url specified (e2)") + + if 'host' in service: + host = service["host"] + validate_upstream_host(host, errors, allow_protected_ns, protected_kube_namespaces) + + if len(errors) != 0: + raise Exception('\n'.join(errors)) + + +def validate_upstream_host(_host, errors, allow_protected_ns, protected_kube_namespaces): + host = _host.lower() + + restricted = ['localhost', '127.0.0.1', '0.0.0.0'] + + if host in restricted: + errors.append("service upstream is invalid (e1)") + if host.endswith('svc'): + partials = host.split('.') + # get the namespace, and make sure it is not in the protected_kube_namespaces list + if len(partials) != 3: + errors.append("service upstream is invalid (e2)") + elif partials[1] in protected_kube_namespaces and allow_protected_ns is False: + errors.append("service upstream is invalid (e3)") + if host.endswith('svc.cluster.local'): + partials = host.split('.') + # get the namespace, and make sure it is not in the protected_kube_namespaces list + if len(partials) != 5: + errors.append("service upstream is invalid (e4)") + elif partials[1] in protected_kube_namespaces and allow_protected_ns is False: + errors.append("service upstream is invalid (e5)") + +def update_routes_check(yaml): + if 'services' in yaml or 'upstreams' not in yaml: + return True + else: + return False + +def validate_hosts(yaml, reserved_hosts, ns_attributes): + errors = [] + + allowed_domains = [] + for domain in ns_attributes.get('perm-domains', ['.api.gov.bc.ca']): + allowed_domains.append("%s" % domain) + + # A host must not exist outside of namespace (reserved_hosts) + if 'services' in yaml: + for service in yaml['services']: + if 'routes' in service: + for route in service['routes']: + if 'hosts' in route: + for host in route['hosts']: + if host in reserved_hosts: + errors.append("service.%s.route.%s The host is already used in another namespace '%s'" % ( + service['name'], route['name'], host)) + if host_valid(host) is False: + errors.append("Host not passing DNS-952 validation '%s'" % host) + if validate_local_host(host) is False: + errors.append("Host failed validation for data plane '%s'" % host) + if host_ends_with_one_of_list(host, allowed_domains) is False: + errors.append("Host invalid: %s %s. Route hosts must end with one of [%s] for this namespace." % ( + route['name'], host, ','.join(allowed_domains))) + else: + errors.append("service.%s.route.%s A host must be specified for routes." % + (service['name'], route['name'])) + + if len(errors) != 0: + raise Exception('\n'.join(errors)) + + +def host_ends_with_one_of_list(a_str, a_list): + for item in a_list: + if a_str.endswith(transform_host(item)): + return True + return False + + +def tags_transformation(namespace, yaml): + return traverse_tags_transform(yaml, namespace, "ns.%s" % namespace) + + +def traverse_tags_transform(yaml, namespace, required_tag): + object_count = 0 + log = app.logger + traversables = ['services', 'routes', 'plugins', 'upstreams', 'consumers', 'certificates', 'caCertificates'] + for k in yaml: + if k in traversables: + for item in yaml[k]: + if 'tags' in item: + new_tags = [] + for tag in item['tags']: + new_tags.append(tag) + # add the base required tag automatically if there is already a qualifying namespace + if tag.startswith("ns.") and tag.startswith("%s." % required_tag) and required_tag not in item['tags']: + log.debug("[%s] Adding base tag %s to %s" % (namespace, required_tag, k)) + new_tags.append(required_tag) + item['tags'] = new_tags + object_count = object_count + 1 + object_count = object_count + traverse_tags_transform(item, namespace, required_tag) + return object_count + +def traverse_has_ns_qualifier(yaml, required_tag): + log = app.logger + traversables = ['services', 'routes', 'plugins', 'upstreams', 'consumers', 'certificates', 'caCertificates'] + for k in yaml: + if k in traversables: + for item in yaml[k]: + if 'tags' in item: + for tag in item['tags']: + if tag.startswith("%s." % required_tag): + return True + if traverse_has_ns_qualifier(item, required_tag) == True: + return True + return False + + +def traverse_has_ns_tag_only(yaml, required_tag): + log = app.logger + traversables = ['services', 'routes', 'plugins', 'upstreams', 'consumers', 'certificates', 'caCertificates'] + for k in yaml: + if k in traversables: + for item in yaml[k]: + if 'tags' in item: + if required_tag in item['tags'] and has_ns_qualifier(item['tags'], required_tag) is False: + return True + if traverse_has_ns_tag_only(item, required_tag) == True: + return True + return False + + +def has_ns_qualifier(tags, required_tag): + for tag in tags: + if tag.startswith("%s." % required_tag): + return True + return False + + +def traverse_get_ns_qualifier(yaml, required_tag): + log = app.logger + traversables = ['services', 'routes', 'plugins', 'upstreams', 'consumers', 'certificates', 'caCertificates'] + for k in yaml: + if k in traversables: + for item in yaml[k]: + if 'tags' in item: + for tag in item['tags']: + if tag.startswith("%s." % required_tag): + return tag + qualifier = traverse_get_ns_qualifier(item, required_tag) + if qualifier is not None: + return qualifier + return None + + +def get_data_plane(ns_attributes): + default_data_plane = app.config['defaultDataPlane'] + return ns_attributes.get('perm-data-plane', [default_data_plane])[0] + + +def is_host_transform_enabled(): + conf = app.config['hostTransformation'] + return conf['enabled'] is True + + +def should_we_apply_nsp_policies(): + conf = app.config['applyAporetoNSP'] + return conf is True + +def load_yaml_files (dfile): + yaml_documents_iter = yaml.load_all(dfile, Loader=yaml.FullLoader) + yaml_documents = [] + for doc in yaml_documents_iter: + yaml_documents.append(doc) + return yaml_documents + +def clone_yaml_files (yaml_documents): + cloned_yaml = [] + for doc in yaml_documents: + cloned_yaml.append(yaml.load(yaml.dump(doc), Loader=yaml.FullLoader)) + return cloned_yaml \ No newline at end of file diff --git a/microservices/gatewayApi/v3/routes/gw_status.py b/microservices/gatewayApi/v3/routes/gw_status.py new file mode 100644 index 0000000..cb92e91 --- /dev/null +++ b/microservices/gatewayApi/v3/routes/gw_status.py @@ -0,0 +1,160 @@ +import requests +import sys +import traceback +import urllib3 +import certifi +import socket +from urllib.parse import urlparse +from flask import Blueprint, jsonify, request, Response, make_response, abort, g, current_app as app + +from v3.auth.auth import admin_jwt, uma_enforce + +from clients.kong import get_services_by_ns, get_routes_by_ns + +gw_status = Blueprint('gw_status_v3', 'gw_status') + +@gw_status.route('', + methods=['GET'], strict_slashes=False) +@admin_jwt(None) +@uma_enforce('namespace', 'GatewayConfig.Publish') +def get_statuses(namespace: str) -> object: + + log = app.logger + + log.info("Get status for %s" % namespace) + + services = get_services_by_ns (namespace) + routes = get_routes_by_ns (namespace) + + response = [] + + for service in services: + url = build_url (service) + status = "UP" + reason = "" + + actual_host = None + host = None + for route in routes: + if route['service']['id'] == service['id'] and 'hosts' in route: + actual_host = route['hosts'][0] + if route['preserve_host']: + host = clean_host(actual_host) + + try: + addr = socket.gethostbyname(service['host']) + log.info("Address = %s" % addr) + except: + status = "DOWN" + reason = "DNS" + + if status == "UP": + try: + headers = {} + if host is None or service['host'].endswith('.svc'): + r = requests.get(url, headers=headers, timeout=3.0) + status_code = r.status_code + else: + u = urlparse(url) + + if host is None: + headers['Host'] = u.hostname + else: + headers['Host'] = host + + log.info("GET %-30s %s" % ("%s://%s" % (u.scheme, u.netloc), headers)) + + urllib3.disable_warnings() + if u.scheme == "https": + pool = urllib3.HTTPSConnectionPool( + "%s" % (u.netloc), + assert_hostname=host, + server_hostname=host, + cert_reqs='CERT_NONE', + ca_certs=certifi.where() + ) + else: + pool = urllib3.HTTPConnectionPool( + "%s" % (u.netloc) + ) + req = pool.urlopen( + "GET", + u.path, + headers={"Host": host}, + assert_same_host=False, + timeout=1.0, + retries=False + ) + + status_code = req.status + + log.info("Result received!! %d" % status_code) + if status_code < 400: + status = "UP" + reason = "%d Response" % status_code + elif status_code == 401 or status_code == 403: + status = "UP" + reason = "AUTH %d" % status_code + else: + status = "DOWN" + reason = "%d Response" % status_code + except requests.exceptions.Timeout as ex: + status = "DOWN" + reason = "TIMEOUT" + except urllib3.exceptions.ConnectTimeoutError as ex: + status = "DOWN" + reason = "TIMEOUT" + except requests.exceptions.ConnectionError as ex: + log.error("ConnError %s" % ex) + status = "DOWN" + reason = "CONNECTION" + except requests.exceptions.SSLError as ex: + status = "DOWN" + reason = "SSL" + except urllib3.exceptions.NewConnectionError as ex: + log.error("NewConnError %s" % ex) + status = "DOWN" + reason = "CON_ERR" + except urllib3.exceptions.SSLError as ex: + log.error(ex) + status = "DOWN" + reason = "SSL_URLLIB3" + except Exception as ex: + log.error(ex) + traceback.print_exc(file=sys.stdout) + status = "DOWN" + reason = "UNKNOWN" + + log.info("GET %-30s %s" % (url,reason)) + response.append({"name": service['name'], "upstream": url, "status": status, "reason": reason, "host": host, "env_host": actual_host}) + + return make_response(jsonify(response)) + +def build_url (s): + schema = default(s, "protocol", "http") + defaultPort = 80 + if schema == "https": + defaultPort = 443 + host = s['host'] + port = default(s, "port", defaultPort) + path = default(s, "path", "/") + if 'url' in s: + return s['url'] + else: + return "%s://%s:%d%s" % (schema, host, port, path) + + +def default (s, key, val): + if key in s and s[key] is not None: + return s[key] + else: + return val + + +def clean_host (host): + conf = app.config['hostTransformation'] + if conf['enabled'] is True: + conf = app.config['hostTransformation'] + return host.replace(conf['baseUrl'], 'gov.bc.ca').replace('-data-gov-bc-ca', '.data').replace('-api-gov-bc-ca', '.api').replace('-apps-gov-bc-ca', '.apps') + else: + return host diff --git a/microservices/gatewayApi/v3/routes/whoami.py b/microservices/gatewayApi/v3/routes/whoami.py new file mode 100644 index 0000000..f4ae1cc --- /dev/null +++ b/microservices/gatewayApi/v3/routes/whoami.py @@ -0,0 +1,30 @@ +import os +import shutil +from subprocess import Popen, PIPE +import uuid +import logging +import yaml +from flask import Blueprint, jsonify, request, Response, make_response, abort, g, current_app as app +from io import TextIOWrapper + +from v3.auth.auth import admin_jwt + +whoami = Blueprint('whoami_v3', 'whoami') + +@whoami.route('', + methods=['GET'], strict_slashes=False) +@admin_jwt(None) +def who_am_i() -> object: + """ + :return: JSON of some key information about the authenticated principal + """ + output = { + "authorized-party": g.principal['azp'], + "scope": g.principal['scope'], + "issuer": g.principal['iss'] + } + if ('aud' in g.principal): + output['audience'] = g.principal['aud'] + if ('clientAddress' in g.principal): + output['client-address'] = g.principal['clientAddress'] + return make_response(jsonify(output)) diff --git a/microservices/gatewayApi/v3/spec/spec.yaml b/microservices/gatewayApi/v3/spec/spec.yaml new file mode 100644 index 0000000..1a1e158 --- /dev/null +++ b/microservices/gatewayApi/v3/spec/spec.yaml @@ -0,0 +1,177 @@ +openapi: 3.0.0 +info: + version: 3.0.0 + title: Gateway Administration (GWA) API + license: + name: Apache 2.0 + description: |- + # Introduction + This set of APIs are responsible for: + * validating Kong Declarative Config and applying it to Kong + * managing namespaces to segment services +servers: + - url: "{{server_url}}" +tags: + - name: Status + description: API status + +paths: + /namespaces/{namespace}/services: + get: + summary: Get status of services + description: Ping the upstream service and get status of services + tags: + - Service Status + parameters: + - name: namespace + in: path + required: true + schema: + type: string + responses: + "200": + description: Success + "401": + $ref: "#/components/responses/401Unauthorized" + "400": + $ref: "#/components/responses/400BadRequest" + "500": + $ref: "#/components/responses/500InternalServerError" + + /namespaces/{namespace}/gateway: + put: + summary: Updates the gateway config + description: Returns the changes that were performed + tags: + - Gateway + parameters: + - name: namespace + in: path + required: true + schema: + type: string + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + dryRun: + type: boolean + configFile: + type: string + format: binary + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/status" + "401": + $ref: "#/components/responses/401Unauthorized" + "500": + $ref: "#/components/responses/500InternalServerError" + + /status: + get: + operationId: v1.v1.get_status + summary: Return overall API status + description: Returns the overall API status + tags: + - Status + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/status" + "401": + $ref: "#/components/responses/401Unauthorized" + "500": + $ref: "#/components/responses/500InternalServerError" + /whoami: + get: + summary: Return key information about authenticated identity + description: Authenticated identity details + tags: + - Who Am I + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/profile" + "401": + $ref: "#/components/responses/401Unauthorized" + "500": + $ref: "#/components/responses/500InternalServerError" + +components: + responses: + 401Unauthorized: + description: Unauthorized + 404NotFound: + description: Not Found + 400BadRequest: + description: Bad Request + content: + application/json: + schema: + $ref: "#/components/schemas/errorResponse" + 500InternalServerError: + description: Unexpected Error + content: + application/json: + schema: + $ref: "#/components/schemas/errorResponse" + schemas: + errorResponse: + type: object + properties: + error: + type: string + description: Error message + example: Something exploded + code: + type: integer + format: int32 + minimum: 0 + description: Error code + example: 42 + status: + type: object + properties: + message: + type: string + description: Human friendly response + example: Record updated + results: + type: string + description: Results from Change + profile: + type: object + properties: + namespace: + type: string + description: Namespace the identity has permission to access + NamespaceAttributes: + type: object + properties: + perm-domains: + type: array + items: + type: string + description: Overrides what domain suffixes are valid + example: ["api.gov.bc.ca"] + + securitySchemes: + oauth2: + type: oauth2 + flows: + clientCredentials: + tokenUrl: "{{accesstoken_url}}" +security: + - oauth2: [] diff --git a/microservices/gatewayApi/v3/v3.py b/microservices/gatewayApi/v3/v3.py new file mode 100644 index 0000000..87cbce2 --- /dev/null +++ b/microservices/gatewayApi/v3/v3.py @@ -0,0 +1,21 @@ +from flask import Blueprint, jsonify +from v3.routes.gateway import gw +from v3.routes.gw_status import gw_status +from v3.routes.whoami import whoami + +v3 = Blueprint('v3', 'v3') + +@v3.route('/status', methods=['GET'], strict_slashes=False) +def get_status(): + """ + Returns the overall API status + :return: JSON of endpoint status + """ + return jsonify({"status": "ok"}) + +class Register: + def __init__(self, app): + app.register_blueprint(v3, url_prefix="/v3") + app.register_blueprint(gw, url_prefix="/v3/namespaces//gateway") + app.register_blueprint(gw_status, url_prefix="/v3/namespaces//services") + app.register_blueprint(whoami, url_prefix="/v3/whoami") diff --git a/microservices/gatewayApi/wsgi.py b/microservices/gatewayApi/wsgi.py index 7f06603..cd1a0a2 100644 --- a/microservices/gatewayApi/wsgi.py +++ b/microservices/gatewayApi/wsgi.py @@ -40,7 +40,7 @@ app = create_app() -threading.Thread(name='swagger docs', target=setup_swagger_docs, args=(app,["v1", "v2"])).start() +threading.Thread(name='swagger docs', target=setup_swagger_docs, args=(app,["v1", "v2", "v3"])).start() def signal_handler(sig, frame): log.info('You pressed Ctrl+C - exiting!')