Enhanced Target Server Validator

tools/target-server-validator/apigee_utils.py

@@ -20,6 +20,13 @@
 import requests
 import shutil
 from time import sleep
+from utilities import (  # pylint: disable=import-error
+    run_validator_proxy,
+    unzip_file,
+    parse_proxy_hosts,
+    get_tes,
+)
+from base_logger import logger
 
 
 class Apigee:
@@ -45,7 +52,7 @@ def is_token_valid(self, token):
         url = f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={token}"  # noqa
         response = requests.get(url)
         if response.status_code == 200:
-            print(f"Token Validated for user {response.json()['email']}")
+            logger.info(f"Token Validated for user {response.json()['email']}")
             return True
         return False
 
@@ -60,19 +67,15 @@ def get_access_token(self):
                 if self.is_token_valid(token):
                     return token
                 else:
-                    print(
-                        'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! '  # noqa type: ignore
-                    )
+                    logger.error('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ')  # noqa
                     sys.exit(1)
             else:
                 return token
         else:
             if self.apigee_type == "x":
-                print(
-                    'please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! '  # noqa
-                )
+                logger.error('please run "export APIGEE_ACCESS_TOKEN=$(gcloud auth print-access-token)" first !! ')  # noqa
             else:
-                print("please export APIGEE_OPDK_ACCESS_TOKEN")
+                logger.error('please export APIGEE_OPDK_ACCESS_TOKEN')
             sys.exit(1)
 
     def set_auth_header(self):
@@ -136,7 +139,7 @@ def create_api(self, api_name, proxy_bundle_path):
         if response.status_code == 200:
             revision = response.json().get('revision', "1")
             return True, revision
-        print(response.text)
+        logger.debug(f"{response.text}")
         return False, None
 
     def get_api_revisions_deployment(self, env, api_name, api_rev):  # noqa
@@ -154,10 +157,10 @@ def get_api_revisions_deployment(self, env, api_name, api_rev):  # noqa
             if self.apigee_type == "opdk":
                 if api_deployment_status == "deployed":
                     return True
-            print(f"API {api_name} is in Status: {api_deployment_status} !")  # noqa
+            logger.debug(f"API {api_name} is in Status: {api_deployment_status} !")  # noqa
             return False
         else:
-            print(response.text)
+            logger.debug(f"{response.text}")
             return False
 
     def deploy_api(self, env, api_name, api_rev):
@@ -171,9 +174,9 @@ def deploy_api(self, env, api_name, api_rev):
         else:
             resp = response.json()
             if "already deployed" in resp["error"]["message"]:
-                print("Proxy {} is already Deployed".format(api_name))
+                logger.info(f"Proxy {api_name} is already Deployed")
                 return True
-            print(response.text)
+            logger.debug(f"{response.text}")
             return False
 
     def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_force_redeploy=False):  # noqa
@@ -185,50 +188,38 @@ def deploy_api_bundle(self, env, api_name, proxy_bundle_path, api_force_redeploy
         if get_api_status:
             api_exists = True
             api_rev = api_revs[-1]
-            print(
-                f"Proxy with name {api_name} with revision {api_rev} already exists in Apigee Org {self.org}"  # noqa
-            )
+            logger.warning(f"Proxy with name {api_name} with revision {api_rev} already exists in Apigee Org {self.org}")  # noqa
             if api_force_redeploy:
                 api_exists = False
         if not api_exists:
             api_created, api_rev = self.create_api(api_name, proxy_bundle_path)
             if api_created:
-                print(
-                    f"Proxy has been imported with name {api_name} in Apigee Org {self.org}"  # noqa
-                )
+                logger.info(f"Proxy has been imported with name {api_name} in Apigee Org {self.org}")  # noqa
                 api_exists = True
             else:
-                print(f"ERROR : Proxy {api_name} import failed !!! ")
+                logger.error(f"ERROR : Proxy {api_name} import failed !!! ")
                 return False
         if api_exists:
             if self.get_api_revisions_deployment(
                         env, api_name, api_rev
                     ):
-                print(f"INFO : Proxy {api_name} already active in to {env} in Apigee Org {self.org} !")  # noqa
+                logger.info(f"Proxy {api_name} already active in to {env} in Apigee Org {self.org} !")  # noqa
                 return True
             else:
                 if self.deploy_api(env, api_name, api_rev):
-                    print(
-                        f"Proxy with name {api_name} has been deployed  to {env} in Apigee Org {self.org}"  # noqa
-                    )
+                    logger.info(f"Proxy with name {api_name} has been deployed  to {env} in Apigee Org {self.org}")  # noqa
                     while api_deployment_retry_count < api_deployment_retry:
                         if self.get_api_revisions_deployment(
                             env, api_name, api_rev
                         ):
-                            print(
-                                f"Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds "  # noqa
-                            )
+                            logger.debug(f"Proxy {api_name} active in runtime after {api_deployment_retry_count*api_deployment_sleep} seconds ")  # noqa
                             return True
                         else:
-                            print(
-                                f"Checking API deployment status in {api_deployment_sleep} seconds"  # noqa
-                            )
+                            logger.debug(f"Checking API deployment status in {api_deployment_sleep} seconds")  # noqa
                             sleep(api_deployment_sleep)
                             api_deployment_retry_count += 1
                 else:
-                    print(
-                        f"ERROR : Proxy deployment  to {env} in Apigee Org {self.org} Failed !!"  # noqa
-                    )
+                    logger.error(f"ERROR : Proxy deployment  to {env} in Apigee Org {self.org} Failed !!")  # noqa
                     return False
 
     def get_api_vhost(self, vhost_name, env):
@@ -244,13 +235,11 @@ def get_api_vhost(self, vhost_name, env):
             else:
                 hosts = response.json()["hostnames"]
             if len(hosts) == 0:
-                print(
-                    f"ERROR: Vhost/Env Group {vhost_name} contains no domains"  # noqa
-                )
+                logger.error(f"Vhost/Env Group {vhost_name} contains no domains")  # noqa
                 return None
             return hosts
         else:
-            print(f"ERROR: Vhost/Env Group {vhost_name} contains no domains")  # noqa
+            logger.error(f"Vhost/Env Group {vhost_name} contains no domains")
             return None
 
     def list_apis(self, api_type):
@@ -289,6 +278,44 @@ def fetch_api_revision(self, api_type, api_name, revision, export_dir):  # noqa
             return True
         return False
 
+    def fetch_api_proxy_ts_parallel(self, arg_tuple):
+        self.fetch_api_revision(arg_tuple[0], arg_tuple[1], arg_tuple[2], arg_tuple[3])  # noqa
+        unzip_file(
+                    f"{arg_tuple[3]}/{arg_tuple[1]}.zip",  # noqa
+                    f"{arg_tuple[3]}/{arg_tuple[1]}",  # noqa
+                )
+        parsed_proxy_hosts = parse_proxy_hosts(f"{arg_tuple[3]}/{arg_tuple[1]}/apiproxy")  # noqa
+        proxy_tes = get_tes(parsed_proxy_hosts)
+        return arg_tuple[0], arg_tuple[1], parsed_proxy_hosts, proxy_tes
+
+    def fetch_env_target_servers_parallel(self, arg_tuple):
+        ts_info = self.get_target_server(arg_tuple[0], arg_tuple[1])
+        return arg_tuple[1], ts_info
+
+    def call_validator_proxy_parallel(self, arg_tuple):
+        response = run_validator_proxy(arg_tuple[0], arg_tuple[1], arg_tuple[2], arg_tuple[3], arg_tuple[4])  # noqa
+        if response.get("error"):
+            logger.error(f"Error while calling the validator proxy - {response['error']}")  # noqa
+        else:
+            report = []
+            outputs = response.get("hostname_portnumbers_status", [])
+            for output in outputs:
+                report.append(
+                    [
+                        output["name"],
+                        output["extracted_from"],
+                        output["host"],
+                        output["port"],
+                        output["env"],
+                        output["status"],
+                        output["info"] if output.get("info") else " & ".join(
+                            list(set(arg_tuple[5][output["name"]])))
+                        if output["name"] in arg_tuple[5]
+                        else "No References in any API",
+                    ]
+                )
+            return report
+
     def write_proxy_bundle(self, export_dir, file_name, data):
         file_path = f"./{export_dir}/{file_name}.zip"
         with open(file_path, "wb") as fl:

tools/target-server-validator/apiproxy/policies/AM-Set-Json-Response.xml

@@ -13,11 +13,6 @@
 -->
 <AssignMessage name="AM-Set-Json-Response">
     <Set>
-        <Payload contentType="application/json">{
-    "host":"{request.header.host_name}", 
-    "port": "{request.header.port_number}", 
-    "status":"{flow.reachableStatus}"
-}
-</Payload>
+        <Payload contentType="application/json">{flow.result}</Payload>
     </Set>
 </AssignMessage>

tools/target-server-validator/apiproxy/policies/JSON-Threat-Protection.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<JSONThreatProtection continueOnError="false" enabled="true" name="JSON-Threat-Protection">
+    <DisplayName>JSON Threat Protection</DisplayName>
+    <Properties/>
+    <ArrayElementCount>5</ArrayElementCount>
+    <Source>request</Source>
+</JSONThreatProtection>

tools/target-server-validator/apiproxy/proxies/default.xml

@@ -14,6 +14,9 @@
 <ProxyEndpoint name="default">
     <PreFlow name="PreFlow">
         <Request>
+            <Step>
+                <Name>JSON-Threat-Protection</Name>
+            </Step>
             <Step>
                 <Name>JC-Port-Open-Check</Name>
             </Step>

tools/target-server-validator/base_logger.py

@@ -0,0 +1,49 @@
+import os
+import logging
+
+EXEC_INFO = True if os.getenv("EXEC_INFO") == "True" else False
+LOG_HANDLER = os.getenv("LOG_HANDLER", "Stream")
+LOG_FILE_PATH = os.getenv("LOG_FILE_PATH", "app.log")
+LOGLEVEL = os.getenv('LOGLEVEL', 'INFO').upper()
+
+if LOG_HANDLER not in {"File", "Stream"}:
+    LOG_HANDLER = "Stream"
+
+if LOGLEVEL not in {"CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"}:
+    LOGLEVEL = "INFO"
+
+
+class CustomFormatter(logging.Formatter):
+
+    grey = "\x1b[38;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"  # noqa
+
+    FORMATS = {
+        logging.DEBUG: grey + format + reset,
+        logging.INFO: grey + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red + format + reset
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
+logger = logging.getLogger("TargetServerValidator")
+logger.setLevel(getattr(logging, LOGLEVEL))  # Add this line
+
+if LOG_HANDLER == "File":
+    ch = logging.FileHandler(LOG_FILE_PATH, mode="a")
+else:
+    ch = logging.StreamHandler()
+
+ch.setFormatter(CustomFormatter())
+
+logger.addHandler(ch)

tools/target-server-validator/callout/pom.xml

@@ -24,7 +24,8 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <java.version>1.7</java.version>
+    <javaxjson.version>1.1.4</javaxjson.version>
+    <java.version>1.8</java.version>
     <apiproxy.java.rsrc.dir>../apiproxy/resources/java</apiproxy.java.rsrc.dir>
     <testng.version>6.8.7</testng.version> <!-- current: 6.9.4 -->
     <jmockit.version>1.7</jmockit.version>
@@ -70,6 +71,16 @@ do it manually by running these commands:
       <artifactId>expressions</artifactId>
       <version>1.0.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish</groupId>
+      <artifactId>javax.json</artifactId>
+      <version>1.1.4</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.json</groupId>
+      <artifactId>javax.json-api</artifactId>
+      <version>1.1.4</version>
+    </dependency>
 
   </dependencies>
 
@@ -134,18 +145,23 @@ do it manually by running these commands:
           <execution>
             <phase>package</phase>
             <configuration>
-              <tasks>
+              <target>
                 <copy todir="${apiproxy.java.rsrc.dir}">
                   <fileset dir="${project.build.directory}">
                     <include name="${project.artifactId}.jar"/>
-                    <!-- copy any required dependencies tht are not
-                    already included in the Edge runtime: -->
-
-                    <!--                    -->
                   </fileset>
                   <mapper type="flatten"/>
                 </copy>
-              </tasks>
+
+                <!-- Copy the javax jar files -->
+                <copy todir="${apiproxy.java.rsrc.dir}">
+                  <fileset dir="${project.build.directory}/lib">
+                    <include name="javax.json-1.1.4.jar"/>
+                    <include name="javax.json-api-1.1.4.jar"/>
+                  </fileset>
+                  <mapper type="flatten"/>
+                </copy>
+              </target>
             </configuration>
             <goals>
               <goal>run</goal>