Merge pull request #832 from cncf/update-keggs

Update metadata gathering functions

apps/snoopdb/postgres/snoopUtils.py

@@ -20,8 +20,8 @@
 AKC_BUCKET="ci-audit-kind-conformance"
 KGCL_BUCKET="ci-kubernetes-gce-conformance-latest"
 KEGG_BUCKET="ci-kubernetes-e2e-gci-gce"
+CONFORMANCE_RUNS="https://prow.k8s.io/job-history/kubernetes-jenkins/logs/"
 
-AUDIT_KIND_CONFORMANCE_RUNS="https://prow.k8s.io/job-history/kubernetes-jenkins/logs/ci-audit-kind-conformance"
 AUDIT_KIND_CONFORMANCE_LOGS="https://storage.googleapis.com/kubernetes-jenkins/logs/ci-audit-kind-conformance"
 GCS_LOGS="https://storage.googleapis.com/kubernetes-jenkins/logs/"
 
@@ -295,11 +295,12 @@ def find_operation_id(openapi_spec, event):
     openapi_spec['hit_cache'][url.path][method]=op_id
   return op_id, None
 
-def akc_latest_success():
+def bucket_latest_success(bucket):
     """
     determines latest successful run for ci-audit-kind-conformance and returns its ID as a string.
     """
-    soup = get_html(AUDIT_KIND_CONFORMANCE_RUNS)
+    test_runs = CONFORMANCE_RUNS + bucket
+    soup = get_html(test_runs)
     scripts = soup.find(is_spyglass_script)
     if scripts is None :
         raise ValueError("No spyglass script found in akc page")
@@ -345,34 +346,42 @@ def akc_timestamp(job):
     started = json.loads(urlopen(started_url).read().decode('utf-8'))
     return started["timestamp"]
 
-def akc_meta(custom_job=None):
+def akc_meta(bucket, custom_job=None):
     """
     Compose a Meta object for job of given AKC bucket.
     Meta object contains the job, the k8s version, the k8s commit, the audit log links for the test run, and thed timestamp of the testrun
     """
-    job = akc_latest_success() if custom_job is None else custom_job
+    job = bucket_latest_success(bucket) if custom_job is None else custom_job
     return Meta(job,
                 akc_version(job),
                 akc_commit(job),
                 akc_loglinks(job),
                 akc_timestamp(job))
 
-def kgcl_version(job_version):
+def kgcl_version(job):
     """
     return k8s semver for version of k8s run in given job's test run
     """
+    finished_url = GCS_LOGS + KGCL_BUCKET + '/' + job + '/finished.json'
+    finished = get_json(finished_url)
+    job_version = finished["metadata"]["job-version"]
+
     match = re.match("^v([0-9.]+)-",job_version)
     if match is None:
         raise ValueError("Could not find version in given job_version.", job_version)
     else:
         version = match.group(1)
         return version
 
-def kgcl_commit(job_version):
+def kgcl_commit(job):
     """
     return k8s/k8s commit for k8s used in given job's test run
     """
     # we want the end of the string, after the '+'. A commit should only be numbers and letters
+    finished_url = GCS_LOGS + KGCL_BUCKET + '/' + job + '/finished.json'
+    finished = get_json(finished_url)
+    job_version = finished["metadata"]["job-version"]
+
     match = re.match(".+\+([0-9a-zA-Z]+)$",job_version)
     if match is None:
         raise ValueError("Could not find commit in given job_version", job_version)
@@ -396,40 +405,41 @@ def kgcl_timestamp(job):
     finished = get_json(finished_url)
     return finished["timestamp"]
 
-def kgcl_meta(custom_job=None):
+def kgcl_meta(bucket, custom_job=None):
     """
     Compose a Meta object for job of given KGCL bucket.
     Meta object contains the job, the k8s version, the k8s commit, the audit log links for the test run, and thed timestamp of the testrun
     """
-    testgrid_history = get_json(GCS_LOGS + KGCL_BUCKET + "/jobResultsCache.json")
-    if custom_job is not None:
-        build = [x for x in testgrid_history if x['buildnumber'] == custom_job][0]
-    else:
-        build = [x for x in testgrid_history if x['result'] == 'SUCCESS'][-1]
-    job = build["buildnumber"]
-    job_version = build["job-version"]
+    job = bucket_latest_success(bucket) if custom_job is None else custom_job
     return Meta(job,
-                kgcl_version(job_version),
-                kgcl_commit(job_version),
+                kgcl_version(job),
+                kgcl_commit(job),
                 kgcl_loglinks(job),
                 kgcl_timestamp(job))
 
-def kegg_version(job_version):
+def kegg_version(job):
     """
     return k8s semver for version of k8s run in given job's test run
     """
+    finished_url = GCS_LOGS + KEGG_BUCKET + '/' + job + '/finished.json'
+    finished = get_json(finished_url)
+    job_version = finished["metadata"]["job-version"]
+
     match = re.match("^v([0-9.]+)-",job_version)
     if match is None:
         raise ValueError("Could not find version in given job_version.", job_version)
     else:
         version = match.group(1)
         return version
 
-def kegg_commit(job_version):
+def kegg_commit(job):
     """
     return k8s/k8s commit for k8s used in given job's test run
     """
     # we want the end of the string, after the '+'. A commit should only be numbers and letters
+    finished_url = GCS_LOGS + KEGG_BUCKET + '/' + job + '/finished.json'
+    finished = get_json(finished_url)
+    job_version = finished["metadata"]["job-version"]
     match = re.match(".+\+([0-9a-zA-Z]+)$",job_version)
     if match is None:
         raise ValueError("Could not find commit in given job_version.", job_version)
@@ -453,33 +463,27 @@ def kegg_timestamp(job):
     finished = get_json(finished_url)
     return finished["timestamp"]
 
-def kegg_meta(custom_job=None):
+def kegg_meta(bucket, custom_job=None):
     """
     Compose a Meta object for job of given KEGG bucket.
     Meta object contains the job, the k8s version, the k8s commit, the audit log links for the test run, and thed timestamp of the testrun
     """
-    testgrid_history = get_json(GCS_LOGS + KEGG_BUCKET + "/jobResultsCache.json")
-    if custom_job is not None:
-        build = [x for x in testgrid_history if x['buildnumber'] == custom_job][0]
-    else:
-        build = [x for x in testgrid_history if x['result'] == 'SUCCESS'][-1]
-    job = build["buildnumber"]
-    job_version = build["job-version"]
+    job = bucket_latest_success(bucket) if custom_job is None else custom_job
     return Meta(job,
-                kegg_version(job_version),
-                kegg_commit(job_version),
+                kegg_version(job),
+                kegg_commit(job),
                 kegg_loglinks(job),
                 kegg_timestamp(job))
 
 def get_meta(bucket,job=None):
     """Returns meta object for given bucket.
     Meta includes job, k8s version, k8s commit, all auditlog links, and timestamp of the test run"""
     if(bucket == AKC_BUCKET):
-        return akc_meta(job)
+        return akc_meta(bucket,job)
     elif(bucket == KGCL_BUCKET):
-        return kgcl_meta(job)
+        return kgcl_meta(bucket,job)
     elif(bucket == KEGG_BUCKET):
-        return kegg_meta(job)
+        return kegg_meta(bucket, job)
 
 def download_and_process_auditlogs(bucket,job):
     """