From 593881a2382889ad1d44a5c1c2071ccf1f5f9b58 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Sat, 19 Feb 2022 15:14:48 -0800 Subject: [PATCH 1/7] feat: add pbt support to new-ui --- .../constants/algorithms-settings.const.ts | 24 +++++++++++++++++++ .../app/constants/algorithms-types.const.ts | 1 + .../src/app/enumerations/algorithms.enum.ts | 1 + 3 files changed, 26 insertions(+) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts index 0ca7f3c0b90..243cd321e4a 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts @@ -242,6 +242,29 @@ export const DartsSettings: AlgorithmSetting[] = [ }, ]; +export const PbtSettings: AlgorithmSetting[] = [ + { + name: 'checkpoint_dir', + value: '/var/log/katib/checkpoints/', + type: AlgorithmSettingType.STRING, + }, + { + name: 'n_population', + value: 40, + type: AlgorithmSettingType.INTEGER, + }, + { + name: 'resample_probability', + value: null, + type: AlgorithmSettingType.FLOAT, + }, + { + name: 'truncation_threshold', + value: 0.2, + type: AlgorithmSettingType.FLOAT, + }, +]; + export const EarlyStoppingSettings: AlgorithmSetting[] = [ { name: 'min_trials_required', @@ -271,4 +294,5 @@ export const AlgorithmSettingsMap: { [key: string]: AlgorithmSetting[] } = { [AlgorithmsEnum.SOBOL]: SOBOLSettings, [AlgorithmsEnum.ENAS]: ENASSettings, [AlgorithmsEnum.DARTS]: DartsSettings, + [AlgorithmsEnum.PBT]: PbtSettings, }; diff --git a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts index 607e6f2db63..4f1f0aa0cfd 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts @@ -12,6 +12,7 @@ export const AlgorithmNames = { [AlgorithmsEnum.MULTIVARIATE_TPE]: 'Multivariate Tree of Parzen Estimators', [AlgorithmsEnum.CMAES]: 'Covariance Matrix Adaptation: Evolution Strategy', [AlgorithmsEnum.SOBOL]: 'Sobol Quasirandom Sequence', + [AlgorithmsEnum.PBT]: 'Population Based Training', }; export const NasAlgorithmNames = { diff --git a/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts b/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts index 425d423199d..8f7476468b1 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts @@ -9,6 +9,7 @@ export enum AlgorithmsEnum { SOBOL = 'sobol', ENAS = 'enas', DARTS = 'darts', + PBT = 'pbt', } export enum EarlyStoppingAlgorithmsEnum { From 319607b0cd8c54f597ecdfd4ae89ebd59e66da58 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Mon, 21 Mar 2022 23:19:33 -0700 Subject: [PATCH 2/7] feat: add endpoint for accessing annotation field --- cmd/new-ui/v1beta1/main.go | 1 + .../src/app/services/backend.service.ts | 6 ++ pkg/new-ui/v1beta1/hp.go | 63 ++++++++++++++++++- pkg/new-ui/v1beta1/util.go | 4 +- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/cmd/new-ui/v1beta1/main.go b/cmd/new-ui/v1beta1/main.go index 1566dfca864..bc349364362 100644 --- a/cmd/new-ui/v1beta1/main.go +++ b/cmd/new-ui/v1beta1/main.go @@ -59,6 +59,7 @@ func main() { http.HandleFunc("/katib/fetch_hp_job_info/", kuh.FetchHPJobInfo) http.HandleFunc("/katib/fetch_hp_job_trial_info/", kuh.FetchHPJobTrialInfo) + http.HandleFunc("/katib/fetch_hp_job_annotation_info/", kuh.FetchHPJobAnnotationInfo) http.HandleFunc("/katib/fetch_nas_job_info/", kuh.FetchNASJobInfo) http.HandleFunc("/katib/fetch_trial_templates/", kuh.FetchTrialTemplates) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts index 60ac1ab0c34..7bec30ecf31 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts @@ -56,6 +56,12 @@ export class KWABackendService extends BackendService { return this.http.get(url).pipe(catchError(error => this.parseError(error))); } + getExperimentAnnotationsInfo(name: string, namespace: string): Observable { + const url = `/katib/fetch_hp_job_annotation_info/?experimentName=${name}&namespace=${namespace}`; + + return this.http.get(url).pipe(catchError(error => this.parseError(error))); + } + getExperiment(name: string, namespace: string): Observable { const url = `/katib/fetch_experiment/?experimentName=${name}&namespace=${namespace}`; diff --git a/pkg/new-ui/v1beta1/hp.go b/pkg/new-ui/v1beta1/hp.go index 006d799c78d..c882a1bded1 100644 --- a/pkg/new-ui/v1beta1/hp.go +++ b/pkg/new-ui/v1beta1/hp.go @@ -72,13 +72,13 @@ func (k *KatibUIHandler) FetchHPJobInfo(w http.ResponseWriter, r *http.Request) log.Printf("Got Trial List") // append a column for the Pipeline UID associated with the Trial - if havePipelineUID(trialList.Items) { + if hasAnnotation(trialList.Items, kfpRunIDAnnotation) { resultText += ",KFP Run" } foundPipelineUID := false for _, t := range trialList.Items { - runUid, ok := t.GetAnnotations()[kfpRunIDAnnotation] + runUid, ok := t.Spec.Annotations[kfpRunIDAnnotation] if !ok { log.Printf("Trial %s has no pipeline run.", t.Name) runUid = "" @@ -245,3 +245,62 @@ func (k *KatibUIHandler) FetchHPJobTrialInfo(w http.ResponseWriter, r *http.Requ return } } + +func (k *KatibUIHandler) FetchHPJobAnnotationInfo(w http.ResponseWriter, r *http.Request) { + //enableCors(&w) + experimentName := r.URL.Query()["experimentName"][0] + namespace := r.URL.Query()["namespace"][0] + + resultText := "trialName" + + trialList, err := k.katibClient.GetTrialList(experimentName, namespace) + if err != nil { + log.Printf("GetTrialList from HP job failed: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + log.Printf("Got Trial List") + + // Find set(annotations) + annotations := map[string]int{} + for _, t := range trialList.Items { + for a := range t.Spec.Annotations { + annotations[a] = 0 + } + } + + // Set the index of each annotation + a_index := 0 + for a, _ := range annotations { + annotations[a] = a_index + resultText += "," + a + a_index++ + } + log.Printf("Got Annotations List") + + for _, t := range trialList.Items { + trialResText := make([]string, len(annotations)) + for a, n := range annotations { + a_value, ok := t.Spec.Annotations[a] + if !ok { + log.Printf("Trial %s has no annotation value for %s", t.Name, a) + a_value = "" + } + trialResText[n] = a_value + } + resultText += "\n" + t.Name + "," + strings.Join(trialResText, ",") + } + + log.Printf("Logs parsed, results:\n %v", resultText) + response, err := json.Marshal(resultText) + if err != nil { + log.Printf("Marshal result text for HP job failed: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if _, err = w.Write(response); err != nil { + log.Printf("Write result text for HP job failed: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } +} diff --git a/pkg/new-ui/v1beta1/util.go b/pkg/new-ui/v1beta1/util.go index c5d6c5cf13c..195d54bf58e 100644 --- a/pkg/new-ui/v1beta1/util.go +++ b/pkg/new-ui/v1beta1/util.go @@ -65,9 +65,9 @@ func (k *KatibUIHandler) getExperiments(namespace []string) ([]ExperimentView, e return experiments, nil } -func havePipelineUID(trials []trialv1beta1.Trial) bool { +func hasAnnotation(trials []trialv1beta1.Trial, annotation string) bool{ for _, t := range trials { - _, ok := t.GetAnnotations()[kfpRunIDAnnotation] + _, ok := t.Spec.Annotations[annotation] if ok { return true } From 355b6594fd11f6a498cfd9e12a1dfe605f7f7405 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Sun, 3 Apr 2022 00:02:32 -0700 Subject: [PATCH 3/7] feat: add pbt tab to experiment view to enable more suitable inspection of HP sweep --- .../experiment-details.component.html | 7 + .../experiment-details.component.ts | 6 + .../experiment-details.module.ts | 2 + .../pbt/pbt-tab-loader.module.ts | 16 + .../pbt/pbt-tab.component.html | 18 + .../pbt/pbt-tab.component.scss | 48 +++ .../pbt/pbt-tab.component.ts | 393 ++++++++++++++++++ 7 files changed, 490 insertions(+) create mode 100644 pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts create mode 100644 pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html create mode 100644 pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss create mode 100644 pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html index 4d1233302cf..68d177a347a 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html @@ -60,6 +60,13 @@ [experimentJson]="experimentDetails" > + + + diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts index 33f129b07c7..4b0605b86e4 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts @@ -30,6 +30,7 @@ export class ExperimentDetailsComponent implements OnInit, OnDestroy { columns: string[] = []; details: string[][] = []; experimentTrialsCsv: string; + annotationsCsv: string; hoveredTrial: number; experimentDetails: ExperimentK8s; showGraph: boolean; @@ -94,6 +95,11 @@ export class ExperimentDetailsComponent implements OnInit, OnDestroy { this.details = this.parseTrialsDetails(data.details); this.showGraph = true; }); + this.backendService + .getExperimentAnnotationsInfo(this.name, this.namespace) + .subscribe(response => { + this.annotationsCsv = response; + }); this.backendService .getExperiment(this.name, this.namespace) .subscribe((response: ExperimentK8s) => { diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.module.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.module.ts index 541edb0af97..4a8983364f8 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.module.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.module.ts @@ -16,6 +16,7 @@ import { ExperimentOverviewModule } from './overview/experiment-overview.module' import { ExperimentDetailsTabModule } from './details/experiment-details-tab.module'; import { TrialsGraphModule } from './trials-graph/trials-graph.module'; import { ExperimentYamlModule } from './yaml/experiment-yaml.module'; +import { PbtTabModule } from './pbt/pbt-tab-loader.module'; @NgModule({ declarations: [ExperimentDetailsComponent], @@ -33,6 +34,7 @@ import { ExperimentYamlModule } from './yaml/experiment-yaml.module'; MatProgressSpinnerModule, ExperimentYamlModule, TitleActionsToolbarModule, + PbtTabModule, ], exports: [ExperimentDetailsComponent], }) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts new file mode 100644 index 00000000000..2906abffe75 --- /dev/null +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts @@ -0,0 +1,16 @@ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; + +import { FormsModule } from '@angular/forms'; +import { MatFormFieldModule } from '@angular/material/form-field'; +import { MatSelectModule } from '@angular/material/select'; +import { MatCheckboxModule } from '@angular/material/checkbox'; + +import { PbtTabComponent } from './pbt-tab.component'; + +@NgModule({ + declarations: [PbtTabComponent], + imports: [CommonModule, FormsModule, MatFormFieldModule, MatSelectModule, MatCheckboxModule], + exports: [PbtTabComponent], +}) +export class PbtTabModule {} diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html new file mode 100644 index 00000000000..8f4cdbbcc10 --- /dev/null +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html @@ -0,0 +1,18 @@ +
+ +
+ + Y-Axis + + + {{name}} + + + + + Display Seed Traces +
+ +
+ +
diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss new file mode 100644 index 00000000000..89c19105543 --- /dev/null +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss @@ -0,0 +1,48 @@ +:host { + display: block; +} + +.pbt-wrapper { + position: relative; + display: flex; + justify-content: center; + align-items: center; + flex-direction: column; +} + +.pbt-options-wrapper { + display: flex; + align-items: center; + flex-direction: row; +} + +.pbt-option { + margin: 10px +} + +.d3-tab-graph { + width: 400px; + text-align: center; + + @media (min-width: 768px) { + width: 700px; + } + + @media (min-width: 1024px) { + width: 1000px; + } + + @media (min-width: 1400px) { + width: 1300px; + } + + @media (min-width: 1650px) { + width: 1600px; + } + + @media (min-width: 2000px) { + width: 1900px; + } + + height: 600px; +} diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts new file mode 100644 index 00000000000..1f4aacfd5eb --- /dev/null +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts @@ -0,0 +1,393 @@ +import { + ChangeDetectionStrategy, + Component, + Input, + OnChanges, + OnInit, + AfterViewInit, + SimpleChanges, + ElementRef, + ViewChild, +} from '@angular/core'; +import lowerCase from 'lodash-es/lowerCase'; +import { safeDivision } from 'src/app/shared/utils'; +import { ExperimentK8s } from '../../../models/experiment.k8s.model'; + +import { Inject } from '@angular/core'; +import { DOCUMENT } from '@angular/common'; + +declare let d3: any; + +type PbtPoint = { + uid: string; + parentUid: string; + trialName: string; + parameters: Object; // all y-axis possible values (parameters + alternativeMetrics) + generation: number; // generation + metricValue: number; // evaluation metric +}; + +type PbtAnnotation = { + uid: string; + parentUid: string; + generation: number; + trialName: string; +} + +@Component({ + selector: 'app-experiment-pbt-tab', + templateUrl: './pbt-tab.component.html', + styleUrls: ['./pbt-tab.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { + @ViewChild('pbtGraph') graphWrapper:ElementRef; // used to manipulate svg dom + + graph: any; // svg dom + selectableNames: string[]; // list of parameters/metrics for UI dropdown + selectedName: string; // user selected parameter/metric + displayTrace: boolean; + + private annotData: {[uid: string]: PbtAnnotation} = {}; + private trialData = {}; // primary key: trialName + private parameterNames: string[]; // parameter names + private goalName: string = ""; + private data: PbtPoint[][] = []; // data sorted by generation and segment + private graphHelper: any; // graph metadata and auxiliary info + + @Input() + experiment: ExperimentK8s; + + @Input() + annotationsCsv: string[] = []; + + @Input() + experimentTrialsCsv: string[] = []; + + constructor(@Inject(DOCUMENT) private document: Document) { + this.graphHelper = {}; + this.graphHelper.margin = {top: 10, right: 30, bottom: 30, left: 60}; + this.graphHelper.width = 460 - this.graphHelper.margin.left - this.graphHelper.margin.right; + this.graphHelper.height = 400 - this.graphHelper.margin.top - this.graphHelper.margin.bottom; + // Track angular initialization since using @Input from template + this.graphHelper.ngInit = false; + } + + ngOnInit(): void { + if (this.experiment.spec.algorithm.algorithmName != "pbt") { + // Prevent initialization if not Pbt + return; + } + + // Create full list of parameters + this.parameterNames = this.experiment.spec.parameters.map(param => { + return param.name; + }); + // Identify goal + this.goalName = this.experiment.spec.objective.objectiveMetricName; + // Create full list of selectable names + this.selectableNames = [...this.parameterNames]; + if (this.experiment.spec.objective.additionalMetricNames && this.experiment.spec.objective.additionalMetricNames.length > 0) { + this.selectableNames = [...this.selectableNames, ...this.experiment.spec.objective.additionalMetricNames]; + } + // Create converters for all possible y-axes + this.graphHelper.yMeta = {}; + for (const param of this.experiment.spec.parameters) { + this.graphHelper.yMeta[param.name] = {}; + if (param.parameterType == "discrete" || param.parameterType == "categorical") { + this.graphHelper.yMeta[param.name].transform = (x) => { return x; }; + this.graphHelper.yMeta[param.name].isNumber = false; + } else if (param.parameterType == "double") { + this.graphHelper.yMeta[param.name].transform = (x) => { return parseFloat(x); }; + this.graphHelper.yMeta[param.name].isNumber = true; + } else { + this.graphHelper.yMeta[param.name].transform = (x) => { return parseInt(x); }; + this.graphHelper.yMeta[param.name].isNumber = true; + } + } + if (this.experiment.spec.objective.additionalMetricNames) { + for (const metricName of this.experiment.spec.objective.additionalMetricNames) { + if (this.graphHelper.yMeta.hasOwnProperty(metricName)){ + console.warn("Additional metric name conflict with parameter name; ignoring metric:", metricName); + continue; + } + this.graphHelper.yMeta[metricName].transform = (x) => { return parseFloat(x); }; + this.graphHelper.yMeta[metricName].isNumber = true; + } + } + + this.graphHelper.ngInit = true; + } + + ngAfterViewInit(): void { + if (this.experiment.spec.algorithm.algorithmName != "pbt") { + // Remove pbt tab and tab content + const tabs = document.querySelectorAll(".mat-tab-labels .mat-tab-label"); + for (let i = 0; i < tabs.length; i++) { + if (tabs[i].querySelector(".mat-tab-label-content").innerHTML.includes("PBT")){ + const tabId = tabs[i].getAttribute("id"); + const tabBodyId = tabId.replace("label", "content") + const tabBody = document.querySelector("#" + tabBodyId); + tabBody.remove(); + tabs[i].remove(); + break; + } + } + return; + } + // Specify default choice for dropdown menu + this.selectedName = this.selectableNames[0]; + // Specify default trace view + this.displayTrace = false; + } + + onDropdownChange(){ + // Trigger graph redraw on dropdown change event + this.clearGraph(); + this.updateGraph(); + } + + onTraceChange(){ + // Trigger graph redraw on trace change event + // TODO: could use d3.select(..).remove() instead of recreating + this.clearGraph(); + this.updateGraph(); + } + + ngOnChanges(changes: SimpleChanges): void { + if (!this.graphHelper || !this.graphHelper.ngInit) { + // Wait for view to properly initialize + return; + } + // Recompute formatted plotting points on data input changes + let updatePoints = false; + if (changes.experimentTrialsCsv && this.experimentTrialsCsv) { + let trialArr = d3.csv.parse(this.experimentTrialsCsv); + for (let trial of trialArr) { + let existingEntry = this.trialData[trial["trialName"]]; + if (existingEntry != trial) { + this.trialData[trial["trialName"]] = trial; + updatePoints = true; + } + } + } + + if (changes.annotationsCsv && this.annotationsCsv) { + let annotArr = d3.csv.parse(this.annotationsCsv); + for (let annot of annotArr) { + let newEntry: PbtAnnotation = { + trialName: annot["trialName"], + uid: annot["pbt.suggestion.katib.kubeflow.org/uid"], + parentUid: annot["pbt.suggestion.katib.kubeflow.org/parent"], + generation: parseInt(annot["pbt.suggestion.katib.kubeflow.org/generation"]), + }; + let existingEntry = this.annotData[newEntry.uid]; + if (existingEntry != newEntry) { + this.annotData[newEntry.uid] = newEntry; + updatePoints = true; + } + } + } + + if (updatePoints) { + // Lazy; reprocess all points + let points: PbtPoint[] = [] + Object.values(this.annotData).forEach(entry => { + let point = {} as PbtPoint; + point.uid = entry.uid; + point.generation = entry.generation; + point.parentUid = entry.parentUid; + point.trialName = entry.trialName; + + // Find corresponding trial data + let trial = this.trialData[point.trialName]; + if (trial !== undefined) { + point.metricValue = parseFloat(trial[this.goalName]); + point.parameters = {}; + for (let p of this.selectableNames) { + point.parameters[p] = this.graphHelper.yMeta[p].transform(trial[p]); + } + } else { + point.metricValue = undefined; + } + points.push(point); + }); + + // Generate segments + // Group seeds + let remaining_points = {} + for (let p of points) { + remaining_points[p.uid] = p; + } + + this.data = []; + while (Object.keys(remaining_points).length > 0) { + let seeds = this.maxGenerationTrials(remaining_points); + for (let seed of seeds) { + let segment = []; + let v = seed; + while (v) { + segment.push(v); + let delete_uid = v.uid; + v = remaining_points[v.parentUid]; + delete remaining_points[delete_uid]; + } + this.data.push(segment); + } + } + + this.clearGraph(); + this.updateGraph(); + } + + } + + private maxGenerationTrials(d) { + let end_seeds = []; + let max_generation = 0; + for (let k of Object.keys(d)) { + if (d[k].generation > max_generation) { + max_generation = d[k].generation; + end_seeds = []; + end_seeds.push(d[k]); + } else if (d[k].generation === max_generation) { + end_seeds.push(d[k]); + } + } + return end_seeds; + } + + private clearGraph() { + // Clear any existing views from graphs object + if (this.graph) { + // this.graph.remove(); // d3 remove from DOM + d3.select(this.graphWrapper.nativeElement).select("svg").remove(); + } + this.graph = undefined; + } + + private createGraph() { + this.graph = d3.select(this.graphWrapper.nativeElement) + .append("svg") + .attr("width", this.graphHelper.width + this.graphHelper.margin.left + this.graphHelper.margin.right) + .attr("height", this.graphHelper.height + this.graphHelper.margin.top + this.graphHelper.margin.bottom) + .append("g") + .attr("transform", + "translate(" + this.graphHelper.margin.left + "," + this.graphHelper.margin.top + ")"); + } + + private getRangeX(){ + const xValues = Object.values(this.annotData).map(entry => { + return entry.generation; + }); + return d3.scale.linear() + .domain(d3.extent(xValues)) + .range([ 0, this.graphHelper.width ]); + } + + private getRangeY(key){ + if (this.selectableNames.includes(key)) { + const paramValues = Object.keys(this.trialData).map(trialName => { + return this.graphHelper.yMeta[key].transform(this.trialData[trialName][key]); + }); + + if (this.graphHelper.yMeta[key].isNumber) { + return d3.scale.linear() + .domain(d3.extent(paramValues)) + .range([ this.graphHelper.height, 0 ]); + } else { + paramValues.sort((a,b)=>a-b); + return d3.scale.ordinal() + .domain(paramValues) + .range([ this.graphHelper.height, 0 ]); + } + + } else { + console.error("Key(" + key + ") not found in y-axis list"); + } + } + + private getColorScaleZ() { + let values = []; + for (const segment of this.data) { + for (const point of segment) { + values.push(point.metricValue); + } + } + return d3.scale.linear() + .domain(d3.extent(values)) + .interpolate(d3.interpolateHcl) + .range([d3.rgb("#cfd8dc"), d3.rgb('#263238')]); + } + + private updateGraph() { + if (!this.graphHelper || !this.graphHelper.ngInit){ + // ngOnInit not called yet + return; + } + if (!this.data.length || this.data.length == 0) { + // Data not initialized + return; + } + if (!this.graph) { + this.createGraph(); + } + + // Add X axis + let xAxis = this.getRangeX(); + this.graph.append("g") + .attr("transform", "translate(0," + this.graphHelper.height + ")") + .call(d3.svg.axis().scale(xAxis).orient("bottom")); + this.graph.append("text") + .attr("text-anchor", "middle") + .attr("x", this.graphHelper.width/2) + .attr("y", this.graphHelper.height + 30) + .text("Generation"); + // Add Y axis + let yAxis = this.getRangeY(this.selectedName); + this.graph.append("g") + .call(d3.svg.axis().scale(yAxis).orient("left")); + this.graph.append("text") + .attr("text-anchor", "middle") + .attr("x", -this.graphHelper.height/2) + .attr("y", -50) + .attr("transform", "rotate(-90)") + .text(this.selectedName); + // Change line width + this.graph.selectAll('path').style({ 'stroke': 'black', 'fill': 'none', 'stroke-width': '1px'}); + + // Add the points + const sparam = this.selectedName; + const colorScale = this.getColorScaleZ(); + for (const segment of this.data) { + // Plot only valid points + const validSegment = segment.filter(point => point.parameters && point.parameters[sparam] && point.metricValue !== undefined); + this.graph + .append("g") + .selectAll("dot") + .data(validSegment) + .enter() + .append("circle") + .attr("cx", function(d) { return xAxis(d.generation) } ) + .attr("cy", function(d) { return yAxis(d.parameters[sparam]) } ) + .attr("r", 2) + .attr("fill", function(d) { return colorScale(d.metricValue); } ); + + if (this.displayTrace && validSegment.length > 0) { + // Add the lines + const strokeColor = colorScale(validSegment[0].metricValue); + this.graph + .append("path") + .datum(validSegment) + .attr("fill", "none") + .attr("stroke", strokeColor) + .attr("stroke-width", 1) + .attr("d", d3.svg.line() + .x(function(d) { return xAxis(d.generation) } ) + .y(function(d) { return yAxis(d.parameters[sparam]) } ) + ); + } + } + } + +} From a4ec05f512b07b1431bc4c7ec45559ec49e3e9e6 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Wed, 4 May 2022 22:35:51 -0700 Subject: [PATCH 4/7] chore: prettier run across PBT changes --- .../pbt/pbt-tab-loader.module.ts | 8 +- .../pbt/pbt-tab.component.html | 13 +- .../pbt/pbt-tab.component.scss | 2 +- .../pbt/pbt-tab.component.ts | 271 +++++++++++------- .../src/app/services/backend.service.ts | 5 +- pkg/new-ui/v1beta1/hp.go | 2 +- pkg/new-ui/v1beta1/util.go | 2 +- 7 files changed, 193 insertions(+), 110 deletions(-) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts index 2906abffe75..0b45f713e29 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab-loader.module.ts @@ -10,7 +10,13 @@ import { PbtTabComponent } from './pbt-tab.component'; @NgModule({ declarations: [PbtTabComponent], - imports: [CommonModule, FormsModule, MatFormFieldModule, MatSelectModule, MatCheckboxModule], + imports: [ + CommonModule, + FormsModule, + MatFormFieldModule, + MatSelectModule, + MatCheckboxModule, + ], exports: [PbtTabComponent], }) export class PbtTabModule {} diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html index 8f4cdbbcc10..f9627740ca0 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.html @@ -1,18 +1,21 @@
-
Y-Axis - + - {{name}} + {{ name }} - Display Seed Traces + Display Seed Traces
-
diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss index 89c19105543..47bac1bf984 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.scss @@ -17,7 +17,7 @@ } .pbt-option { - margin: 10px + margin: 10px; } .d3-tab-graph { diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts index 1f4aacfd5eb..6151c736258 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts @@ -19,12 +19,12 @@ import { DOCUMENT } from '@angular/common'; declare let d3: any; type PbtPoint = { - uid: string; - parentUid: string; - trialName: string; - parameters: Object; // all y-axis possible values (parameters + alternativeMetrics) - generation: number; // generation - metricValue: number; // evaluation metric + uid: string; + parentUid: string; + trialName: string; + parameters: Object; // all y-axis possible values (parameters + alternativeMetrics) + generation: number; // generation + metricValue: number; // evaluation metric }; type PbtAnnotation = { @@ -32,7 +32,7 @@ type PbtAnnotation = { parentUid: string; generation: number; trialName: string; -} +}; @Component({ selector: 'app-experiment-pbt-tab', @@ -41,17 +41,17 @@ type PbtAnnotation = { changeDetection: ChangeDetectionStrategy.OnPush, }) export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { - @ViewChild('pbtGraph') graphWrapper:ElementRef; // used to manipulate svg dom + @ViewChild('pbtGraph') graphWrapper: ElementRef; // used to manipulate svg dom graph: any; // svg dom selectableNames: string[]; // list of parameters/metrics for UI dropdown selectedName: string; // user selected parameter/metric displayTrace: boolean; - private annotData: {[uid: string]: PbtAnnotation} = {}; + private annotData: { [uid: string]: PbtAnnotation } = {}; private trialData = {}; // primary key: trialName private parameterNames: string[]; // parameter names - private goalName: string = ""; + private goalName: string = ''; private data: PbtPoint[][] = []; // data sorted by generation and segment private graphHelper: any; // graph metadata and auxiliary info @@ -66,15 +66,17 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { constructor(@Inject(DOCUMENT) private document: Document) { this.graphHelper = {}; - this.graphHelper.margin = {top: 10, right: 30, bottom: 30, left: 60}; - this.graphHelper.width = 460 - this.graphHelper.margin.left - this.graphHelper.margin.right; - this.graphHelper.height = 400 - this.graphHelper.margin.top - this.graphHelper.margin.bottom; + this.graphHelper.margin = { top: 10, right: 30, bottom: 30, left: 60 }; + this.graphHelper.width = + 460 - this.graphHelper.margin.left - this.graphHelper.margin.right; + this.graphHelper.height = + 400 - this.graphHelper.margin.top - this.graphHelper.margin.bottom; // Track angular initialization since using @Input from template this.graphHelper.ngInit = false; } ngOnInit(): void { - if (this.experiment.spec.algorithm.algorithmName != "pbt") { + if (this.experiment.spec.algorithm.algorithmName != 'pbt') { // Prevent initialization if not Pbt return; } @@ -87,47 +89,72 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { this.goalName = this.experiment.spec.objective.objectiveMetricName; // Create full list of selectable names this.selectableNames = [...this.parameterNames]; - if (this.experiment.spec.objective.additionalMetricNames && this.experiment.spec.objective.additionalMetricNames.length > 0) { - this.selectableNames = [...this.selectableNames, ...this.experiment.spec.objective.additionalMetricNames]; + if ( + this.experiment.spec.objective.additionalMetricNames && + this.experiment.spec.objective.additionalMetricNames.length > 0 + ) { + this.selectableNames = [ + ...this.selectableNames, + ...this.experiment.spec.objective.additionalMetricNames, + ]; } // Create converters for all possible y-axes this.graphHelper.yMeta = {}; for (const param of this.experiment.spec.parameters) { this.graphHelper.yMeta[param.name] = {}; - if (param.parameterType == "discrete" || param.parameterType == "categorical") { - this.graphHelper.yMeta[param.name].transform = (x) => { return x; }; + if ( + param.parameterType == 'discrete' || + param.parameterType == 'categorical' + ) { + this.graphHelper.yMeta[param.name].transform = x => { + return x; + }; this.graphHelper.yMeta[param.name].isNumber = false; - } else if (param.parameterType == "double") { - this.graphHelper.yMeta[param.name].transform = (x) => { return parseFloat(x); }; + } else if (param.parameterType == 'double') { + this.graphHelper.yMeta[param.name].transform = x => { + return parseFloat(x); + }; this.graphHelper.yMeta[param.name].isNumber = true; } else { - this.graphHelper.yMeta[param.name].transform = (x) => { return parseInt(x); }; + this.graphHelper.yMeta[param.name].transform = x => { + return parseInt(x); + }; this.graphHelper.yMeta[param.name].isNumber = true; } } if (this.experiment.spec.objective.additionalMetricNames) { - for (const metricName of this.experiment.spec.objective.additionalMetricNames) { - if (this.graphHelper.yMeta.hasOwnProperty(metricName)){ - console.warn("Additional metric name conflict with parameter name; ignoring metric:", metricName); + for (const metricName of this.experiment.spec.objective + .additionalMetricNames) { + if (this.graphHelper.yMeta.hasOwnProperty(metricName)) { + console.warn( + 'Additional metric name conflict with parameter name; ignoring metric:', + metricName, + ); continue; } - this.graphHelper.yMeta[metricName].transform = (x) => { return parseFloat(x); }; + this.graphHelper.yMeta[metricName].transform = x => { + return parseFloat(x); + }; this.graphHelper.yMeta[metricName].isNumber = true; } } - + this.graphHelper.ngInit = true; } ngAfterViewInit(): void { - if (this.experiment.spec.algorithm.algorithmName != "pbt") { + if (this.experiment.spec.algorithm.algorithmName != 'pbt') { // Remove pbt tab and tab content - const tabs = document.querySelectorAll(".mat-tab-labels .mat-tab-label"); + const tabs = document.querySelectorAll('.mat-tab-labels .mat-tab-label'); for (let i = 0; i < tabs.length; i++) { - if (tabs[i].querySelector(".mat-tab-label-content").innerHTML.includes("PBT")){ - const tabId = tabs[i].getAttribute("id"); - const tabBodyId = tabId.replace("label", "content") - const tabBody = document.querySelector("#" + tabBodyId); + if ( + tabs[i] + .querySelector('.mat-tab-label-content') + .innerHTML.includes('PBT') + ) { + const tabId = tabs[i].getAttribute('id'); + const tabBodyId = tabId.replace('label', 'content'); + const tabBody = document.querySelector('#' + tabBodyId); tabBody.remove(); tabs[i].remove(); break; @@ -141,13 +168,13 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { this.displayTrace = false; } - onDropdownChange(){ + onDropdownChange() { // Trigger graph redraw on dropdown change event this.clearGraph(); this.updateGraph(); } - onTraceChange(){ + onTraceChange() { // Trigger graph redraw on trace change event // TODO: could use d3.select(..).remove() instead of recreating this.clearGraph(); @@ -164,9 +191,9 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { if (changes.experimentTrialsCsv && this.experimentTrialsCsv) { let trialArr = d3.csv.parse(this.experimentTrialsCsv); for (let trial of trialArr) { - let existingEntry = this.trialData[trial["trialName"]]; + let existingEntry = this.trialData[trial['trialName']]; if (existingEntry != trial) { - this.trialData[trial["trialName"]] = trial; + this.trialData[trial['trialName']] = trial; updatePoints = true; } } @@ -176,10 +203,12 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { let annotArr = d3.csv.parse(this.annotationsCsv); for (let annot of annotArr) { let newEntry: PbtAnnotation = { - trialName: annot["trialName"], - uid: annot["pbt.suggestion.katib.kubeflow.org/uid"], - parentUid: annot["pbt.suggestion.katib.kubeflow.org/parent"], - generation: parseInt(annot["pbt.suggestion.katib.kubeflow.org/generation"]), + trialName: annot['trialName'], + uid: annot['pbt.suggestion.katib.kubeflow.org/uid'], + parentUid: annot['pbt.suggestion.katib.kubeflow.org/parent'], + generation: parseInt( + annot['pbt.suggestion.katib.kubeflow.org/generation'], + ), }; let existingEntry = this.annotData[newEntry.uid]; if (existingEntry != newEntry) { @@ -191,7 +220,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { if (updatePoints) { // Lazy; reprocess all points - let points: PbtPoint[] = [] + let points: PbtPoint[] = []; Object.values(this.annotData).forEach(entry => { let point = {} as PbtPoint; point.uid = entry.uid; @@ -215,7 +244,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { // Generate segments // Group seeds - let remaining_points = {} + let remaining_points = {}; for (let p of points) { remaining_points[p.uid] = p; } @@ -239,7 +268,6 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { this.clearGraph(); this.updateGraph(); } - } private maxGenerationTrials(d) { @@ -261,49 +289,70 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { // Clear any existing views from graphs object if (this.graph) { // this.graph.remove(); // d3 remove from DOM - d3.select(this.graphWrapper.nativeElement).select("svg").remove(); + d3.select(this.graphWrapper.nativeElement).select('svg').remove(); } this.graph = undefined; } private createGraph() { - this.graph = d3.select(this.graphWrapper.nativeElement) - .append("svg") - .attr("width", this.graphHelper.width + this.graphHelper.margin.left + this.graphHelper.margin.right) - .attr("height", this.graphHelper.height + this.graphHelper.margin.top + this.graphHelper.margin.bottom) - .append("g") - .attr("transform", - "translate(" + this.graphHelper.margin.left + "," + this.graphHelper.margin.top + ")"); + this.graph = d3 + .select(this.graphWrapper.nativeElement) + .append('svg') + .attr( + 'width', + this.graphHelper.width + + this.graphHelper.margin.left + + this.graphHelper.margin.right, + ) + .attr( + 'height', + this.graphHelper.height + + this.graphHelper.margin.top + + this.graphHelper.margin.bottom, + ) + .append('g') + .attr( + 'transform', + 'translate(' + + this.graphHelper.margin.left + + ',' + + this.graphHelper.margin.top + + ')', + ); } - private getRangeX(){ + private getRangeX() { const xValues = Object.values(this.annotData).map(entry => { return entry.generation; }); - return d3.scale.linear() - .domain(d3.extent(xValues)) - .range([ 0, this.graphHelper.width ]); + return d3.scale + .linear() + .domain(d3.extent(xValues)) + .range([0, this.graphHelper.width]); } - private getRangeY(key){ + private getRangeY(key) { if (this.selectableNames.includes(key)) { const paramValues = Object.keys(this.trialData).map(trialName => { - return this.graphHelper.yMeta[key].transform(this.trialData[trialName][key]); + return this.graphHelper.yMeta[key].transform( + this.trialData[trialName][key], + ); }); if (this.graphHelper.yMeta[key].isNumber) { - return d3.scale.linear() + return d3.scale + .linear() .domain(d3.extent(paramValues)) - .range([ this.graphHelper.height, 0 ]); + .range([this.graphHelper.height, 0]); } else { - paramValues.sort((a,b)=>a-b); - return d3.scale.ordinal() + paramValues.sort((a, b) => a - b); + return d3.scale + .ordinal() .domain(paramValues) - .range([ this.graphHelper.height, 0 ]); + .range([this.graphHelper.height, 0]); } - } else { - console.error("Key(" + key + ") not found in y-axis list"); + console.error('Key(' + key + ') not found in y-axis list'); } } @@ -314,14 +363,15 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { values.push(point.metricValue); } } - return d3.scale.linear() + return d3.scale + .linear() .domain(d3.extent(values)) .interpolate(d3.interpolateHcl) - .range([d3.rgb("#cfd8dc"), d3.rgb('#263238')]); + .range([d3.rgb('#cfd8dc'), d3.rgb('#263238')]); } private updateGraph() { - if (!this.graphHelper || !this.graphHelper.ngInit){ + if (!this.graphHelper || !this.graphHelper.ngInit) { // ngOnInit not called yet return; } @@ -335,59 +385,80 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { // Add X axis let xAxis = this.getRangeX(); - this.graph.append("g") - .attr("transform", "translate(0," + this.graphHelper.height + ")") - .call(d3.svg.axis().scale(xAxis).orient("bottom")); - this.graph.append("text") - .attr("text-anchor", "middle") - .attr("x", this.graphHelper.width/2) - .attr("y", this.graphHelper.height + 30) - .text("Generation"); + this.graph + .append('g') + .attr('transform', 'translate(0,' + this.graphHelper.height + ')') + .call(d3.svg.axis().scale(xAxis).orient('bottom')); + this.graph + .append('text') + .attr('text-anchor', 'middle') + .attr('x', this.graphHelper.width / 2) + .attr('y', this.graphHelper.height + 30) + .text('Generation'); // Add Y axis let yAxis = this.getRangeY(this.selectedName); - this.graph.append("g") - .call(d3.svg.axis().scale(yAxis).orient("left")); - this.graph.append("text") - .attr("text-anchor", "middle") - .attr("x", -this.graphHelper.height/2) - .attr("y", -50) - .attr("transform", "rotate(-90)") + this.graph.append('g').call(d3.svg.axis().scale(yAxis).orient('left')); + this.graph + .append('text') + .attr('text-anchor', 'middle') + .attr('x', -this.graphHelper.height / 2) + .attr('y', -50) + .attr('transform', 'rotate(-90)') .text(this.selectedName); // Change line width - this.graph.selectAll('path').style({ 'stroke': 'black', 'fill': 'none', 'stroke-width': '1px'}); + this.graph + .selectAll('path') + .style({ stroke: 'black', fill: 'none', 'stroke-width': '1px' }); // Add the points const sparam = this.selectedName; const colorScale = this.getColorScaleZ(); for (const segment of this.data) { // Plot only valid points - const validSegment = segment.filter(point => point.parameters && point.parameters[sparam] && point.metricValue !== undefined); + const validSegment = segment.filter( + point => + point.parameters && + point.parameters[sparam] && + point.metricValue !== undefined, + ); this.graph - .append("g") - .selectAll("dot") + .append('g') + .selectAll('dot') .data(validSegment) .enter() - .append("circle") - .attr("cx", function(d) { return xAxis(d.generation) } ) - .attr("cy", function(d) { return yAxis(d.parameters[sparam]) } ) - .attr("r", 2) - .attr("fill", function(d) { return colorScale(d.metricValue); } ); + .append('circle') + .attr('cx', function (d) { + return xAxis(d.generation); + }) + .attr('cy', function (d) { + return yAxis(d.parameters[sparam]); + }) + .attr('r', 2) + .attr('fill', function (d) { + return colorScale(d.metricValue); + }); if (this.displayTrace && validSegment.length > 0) { // Add the lines const strokeColor = colorScale(validSegment[0].metricValue); this.graph - .append("path") + .append('path') .datum(validSegment) - .attr("fill", "none") - .attr("stroke", strokeColor) - .attr("stroke-width", 1) - .attr("d", d3.svg.line() - .x(function(d) { return xAxis(d.generation) } ) - .y(function(d) { return yAxis(d.parameters[sparam]) } ) - ); + .attr('fill', 'none') + .attr('stroke', strokeColor) + .attr('stroke-width', 1) + .attr( + 'd', + d3.svg + .line() + .x(function (d) { + return xAxis(d.generation); + }) + .y(function (d) { + return yAxis(d.parameters[sparam]); + }), + ); } } } - } diff --git a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts index 7bec30ecf31..c33ffc6199b 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts @@ -56,7 +56,10 @@ export class KWABackendService extends BackendService { return this.http.get(url).pipe(catchError(error => this.parseError(error))); } - getExperimentAnnotationsInfo(name: string, namespace: string): Observable { + getExperimentAnnotationsInfo( + name: string, + namespace: string, + ): Observable { const url = `/katib/fetch_hp_job_annotation_info/?experimentName=${name}&namespace=${namespace}`; return this.http.get(url).pipe(catchError(error => this.parseError(error))); diff --git a/pkg/new-ui/v1beta1/hp.go b/pkg/new-ui/v1beta1/hp.go index c882a1bded1..c621c1873e9 100644 --- a/pkg/new-ui/v1beta1/hp.go +++ b/pkg/new-ui/v1beta1/hp.go @@ -271,7 +271,7 @@ func (k *KatibUIHandler) FetchHPJobAnnotationInfo(w http.ResponseWriter, r *http // Set the index of each annotation a_index := 0 - for a, _ := range annotations { + for a := range annotations { annotations[a] = a_index resultText += "," + a a_index++ diff --git a/pkg/new-ui/v1beta1/util.go b/pkg/new-ui/v1beta1/util.go index 195d54bf58e..8b54ac214e5 100644 --- a/pkg/new-ui/v1beta1/util.go +++ b/pkg/new-ui/v1beta1/util.go @@ -65,7 +65,7 @@ func (k *KatibUIHandler) getExperiments(namespace []string) ([]ExperimentView, e return experiments, nil } -func hasAnnotation(trials []trialv1beta1.Trial, annotation string) bool{ +func hasAnnotation(trials []trialv1beta1.Trial, annotation string) bool { for _, t := range trials { _, ok := t.Spec.Annotations[annotation] if ok { From 6145922bc42d8df7d4bc216f047071b596846fb7 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Wed, 4 May 2022 18:44:45 -0700 Subject: [PATCH 5/7] feedback: change suggestion mutation mount variable name --- .../frontend/src/app/constants/algorithms-settings.const.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts index 243cd321e4a..b34b2b8c01b 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts @@ -244,7 +244,7 @@ export const DartsSettings: AlgorithmSetting[] = [ export const PbtSettings: AlgorithmSetting[] = [ { - name: 'checkpoint_dir', + name: 'suggestion_trial_dir', value: '/var/log/katib/checkpoints/', type: AlgorithmSettingType.STRING, }, From cc32156f059dc94d30ffcd25219d120cd4f13035 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Sun, 8 May 2022 16:05:25 -0700 Subject: [PATCH 6/7] feedback: remove uid annotation and fix issue with additional metrics --- .../src/app/pages/experiment-details/pbt/pbt-tab.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts index 6151c736258..3fbd1f2f606 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts @@ -132,6 +132,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { ); continue; } + this.graphHelper.yMeta[metricName] = {}; this.graphHelper.yMeta[metricName].transform = x => { return parseFloat(x); }; @@ -204,7 +205,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { for (let annot of annotArr) { let newEntry: PbtAnnotation = { trialName: annot['trialName'], - uid: annot['pbt.suggestion.katib.kubeflow.org/uid'], + uid: annot['trialName'], parentUid: annot['pbt.suggestion.katib.kubeflow.org/parent'], generation: parseInt( annot['pbt.suggestion.katib.kubeflow.org/generation'], From 5ca9a9752a65d0b62d6f9862670ab4a6fca7fb67 Mon Sep 17 00:00:00 2001 From: a9p <58503488+a9p@users.noreply.github.com> Date: Sun, 19 Jun 2022 01:53:29 -0400 Subject: [PATCH 7/7] feedback: read metadata from labels and improve label query backend performance. --- cmd/new-ui/v1beta1/main.go | 2 +- .../experiment-details.component.html | 2 +- .../experiment-details.component.ts | 6 +- .../pbt/pbt-tab.component.ts | 70 +++++++++---------- .../src/app/services/backend.service.ts | 7 +- pkg/new-ui/v1beta1/hp.go | 53 +++++++------- pkg/new-ui/v1beta1/util.go | 4 +- 7 files changed, 70 insertions(+), 74 deletions(-) diff --git a/cmd/new-ui/v1beta1/main.go b/cmd/new-ui/v1beta1/main.go index bc349364362..6c94e1605b3 100644 --- a/cmd/new-ui/v1beta1/main.go +++ b/cmd/new-ui/v1beta1/main.go @@ -59,7 +59,7 @@ func main() { http.HandleFunc("/katib/fetch_hp_job_info/", kuh.FetchHPJobInfo) http.HandleFunc("/katib/fetch_hp_job_trial_info/", kuh.FetchHPJobTrialInfo) - http.HandleFunc("/katib/fetch_hp_job_annotation_info/", kuh.FetchHPJobAnnotationInfo) + http.HandleFunc("/katib/fetch_hp_job_label_info/", kuh.FetchHPJobLabelInfo) http.HandleFunc("/katib/fetch_nas_job_info/", kuh.FetchNASJobInfo) http.HandleFunc("/katib/fetch_trial_templates/", kuh.FetchTrialTemplates) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html index 68d177a347a..e650ccee888 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.html @@ -63,7 +63,7 @@ diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts index 4b0605b86e4..f4e4889f5d4 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/experiment-details.component.ts @@ -30,7 +30,7 @@ export class ExperimentDetailsComponent implements OnInit, OnDestroy { columns: string[] = []; details: string[][] = []; experimentTrialsCsv: string; - annotationsCsv: string; + labelCsv: string; hoveredTrial: number; experimentDetails: ExperimentK8s; showGraph: boolean; @@ -96,9 +96,9 @@ export class ExperimentDetailsComponent implements OnInit, OnDestroy { this.showGraph = true; }); this.backendService - .getExperimentAnnotationsInfo(this.name, this.namespace) + .getExperimentLabelInfo(this.name, this.namespace) .subscribe(response => { - this.annotationsCsv = response; + this.labelCsv = response; }); this.backendService .getExperiment(this.name, this.namespace) diff --git a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts index 3fbd1f2f606..c2f6172f37b 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/pages/experiment-details/pbt/pbt-tab.component.ts @@ -16,24 +16,18 @@ import { ExperimentK8s } from '../../../models/experiment.k8s.model'; import { Inject } from '@angular/core'; import { DOCUMENT } from '@angular/common'; +import { StatusEnum } from 'src/app/enumerations/status.enum'; + declare let d3: any; type PbtPoint = { - uid: string; - parentUid: string; trialName: string; + parentUid: string; parameters: Object; // all y-axis possible values (parameters + alternativeMetrics) generation: number; // generation metricValue: number; // evaluation metric }; -type PbtAnnotation = { - uid: string; - parentUid: string; - generation: number; - trialName: string; -}; - @Component({ selector: 'app-experiment-pbt-tab', templateUrl: './pbt-tab.component.html', @@ -48,8 +42,8 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { selectedName: string; // user selected parameter/metric displayTrace: boolean; - private annotData: { [uid: string]: PbtAnnotation } = {}; - private trialData = {}; // primary key: trialName + private labelData: { [trialName: string]: PbtPoint } = {}; + private trialData: { [trialName: string]: Object } = {}; private parameterNames: string[]; // parameter names private goalName: string = ''; private data: PbtPoint[][] = []; // data sorted by generation and segment @@ -59,7 +53,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { experiment: ExperimentK8s; @Input() - annotationsCsv: string[] = []; + labelCsv: string[] = []; @Input() experimentTrialsCsv: string[] = []; @@ -184,50 +178,54 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { ngOnChanges(changes: SimpleChanges): void { if (!this.graphHelper || !this.graphHelper.ngInit) { - // Wait for view to properly initialize - return; + console.warn( + 'graphHelper not initialized yet, attempting manual call to ngOnInit()', + ); + this.ngOnInit(); } // Recompute formatted plotting points on data input changes let updatePoints = false; if (changes.experimentTrialsCsv && this.experimentTrialsCsv) { let trialArr = d3.csv.parse(this.experimentTrialsCsv); for (let trial of trialArr) { - let existingEntry = this.trialData[trial['trialName']]; - if (existingEntry != trial) { + if ( + trial['Status'] == StatusEnum.SUCCEEDED && + !this.trialData.hasOwnProperty(trial['trialName']) + ) { this.trialData[trial['trialName']] = trial; updatePoints = true; } } } - if (changes.annotationsCsv && this.annotationsCsv) { - let annotArr = d3.csv.parse(this.annotationsCsv); - for (let annot of annotArr) { - let newEntry: PbtAnnotation = { - trialName: annot['trialName'], - uid: annot['trialName'], - parentUid: annot['pbt.suggestion.katib.kubeflow.org/parent'], + if (changes.labelCsv && this.labelCsv) { + let labelArr = d3.csv.parse(this.labelCsv); + for (let label of labelArr) { + if (this.labelData.hasOwnProperty(label['trialName'])) { + continue; + } + let newEntry: PbtPoint = { + trialName: label['trialName'], + parentUid: label['pbt.suggestion.katib.kubeflow.org/parent'], generation: parseInt( - annot['pbt.suggestion.katib.kubeflow.org/generation'], + label['pbt.suggestion.katib.kubeflow.org/generation'], ), + parameters: undefined, + metricValue: undefined, }; - let existingEntry = this.annotData[newEntry.uid]; - if (existingEntry != newEntry) { - this.annotData[newEntry.uid] = newEntry; - updatePoints = true; - } + this.labelData[newEntry.trialName] = newEntry; + updatePoints = true; } } if (updatePoints) { // Lazy; reprocess all points let points: PbtPoint[] = []; - Object.values(this.annotData).forEach(entry => { + Object.values(this.labelData).forEach(entry => { let point = {} as PbtPoint; - point.uid = entry.uid; + point.trialName = entry.trialName; point.generation = entry.generation; point.parentUid = entry.parentUid; - point.trialName = entry.trialName; // Find corresponding trial data let trial = this.trialData[point.trialName]; @@ -247,7 +245,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { // Group seeds let remaining_points = {}; for (let p of points) { - remaining_points[p.uid] = p; + remaining_points[p.trialName] = p; } this.data = []; @@ -258,9 +256,9 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { let v = seed; while (v) { segment.push(v); - let delete_uid = v.uid; + let delete_entry = v.trialName; v = remaining_points[v.parentUid]; - delete remaining_points[delete_uid]; + delete remaining_points[delete_entry]; } this.data.push(segment); } @@ -323,7 +321,7 @@ export class PbtTabComponent implements OnChanges, OnInit, AfterViewInit { } private getRangeX() { - const xValues = Object.values(this.annotData).map(entry => { + const xValues = Object.values(this.labelData).map(entry => { return entry.generation; }); return d3.scale diff --git a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts index c33ffc6199b..458d26b17e5 100644 --- a/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts +++ b/pkg/new-ui/v1beta1/frontend/src/app/services/backend.service.ts @@ -56,11 +56,8 @@ export class KWABackendService extends BackendService { return this.http.get(url).pipe(catchError(error => this.parseError(error))); } - getExperimentAnnotationsInfo( - name: string, - namespace: string, - ): Observable { - const url = `/katib/fetch_hp_job_annotation_info/?experimentName=${name}&namespace=${namespace}`; + getExperimentLabelInfo(name: string, namespace: string): Observable { + const url = `/katib/fetch_hp_job_label_info/?experimentName=${name}&namespace=${namespace}`; return this.http.get(url).pipe(catchError(error => this.parseError(error))); } diff --git a/pkg/new-ui/v1beta1/hp.go b/pkg/new-ui/v1beta1/hp.go index c621c1873e9..9f3fb7a8502 100644 --- a/pkg/new-ui/v1beta1/hp.go +++ b/pkg/new-ui/v1beta1/hp.go @@ -72,13 +72,13 @@ func (k *KatibUIHandler) FetchHPJobInfo(w http.ResponseWriter, r *http.Request) log.Printf("Got Trial List") // append a column for the Pipeline UID associated with the Trial - if hasAnnotation(trialList.Items, kfpRunIDAnnotation) { + if havePipelineUID(trialList.Items) { resultText += ",KFP Run" } foundPipelineUID := false for _, t := range trialList.Items { - runUid, ok := t.Spec.Annotations[kfpRunIDAnnotation] + runUid, ok := t.GetAnnotations()[kfpRunIDAnnotation] if !ok { log.Printf("Trial %s has no pipeline run.", t.Name) runUid = "" @@ -246,13 +246,11 @@ func (k *KatibUIHandler) FetchHPJobTrialInfo(w http.ResponseWriter, r *http.Requ } } -func (k *KatibUIHandler) FetchHPJobAnnotationInfo(w http.ResponseWriter, r *http.Request) { +func (k *KatibUIHandler) FetchHPJobLabelInfo(w http.ResponseWriter, r *http.Request) { //enableCors(&w) experimentName := r.URL.Query()["experimentName"][0] namespace := r.URL.Query()["namespace"][0] - resultText := "trialName" - trialList, err := k.katibClient.GetTrialList(experimentName, namespace) if err != nil { log.Printf("GetTrialList from HP job failed: %v", err) @@ -261,34 +259,37 @@ func (k *KatibUIHandler) FetchHPJobAnnotationInfo(w http.ResponseWriter, r *http } log.Printf("Got Trial List") - // Find set(annotations) - annotations := map[string]int{} + labelList := map[string]int{} + labelList["trialName"] = 0 + var trialRes [][]string for _, t := range trialList.Items { - for a := range t.Spec.Annotations { - annotations[a] = 0 + trialResText := make([]string, len(labelList)) + trialResText[labelList["trialName"]] = t.Name + for k, v := range t.ObjectMeta.Labels { + i, exists := labelList[k] + if !exists { + i = len(labelList) + labelList[k] = i + trialResText = append(trialResText, "") + } + trialResText[i] = v } + trialRes = append(trialRes, trialResText) } - // Set the index of each annotation - a_index := 0 - for a := range annotations { - annotations[a] = a_index - resultText += "," + a - a_index++ + // Format header output + headerArr := make([]string, len(labelList)) + for k, v := range labelList { + headerArr[v] = k } - log.Printf("Got Annotations List") + resultText := strings.Join(headerArr, ",") - for _, t := range trialList.Items { - trialResText := make([]string, len(annotations)) - for a, n := range annotations { - a_value, ok := t.Spec.Annotations[a] - if !ok { - log.Printf("Trial %s has no annotation value for %s", t.Name, a) - a_value = "" - } - trialResText[n] = a_value + // Format entry output + for _, row := range trialRes { + resultText += "\n" + strings.Join(row, ",") + for j := 0; j < len(labelList)-len(row); j++ { + resultText += "," } - resultText += "\n" + t.Name + "," + strings.Join(trialResText, ",") } log.Printf("Logs parsed, results:\n %v", resultText) diff --git a/pkg/new-ui/v1beta1/util.go b/pkg/new-ui/v1beta1/util.go index 8b54ac214e5..c5d6c5cf13c 100644 --- a/pkg/new-ui/v1beta1/util.go +++ b/pkg/new-ui/v1beta1/util.go @@ -65,9 +65,9 @@ func (k *KatibUIHandler) getExperiments(namespace []string) ([]ExperimentView, e return experiments, nil } -func hasAnnotation(trials []trialv1beta1.Trial, annotation string) bool { +func havePipelineUID(trials []trialv1beta1.Trial) bool { for _, t := range trials { - _, ok := t.Spec.Annotations[annotation] + _, ok := t.GetAnnotations()[kfpRunIDAnnotation] if ok { return true }