Merge pull request #292 from labmlai/default-chart

Default chart

Author: lakshith-403

app/server/labml_app/analyses/experiments/custom_metrics.py

@@ -3,12 +3,15 @@
 from typing import List, Tuple, Any
 from uuid import UUID
 
+from labml_app.analyses.helper import get_similarity
+from labml_app.db import user, run
 from labml_db import Model, Key, Index
 from labml_db.serializer.pickle import PickleSerializer
 from labml_db.serializer.yaml import YamlSerializer
 from fastapi import Request
 
 from labml_app.analyses.analysis import Analysis
+from labml_app.analyses.experiments.metrics import MetricsAnalysis
 from labml_app.analyses.preferences import MetricPreferenceModel
 
 
@@ -83,6 +86,10 @@ def create_custom_metric(self, data: dict):
         mp.save()
         cm.preference_key = mp.key
 
+        if 'preferences' in data:
+            mp.update_preferences(data['preferences'])
+            mp.save()
+
         cm.save()
         self.metrics.append((cm.metric_id, cm.key))
         self.save()
@@ -100,6 +107,9 @@ def delete_custom_metric(self, metric_id: str):
     def get_data(self):
         return [k[1].load().get_data() for k in self.metrics]
 
+    def get_metrics(self):
+        return [k[1].load() for k in self.metrics]
+
     def update(self, data: dict):
         for (metric_id, key) in self.metrics:
             if metric_id == data['id']:
@@ -115,9 +125,10 @@ async def get_custom_metrics(request: Request, run_uuid: str) -> Any:
         r = CustomMetricsListModel()
         r.save()
         CustomMetricsListIndex.set(run_uuid, r.key)
-    else:
-        r = list_key.load()
+        await create_magic_metric(request, run_uuid)
+        list_key = r.key
 
+    r = list_key.load()
     return {'metrics': r.get_data()}
 
 
@@ -167,3 +178,93 @@ async def delete_custom_metric(request: Request, run_uuid: str) -> Any:
     r.delete_custom_metric(data['id'])
 
     return {'status': 'success'}
+
+
+@Analysis.route('GET', 'custom_metrics/{run_uuid}/magic')
+async def create_magic_metric(request: Request, run_uuid: str) -> Any:
+    list_key = CustomMetricsListIndex.get(run_uuid)
+
+    run_cm = list_key.load()
+
+    current_run = run.get(run_uuid)
+    if current_run is None:
+        return {'is_success': False, 'message': 'Run not found'}
+    current_run = current_run.get_summary()
+
+    analysis_data = MetricsAnalysis.get_or_create(run_uuid).get_tracking()
+    indicators = [track_item['name'] for track_item in analysis_data]
+    indicators = sorted(indicators)
+
+    u = user.get_by_session_token('local')  # todo
+
+    default_project = u.default_project
+    runs = [r.get_summary() for r in default_project.get_runs()]
+
+    runs = sorted(runs, key=lambda i: i['start_time'], reverse=True)
+    similarity = [get_similarity(current_run, x) for x in runs]
+    runs = [x for s, x in sorted(zip(similarity, runs), key=lambda pair: pair[0], reverse=True)]
+    runs = runs[:20]
+
+    current_metrics = run_cm.get_data()
+    current_indicators = []
+    for x in current_metrics:
+        current_indicators += x['preferences']['series_preferences']
+    current_indicators = set(current_indicators)
+
+    indicator_counts = {}
+    for r in runs:
+        list_key = CustomMetricsListIndex.get(r['run_uuid'])
+        if list_key is None:
+            continue
+        cm = list_key.load()
+        cm = cm.get_metrics()
+
+        for m in cm:
+            m_data = m.get_data()
+            preferences = m_data['preferences']
+            if len(preferences['series_preferences']) == 0:
+                continue
+            has_current_indicators = False
+            for indicator in current_indicators:
+                if indicator in preferences['series_preferences']:
+                    has_current_indicators = True
+                    break
+            if has_current_indicators:
+                continue
+
+            preference_map_key = '|'.join(sorted(preferences['series_preferences']))
+            if preference_map_key not in indicator_counts:
+                indicator_counts[preference_map_key] = []
+            indicator_counts[preference_map_key].append((m.key, m_data['created_time']))
+
+    if len(indicator_counts) == 0:
+        return {'is_success': False, 'message': "Couldn't find any related past run."}
+
+    sorted_keys = sorted(indicator_counts.keys(), key=lambda x: len(indicator_counts[x]), reverse=True)
+
+    # find the first indicator list with overlap
+    selected = None
+    for key in sorted_keys:
+        ind = key.split('|')
+        overlap = False
+        for i in indicators:
+            if i in ind:
+                overlap = True
+                break
+        if overlap:
+            selected = key
+            break
+
+    if selected is None:  # return smth
+        return {'status': 'error', 'message': 'No similar indicators found'}
+
+    selected_metric = sorted(indicator_counts[selected], key=lambda x: x[1], reverse=True)[0]
+    selected_metric = selected_metric[0].load()
+
+    new_metric_data = selected_metric.get_data()
+    new_metric_data['preferences']['series_preferences'] = \
+        [x for x in new_metric_data['preferences']['series_preferences'] if x in indicators]
+
+    run_cm.create_custom_metric(new_metric_data)
+
+    return {'is_success': True}

app/server/labml_app/analyses/helper.py

@@ -51,3 +51,35 @@ def get_mean_series(res: List[Dict[str, Any]]) -> Dict[str, Any]:
     last_step = res[0]['last_step']
 
     return {'step': step, 'value': mean_value, 'name': 'mean', 'last_step': last_step}
+
+
+def edit_distance(str1, str2):
+    len1 = len(str1)
+    len2 = len(str2)
+
+    dp = [[0 for i in range(len1 + 1)] for j in range(2)]
+
+    for i in range(0, len1 + 1):
+        dp[0][i] = i
+
+    for i in range(1, len2 + 1):
+        for j in range(0, len1 + 1):
+            if j == 0:
+                dp[i % 2][j] = i
+            elif str1[j - 1] == str2[i - 1]:
+                dp[i % 2][j] = dp[(i - 1) % 2][j - 1]
+            else:
+                dp[i % 2][j] = (1 + min(dp[(i - 1) % 2][j],
+                                        min(dp[i % 2][j - 1],
+                                            dp[(i - 1) % 2][j - 1])))
+    return dp[len2 % 2][len1]
+
+
+def get_similarity(run_a, run_b):
+    name_edit = edit_distance(run_a['name'], run_b['name']) / max(len(run_a['name']), len(run_b['name']))
+
+    return 1 - name_edit
+
+
+
+

app/ui/src/network.ts

@@ -132,6 +132,10 @@ class Network {
         return this.sendHttpRequest('POST', `/${url}/preferences/${runUUID}`, data)['promise']
     }
 
+    async createMagicMetric(runUUID: string): Promise<any> {
+        return this.sendHttpRequest('GET', `/custom_metrics/${runUUID}/magic`, {})['promise']
+    }
+
     async createCustomMetric(runUUID: string, data: object): Promise<any> {
         return this.sendHttpRequest('POST', `/custom_metrics/${runUUID}/create`, data)['promise']
     }

app/ui/src/views/run_view.ts

@@ -4,7 +4,7 @@ import {User} from '../models/user'
 import {ROUTER, SCREEN} from '../app'
 import {Weya as $, WeyaElement} from '../../../lib/weya/weya'
 import {DataLoader} from "../components/loader"
-import {AddButton, BackButton, CustomButton, ExpandButton, ShareButton} from "../components/buttons"
+import {AddButton, BackButton, CustomButton, ExpandButton, IconButton, ShareButton} from "../components/buttons"
 import {UserMessages} from "../components/user_messages"
 import {RunHeaderCard} from "../analyses/experiments/run_header/card"
 import {experimentAnalyses} from "../analyses/analyses"
@@ -15,6 +15,7 @@ import {AwesomeRefreshButton} from '../components/refresh_button'
 import {setTitle} from '../utils/document'
 import {ScreenView} from '../screen_view'
 import metricAnalysis from "../analyses/experiments/custom_metrics";
+import NETWORK from "../network";
 
 class RunView extends ScreenView {
     uuid: string
@@ -37,6 +38,7 @@ class RunView extends ScreenView {
     private refresh: AwesomeRefreshButton
     private share: ShareButton
     private addCustomMetricButton: AddButton
+    private magicMetricButton: IconButton
     private isRankExpanded: boolean
     private rankElems: WeyaElement
     private processContainer: WeyaElement
@@ -69,6 +71,16 @@ class RunView extends ScreenView {
             title: 'Add custom metric',
             parent: this.constructor.name
         })
+        this.magicMetricButton = new IconButton({
+            onButtonClick: () => {
+                this.magicMetricButton.loading = true
+                this.creatMagicMetric().then(() => {
+                    this.magicMetricButton.loading = false
+                })
+            },
+            title: 'Add magic metric',
+            parent: this.constructor.name,
+        }, '.fas.fa-magic')
     }
 
     private get isRank(): boolean {
@@ -185,6 +197,7 @@ class RunView extends ScreenView {
                 }).render($)
             }
             this.addCustomMetricButton.render($)
+            this.magicMetricButton.render($)
         })
     }
 
@@ -194,6 +207,31 @@ class RunView extends ScreenView {
         }
     }
 
+    async creatMagicMetric() {
+        let res = null
+        try {
+            res = await NETWORK.createMagicMetric(this.uuid)
+        } catch (e) {
+            UserMessages.shared.networkError(e, 'Failed to create magic metric')
+            return
+        }
+
+
+        if (res['is_success']) {
+            try {
+                this.customMetrics = await CACHE.getCustomMetrics(this.uuid).get(true)
+            } catch (e) {
+                UserMessages.shared.networkError(e, 'Failed to load custom metrics')
+                return
+            }
+
+            this.renderCards()
+            UserMessages.shared.success('Chart created')
+        } else {
+            UserMessages.shared.warning(res['message'])
+        }
+    }
+
     async createCustomMetric() {
         this.addCustomMetricButton.loading = true
         try {