From 7d557c2070e31ed8effc82046db03bc960469d5d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 23 Apr 2024 22:48:13 +0200 Subject: [PATCH] feat: add label and schedule filters to runs page --- backend/windmill-api/openapi.yaml | 23 ++++ backend/windmill-api/src/db.rs | 8 ++ backend/windmill-api/src/jobs.rs | 26 +++- backend/windmill-common/src/jobs.rs | 2 + .../src/lib/components/runs/JobLoader.svelte | 22 +-- .../src/lib/components/runs/RunRow.svelte | 19 +++ .../src/lib/components/runs/RunsFilter.svelte | 75 +++++++++- .../src/lib/components/runs/RunsTable.svelte | 11 ++ .../(logged)/runs/[...path]/+page.svelte | 129 ++++++++++++------ 9 files changed, 248 insertions(+), 67 deletions(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fac75da495489..c56fc0c0b9f36 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5248,6 +5248,11 @@ paths: in: query schema: type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: "200": description: All queued jobs @@ -5333,6 +5338,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/OrderDesc" - $ref: "#/components/parameters/CreatedBy" + - $ref: "#/components/parameters/Label" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/ScriptExactPath" - $ref: "#/components/parameters/ScriptStartPath" @@ -5362,6 +5368,11 @@ paths: in: query schema: type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: "200": description: All completed jobs @@ -5381,6 +5392,7 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/CreatedBy" + - $ref: "#/components/parameters/Label" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/ScriptExactPath" - $ref: "#/components/parameters/ScriptStartPath" @@ -5423,6 +5435,11 @@ paths: in: query schema: type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: "200": description: All jobs @@ -8032,6 +8049,12 @@ components: in: query schema: type: string + Label: + name: label + description: mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string at key 'wm_label') + in: query + schema: + type: string ParentJob: name: parent_job description: diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 552960cc86f2b..f6fa96ba6b412 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -136,6 +136,14 @@ impl Migrate for CustomMigrator { migration.version, migration.description ); + if migration.version == 20240422144808 { + tracing::info!( + "Special migration to add index concurrently on completed_job labels" + ); + sqlx::query!( + "CREATE INDEX CONCURRENTLY labeled_jobs_on_completed_jobs ON completed_job USING GIN ((result -> 'wm_label')) WHERE result ? 'wm_label';" + ).execute(&mut *self.inner).await?; + } let r = self.inner.apply(migration).await; tracing::info!("Finished applying migration {}", migration.version); r diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index cf3a0189e2756..fb26b157b96c2 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -585,6 +585,7 @@ fn generate_get_job_query(no_logs: bool, table: &str) -> String { result, deleted, is_skipped, + result->>'wm_label' as label, CASE WHEN result is null or pg_column_size(result) < 2000000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result" } else { "scheduled_for, @@ -779,6 +780,8 @@ pub struct ListableCompletedJob { pub tag: String, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, } #[derive(Deserialize, Clone)] @@ -839,6 +842,7 @@ pub struct ListQueueQuery { pub all_workspaces: Option, pub is_flow_step: Option, pub has_null_parent: Option, + pub is_not_schedule: Option, } fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder { @@ -943,6 +947,10 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq sqlb.and_where_le("scheduled_for", "now()"); } + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("schedule_path IS null"); + } + sqlb } @@ -1171,13 +1179,14 @@ async fn list_jobs( "null as concurrent_limit", "null as concurrency_time_window_s", "priority", + "result->>'wm_label' as label", ], )) } else { None }; - let sql = if lq.success.is_none() { + let sql = if lq.success.is_none() && lq.label.is_none() { let sqlq = list_queue_jobs_query( &w_id, &ListQueueQuery { @@ -1203,6 +1212,7 @@ async fn list_jobs( all_workspaces: lq.all_workspaces, is_flow_step: lq.is_flow_step, has_null_parent: lq.has_null_parent, + is_not_schedule: lq.is_not_schedule, }, &[ "'QueuedJob' as typ", @@ -1236,6 +1246,7 @@ async fn list_jobs( "concurrent_limit", "concurrency_time_window_s", "priority", + "null as label", ], ); @@ -1968,6 +1979,7 @@ struct UnifiedJob { concurrent_limit: Option, concurrency_time_window_s: Option, priority: Option, + label: Option, } impl<'a> From for Job { @@ -2005,6 +2017,7 @@ impl<'a> From for Job { mem_peak: uj.mem_peak, tag: uj.tag, priority: uj.priority, + label: uj.label, }), "QueuedJob" => Job::QueuedJob(QueuedJob { workspace_id: uj.workspace_id, @@ -3732,6 +3745,14 @@ fn list_completed_jobs_query( sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); } + if let Some(label) = &lq.label { + sqlb.and_where("result->>'wm_label' = ?".bind(label)); + } + + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("schedule_path IS null"); + } + sqlb } #[derive(Deserialize, Clone)] @@ -3763,6 +3784,8 @@ pub struct ListCompletedQuery { pub scheduled_for_before_now: Option, pub all_workspaces: Option, pub has_null_parent: Option, + pub label: Option, + pub is_not_schedule: Option, } async fn list_completed_jobs( @@ -3810,6 +3833,7 @@ async fn list_completed_jobs( "mem_peak", "tag", "priority", + "result->>'wm_label' as label", "'CompletedJob' as type", ], ) diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index b6a4bb7b7b5a5..f15f73b2df935 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -243,6 +243,8 @@ pub struct CompletedJob { pub tag: String, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, } impl CompletedJob { diff --git a/frontend/src/lib/components/runs/JobLoader.svelte b/frontend/src/lib/components/runs/JobLoader.svelte index 1793df8cb553a..b15c79578851e 100644 --- a/frontend/src/lib/components/runs/JobLoader.svelte +++ b/frontend/src/lib/components/runs/JobLoader.svelte @@ -6,15 +6,15 @@ import { workspaceStore } from '$lib/stores' import { tweened, type Tweened } from 'svelte/motion' - import { forLater } from '$lib/forLater' export let jobs: Job[] | undefined export let user: string | null + export let label: string | null export let folder: string | null export let path: string | null export let success: 'success' | 'failure' | 'running' | undefined = undefined export let isSkipped: boolean = false - export let hideSchedules: boolean = false + export let showSchedules: boolean = false export let argFilter: string | undefined export let resultFilter: string | undefined = undefined export let schedulePath: string | undefined = undefined @@ -39,12 +39,13 @@ $: jobKinds = computeJobKinds(jobKindsCat) $: ($workspaceStore && loadJobsIntern(true)) || (path && + label && success && isSkipped != undefined && jobKinds && user && folder && - hideSchedules != undefined && + showSchedules != undefined && allWorkspaces != undefined && argFilter != undefined && resultFilter != undefined) @@ -107,6 +108,8 @@ running: success == 'running' ? true : undefined, isSkipped: isSkipped ? undefined : false, isFlowStep: jobKindsCat != 'all' ? false : undefined, + label: label === null || label === '' ? undefined : label, + isNotSchedule: showSchedules == false ? true : undefined, args: argFilter && argFilter != '{}' && argFilter != '' && argError == '' ? argFilter : undefined, result: @@ -141,12 +144,6 @@ try { jobs = await fetchJobs(maxTs, minTs) computeCompletedJobs() - - if (hideSchedules && !schedulePath) { - jobs = jobs.filter( - (job) => !(job && 'running' in job && job.scheduled_for && forLater(job.scheduled_for)) - ) - } } catch (err) { sendUserToast(`There was a problem fetching jobs: ${err}`, true) console.error(JSON.stringify(err)) @@ -215,13 +212,6 @@ .forEach((x) => (jobs![jobs?.findIndex((y) => y.id == x.id)!] = x)) jobs = jobs computeCompletedJobs() - - if (hideSchedules && !schedulePath) { - jobs = jobs.filter( - (job) => - !(job && 'running' in job && job.scheduled_for && forLater(job.scheduled_for)) - ) - } } loading = false diff --git a/frontend/src/lib/components/runs/RunRow.svelte b/frontend/src/lib/components/runs/RunRow.svelte index b85ee1c573338..93f95ee3b9142 100644 --- a/frontend/src/lib/components/runs/RunRow.svelte +++ b/frontend/src/lib/components/runs/RunRow.svelte @@ -28,6 +28,7 @@ export let job: Job export let selectedId: string | undefined = undefined export let containerWidth: number = 0 + export let containsLabel: boolean = false let scheduleEditor: ScheduleEditor @@ -170,6 +171,24 @@ {/if} {/if} + {#if containsLabel} +
+ {#if job && job?.['label']} +
+
{job?.['label']}
+ +
+ {/if} +
+ {/if}
{#if job && job.schedule_path}
diff --git a/frontend/src/lib/components/runs/RunsFilter.svelte b/frontend/src/lib/components/runs/RunsFilter.svelte index 923c21cbf74a7..5164310083889 100644 --- a/frontend/src/lib/components/runs/RunsFilter.svelte +++ b/frontend/src/lib/components/runs/RunsFilter.svelte @@ -11,9 +11,11 @@ import Section from '../Section.svelte' import CloseButton from '../common/CloseButton.svelte' import { workspaceStore } from '$lib/stores' + import { createEventDispatcher } from 'svelte' // Filters export let path: string | null = null + export let label: string | null = null export let success: 'running' | 'success' | 'failure' | undefined = undefined export let isSkipped: boolean | undefined = undefined export let argFilter: string @@ -23,7 +25,6 @@ export let jobKindsCat: string export let user: string | null = null export let folder: string | null = null - export let hideSchedules: boolean | undefined = undefined export let mobile: boolean = false // Autocomplete data @@ -32,10 +33,14 @@ export let folders: string[] = [] export let allWorkspaces = false + $: displayedLabel = label + let copyArgFilter = argFilter let copyResultFilter = resultFilter - export let filterBy: 'path' | 'user' | 'folder' = 'path' + export let filterBy: 'path' | 'user' | 'folder' | 'label' = 'path' + + const dispatch = createEventDispatcher() let manuallySet = false @@ -49,8 +54,13 @@ } else if (folder !== null && folder !== '' && filterBy !== 'folder') { manuallySet = true filterBy = 'folder' + } else if (label !== null && label !== '' && filterBy !== 'label') { + manuallySet = true + filterBy = 'label' } } + + let labelTimeout: NodeJS.Timeout | undefined = undefined
@@ -75,6 +85,7 @@ path = null user = null folder = null + label = null } else { manuallySet = false } @@ -83,6 +94,7 @@ +
@@ -94,6 +106,7 @@ class="absolute top-2 right-2 z-50" on:click={() => { user = null + dispatch('reset') }} > @@ -104,6 +117,12 @@ User { + usernames.push(user) + return user + }} + createText="Press enter to use this value" noInputStyles items={usernames} value={user} @@ -123,6 +142,7 @@ class="absolute top-2 right-2 z-50" on:click={() => { folder = null + dispatch('reset') }} > @@ -153,6 +173,7 @@ class="absolute top-2 right-2 z-50" on:click={() => { path = null + dispatch('reset') }} > @@ -164,6 +185,12 @@ Path { + paths.push(path) + return path + }} + createText="Press enter to use this value" noInputStyles items={paths} value={path} @@ -175,6 +202,45 @@ />
{/key} + {:else if filterBy === 'label'} + {#key label} +
+ {#if label} + + {/if} + + Label Labels are values in the result of completed jobs at key 'wm_label' to easily + filter them + + { + if (labelTimeout) { + clearTimeout(labelTimeout) + } + + labelTimeout = setTimeout(() => { + label = displayedLabel + }, 1000) + }} + /> +
+ {/key} {/if}
@@ -262,6 +328,7 @@ path = null user = null folder = null + label = null } else { manuallySet = false } @@ -411,10 +478,6 @@
- - {`Filter by a json being a subset of the args/result. Try '\{"foo": "bar"\}'`} diff --git a/frontend/src/lib/components/runs/RunsTable.svelte b/frontend/src/lib/components/runs/RunsTable.svelte index 168003e886596..20c06c0221a93 100644 --- a/frontend/src/lib/components/runs/RunsTable.svelte +++ b/frontend/src/lib/components/runs/RunsTable.svelte @@ -14,12 +14,17 @@ return job['started_at'] ?? job['scheduled_for'] ?? job['created_at'] } + let containsLabel = false function groupJobsByDay(jobs: Job[]): Record { const groupedLogs: Record = {} if (!jobs) return groupedLogs + let newContainsLabel = false for (const job of jobs) { + if (job?.['label'] != undefined) { + newContainsLabel = true + } const field: string | undefined = getTime(job) if (field) { const date = new Date(field) @@ -38,6 +43,7 @@ groupedLogs[day].push(job) } } + containsLabel = newContainsLabel for (const day in groupedLogs) { groupedLogs[day].sort((a, b) => { @@ -143,6 +149,9 @@ >
Timestamp
Path
+ {#if containsLabel} +
Label
+ {/if}
Triggered by
@@ -165,6 +174,7 @@ {:else}
{ @@ -172,6 +182,7 @@ selectedId = jobOrDate.job.id dispatch('select') }} + on:filterByLabel on:filterByPath on:filterByUser on:filterByFolder diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 34793c6b2c170..d6ee5d9d62002 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -30,6 +30,7 @@ import { twMerge } from 'tailwind-merge' import ManuelDatePicker from '$lib/components/runs/ManuelDatePicker.svelte' import JobLoader from '$lib/components/runs/JobLoader.svelte' + import { Calendar } from 'lucide-svelte' let jobs: Job[] | undefined let selectedId: string | undefined = undefined @@ -40,6 +41,7 @@ let path: string | null = $page.params.path let user: string | null = $page.url.searchParams.get('user') let folder: string | null = $page.url.searchParams.get('folder') + let label: string | null = $page.url.searchParams.get('label') // Rest of filters handled by RunsFilter let success: 'running' | 'success' | 'failure' | undefined = ($page.url.searchParams.get( 'success' @@ -49,10 +51,12 @@ ? $page.url.searchParams.get('is_skipped') == 'true' : false - let hideSchedules: boolean | undefined = - $page.url.searchParams.get('hide_scheduled') != undefined - ? $page.url.searchParams.get('hide_scheduled') == 'true' - : false + let showSchedules: boolean | undefined = + $page.url.searchParams.get('show_schedules') != undefined + ? $page.url.searchParams.get('show_schedules') == 'true' + : localStorage.getItem('show_schedules_in_run') == 'false' + ? false + : true let argFilter: any = $page.url.searchParams.get('arg') ? JSON.parse(decodeURIComponent($page.url.searchParams.get('arg') ?? '{}')) @@ -88,11 +92,12 @@ let manualDatePicker: ManuelDatePicker $: (user || + label || folder || path || success !== undefined || isSkipped || - hideSchedules || + showSchedules || argFilter || resultFilter || schedulePath || @@ -130,10 +135,10 @@ searchParams.delete('is_skipped') } - if (hideSchedules) { - searchParams.set('hide_scheduled', hideSchedules.toString()) + if (showSchedules) { + searchParams.set('show_schedules', showSchedules.toString()) } else { - searchParams.delete('hide_scheduled') + searchParams.delete('show_schedules') } if (allWorkspaces && $workspaceStore == 'admins') { @@ -178,12 +183,18 @@ searchParams.delete('max_ts') } + if (label) { + searchParams.set('label', label) + } else { + searchParams.delete('label') + } + let newPath = path ? `/${path}` : '/' let newUrl = `/runs${newPath}?${searchParams.toString()}` history.replaceState(history.state, '', newUrl.toString()) } - function reloadLogsWithoutFilterError() { + function reloadJobsWithoutFilterError() { if (resultError == '' && argError == '') { filterTimeout && clearTimeout(filterTimeout) filterTimeout = setTimeout(() => { @@ -226,6 +237,34 @@ loadFolders() loadPaths() } + + function filterByPath(e: CustomEvent) { + path = e.detail + user = null + folder = null + label = null + } + + function filterByUser(e: CustomEvent) { + path = null + folder = null + user = e.detail + label = null + } + + function filterByFolder(e: CustomEvent) { + path = null + user = null + folder = e.detail + label = null + } + + function filterByLabel(e: CustomEvent) { + path = null + user = null + folder = null + label = e.detail + } (cancelAllJobs = true)}>Cancel All
+
+ { + localStorage.setItem('show_schedules_in_run', showSchedules ? 'true' : 'false') + }} + /> + Jobs from schedules + + +
@@ -415,21 +467,10 @@ {jobs} bind:selectedId bind:selectedWorkspace - on:filterByPath={(e) => { - user = null - folder = null - path = e.detail - }} - on:filterByUser={(e) => { - path = null - folder = null - user = e.detail - }} - on:filterByFolder={(e) => { - path = null - user = null - folder = e.detail - }} + on:filterByPath={filterByPath} + on:filterByUser={filterByUser} + on:filterByFolder={filterByFolder} + on:filterByLabel={filterByLabel} /> {:else}
@@ -480,10 +521,9 @@ bind:resultFilter bind:argError bind:resultError - bind:hideSchedules bind:allWorkspaces mobile={true} - on:change={reloadLogsWithoutFilterError} + on:change={reloadJobsWithoutFilterError} />
@@ -514,6 +554,18 @@ on:click={async () => (cancelAllJobs = true)}>Cancel All
+
+ { + localStorage.setItem('show_schedules_in_run', showSchedules ? 'true' : 'false') + }} + /> + Schedules + + +
@@ -583,21 +635,10 @@ on:select={() => { runDrawer.openDrawer() }} - on:filterByPath={(e) => { - user = null - folder = null - path = e.detail - }} - on:filterByUser={(e) => { - path = null - folder = null - user = e.detail - }} - on:filterByFolder={(e) => { - path = null - user = null - folder = e.detail - }} + on:filterByPath={filterByPath} + on:filterByUser={filterByUser} + on:filterByFolder={filterByFolder} + on:filterByLabel={filterByLabel} />