From 32260e690ecf50a1c8b548055af20d5c386d9805 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 27 Nov 2024 18:39:15 +0100 Subject: [PATCH 01/60] feat: captures --- .../20241121145420_more_captures.down.sql | 7 + .../20241121145420_more_captures.up.sql | 23 + backend/windmill-api/openapi.yaml | 156 +++++ backend/windmill-api/src/capture.rs | 475 +++++++++++-- backend/windmill-api/src/http_triggers.rs | 50 +- backend/windmill-api/src/lib.rs | 7 +- .../windmill-api/src/websocket_triggers.rs | 636 +++++++++++------- .../src/lib/components/ResourceEditor.svelte | 5 + .../components/ResourceEditorDrawer.svelte | 8 +- .../src/lib/components/ResourcePicker.svelte | 11 +- .../src/lib/components/ScriptBuilder.svelte | 18 +- .../src/lib/components/ScriptEditor.svelte | 194 +++--- .../details/CopyableCodeBlock.svelte | 22 + frontend/src/lib/components/triggers.ts | 41 +- .../components/triggers/CapturePanel.svelte | 514 ++++++++++++++ .../triggers/KafkaTriggerEditor.svelte | 8 +- .../triggers/KafkaTriggerEditorInner.svelte | 18 +- .../triggers/KafkaTriggersPanel.svelte | 16 +- .../components/triggers/RouteEditor.svelte | 8 +- .../triggers/RouteEditorInner.svelte | 8 +- .../components/triggers/RoutesPanel.svelte | 15 +- .../triggers/WebsocketTriggerEditor.svelte | 8 +- .../WebsocketTriggerEditorInner.svelte | 8 +- .../triggers/WebsocketTriggersPanel.svelte | 16 +- 24 files changed, 1815 insertions(+), 457 deletions(-) create mode 100644 backend/migrations/20241121145420_more_captures.down.sql create mode 100644 backend/migrations/20241121145420_more_captures.up.sql create mode 100644 frontend/src/lib/components/details/CopyableCodeBlock.svelte create mode 100644 frontend/src/lib/components/triggers/CapturePanel.svelte diff --git a/backend/migrations/20241121145420_more_captures.down.sql b/backend/migrations/20241121145420_more_captures.down.sql new file mode 100644 index 0000000000000..60199c2b271ca --- /dev/null +++ b/backend/migrations/20241121145420_more_captures.down.sql @@ -0,0 +1,7 @@ +-- Add down migration script here +DROP TABLE capture_config; +DELETE FROM capture; +ALTER TABLE capture DROP CONSTRAINT capture_pkey; +ALTER TABLE capture DROP COLUMN is_flow, DROP COLUMN trigger_kind, DROP COLUMN trigger_extra, DROP COLUMN id; +ALTER TABLE capture ADD CONSTRAINT capture_pkey PRIMARY KEY (workspace_id, path); +DROP TYPE TRIGGER_KIND; diff --git a/backend/migrations/20241121145420_more_captures.up.sql b/backend/migrations/20241121145420_more_captures.up.sql new file mode 100644 index 0000000000000..ce482b1dd7536 --- /dev/null +++ b/backend/migrations/20241121145420_more_captures.up.sql @@ -0,0 +1,23 @@ +-- Add up migration script here +CREATE TYPE TRIGGER_KIND AS ENUM ('webhook', 'http', 'websocket', 'kafka', 'email'); +ALTER TABLE capture ADD COLUMN is_flow BOOLEAN NOT NULL DEFAULT TRUE, ADD COLUMN trigger_kind TRIGGER_KIND NOT NULL DEFAULT 'webhook', ADD COLUMN trigger_extra JSONB; +ALTER TABLE capture ALTER COLUMN is_flow DROP DEFAULT, ALTER COLUMN trigger_kind DROP DEFAULT; +ALTER TABLE capture DROP CONSTRAINT capture_pkey; +ALTER TABLE capture ADD COLUMN id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY; + +CREATE TABLE capture_config ( + workspace_id VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + trigger_kind TRIGGER_KIND NOT NULL, + trigger_config JSONB NULL, + owner VARCHAR(50) NOT NULL, + email VARCHAR(255) NOT NULL, + server_id VARCHAR(50) NULL, + last_client_ping TIMESTAMPTZ NULL, + last_server_ping TIMESTAMPTZ NULL, + error TEXT NULL, + PRIMARY KEY (workspace_id, path, is_flow, trigger_kind), + FOREIGN KEY (workspace_id) REFERENCES workspace(id) ON DELETE CASCADE +); + diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6e95ead2fd887..f8b34617ac917 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8843,6 +8843,116 @@ paths: "204": description: flow preview captured + /w/{workspace}/capture/set_config: + post: + summary: set capture config + operationId: setCaptureConfig + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: capture config + required: true + content: + application/json: + schema: + type: object + properties: + trigger_kind: + $ref: "#/components/schemas/CaptureTriggerKind" + path: + type: string + is_flow: + type: boolean + trigger_config: + type: object + required: + - trigger_kind + - path + - is_flow + responses: + "200": + description: capture config set + + + /w/{workspace}/capture/ping_config/{trigger_kind}/{runnable_kind}/{path}: + post: + summary: ping capture config + operationId: pingCaptureConfig + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: trigger_kind + in: path + required: true + schema: + $ref: "#/components/schemas/CaptureTriggerKind" + - $ref: "#/components/parameters/RunnableKind" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: capture config pinged + + /w/{workspace}/capture/get_configs/{runnable_kind}/{path}: + get: + summary: get capture configs for a script or flow + operationId: getCaptureConfigs + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/RunnableKind" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: capture configs for a script or flow + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/CaptureConfig" + + /w/{workspace}/capture/list/{runnable_kind}/{path}: + get: + summary: list captures for a script or flow + operationId: listCaptures + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/RunnableKind" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: list of captures for a script or flow + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Capture" + + + /w/{workspace}/capture/{id}: + delete: + summary: delete a capture + operationId: deleteCapture + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + schema: + type: integer + responses: + "200": + description: capture deleted + /w/{workspace}/capture/{path}: put: summary: create flow preview capture @@ -10503,6 +10613,13 @@ components: required: true schema: type: string + RunnableKind: + name: runnable_kind + in: path + required: true + schema: + type: string + enum: [script, flow] schemas: $ref: "../../openflow.openapi.yaml#/components/schemas" @@ -13336,3 +13453,42 @@ components: type: string nullable: true description: Workspace id if the alert is in the scope of a workspace + + CaptureTriggerKind: + type: string + enum: [webhook, http, websocket, kafka, email] + + Capture: + type: object + properties: + trigger_kind: + $ref: "#/components/schemas/CaptureTriggerKind" + payload: + type: object + trigger_extra: + type: object + id: + type: integer + created_at: + type: string + format: date-time + required: + - trigger_kind + - payload + - id + - created_at + CaptureConfig: + type: object + properties: + trigger_config: + type: object + trigger_kind: + $ref: "#/components/schemas/CaptureTriggerKind" + error: + type: string + last_server_ping: + type: string + format: date-time + required: + - trigger_config + - trigger_kind diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 241469d05a81e..b209acabc11dd 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -6,134 +6,465 @@ * LICENSE-AGPL for a copy of the license. */ +use std::{collections::HashMap, fmt}; + use axum::{ - extract::{Extension, Path}, - routing::{get, post, put}, - Router, + extract::{Extension, Path, Query}, + routing::{delete, get, head, post}, + Json, Router, }; +use http::HeaderMap; use hyper::StatusCode; -use sqlx::types::Json; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::types::Json as SqlxJson; use windmill_common::{ db::UserDB, - error::{JsonResult, Result}, + error::{Error, JsonResult, Result}, utils::{not_found_if_none, StripPath}, + worker::{to_raw_value, CLOUD_HOSTED}, }; use windmill_queue::{PushArgs, PushArgsOwned}; -use crate::db::{ApiAuthed, DB}; +use crate::{ + db::{ApiAuthed, DB}, + http_triggers::build_http_trigger_extra, +}; -const KEEP_LAST: i64 = 8; +const KEEP_LAST: i64 = 20; pub fn workspaced_service() -> Router { Router::new() - .route("/*path", put(new_payload)) - .route("/*path", get(get_payload)) + .route("/set_config", post(set_config)) + .route( + "/ping_config/:trigger_kind/:runnable_kind/*path", + post(ping_config), + ) + .route("/get_configs/:runnable_kind/*path", get(get_configs)) + .route("/list/:runnable_kind/*path", get(list_captures)) + .route("/:id", delete(delete_capture)) +} + +pub fn workspaced_unauthed_service() -> Router { + Router::new() + .route( + "/webhook/:runnable_kind/*path", + head(|| async {}).post(webhook_payload), + ) + .route( + "/http/:runnable_kind/:path/*route_path", + head(|| async {}).fallback(http_payload), + ) +} + +#[derive(sqlx::Type, Serialize, Deserialize)] +#[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum TriggerKind { + Webhook, + Http, + Websocket, + Kafka, + Email, } -pub fn global_service() -> Router { - Router::new().route("/*path", post(update_payload)) +impl fmt::Display for TriggerKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = match self { + TriggerKind::Webhook => "webhook", + TriggerKind::Http => "http", + TriggerKind::Websocket => "websocket", + TriggerKind::Kafka => "kafka", + TriggerKind::Email => "email", + }; + write!(f, "{}", s) + } } -pub async fn new_payload( +#[derive(Serialize, Deserialize)] +struct NewCaptureConfig { + trigger_kind: TriggerKind, + path: String, + is_flow: bool, + trigger_config: Option>>, +} + +#[derive(Serialize, Deserialize)] +struct CaptureConfig { + trigger_config: Option>>, + trigger_kind: TriggerKind, + error: Option, + last_server_ping: Option>, +} + +async fn get_configs( authed: ApiAuthed, Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { + Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, +) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - " - INSERT INTO capture - (workspace_id, path, created_by) - VALUES ($1, $2, $3) - ON CONFLICT (workspace_id, path) - DO UPDATE SET created_at = now() - ", + let configs = sqlx::query_as!( + CaptureConfig, + r#"SELECT trigger_config as "trigger_config: _", trigger_kind as "trigger_kind: _", error, last_server_ping + FROM capture_config + WHERE workspace_id = $1 AND path = $2 AND is_flow = $3"#, &w_id, &path.to_path(), - &authed.username, + matches!(runnable_kind, RunnableKind::Flow), ) - .execute(&mut *tx) + .fetch_all(&mut *tx) .await?; - /* Retain only KEEP_LAST most recent captures by this user in this workspace. */ + tx.commit().await?; + + Ok(Json(configs)) +} + +async fn set_config( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(nc): Json, +) -> Result<()> { + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( - " - DELETE FROM capture - WHERE workspace_id = $1 - AND created_by = $2 - AND created_at <= - ( SELECT created_at - FROM capture - WHERE workspace_id = $1 - AND created_by = $2 - ORDER BY created_at DESC - OFFSET $3 - LIMIT 1 ) - ", + "INSERT INTO capture_config + (workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (workspace_id, path, is_flow, trigger_kind) + DO UPDATE SET trigger_config = $5, owner = $6, email = $7, server_id = NULL, last_server_ping = NULL, error = NULL", &w_id, + &nc.path, + nc.is_flow, + nc.trigger_kind as TriggerKind, + nc.trigger_config as Option>>, &authed.username, - KEEP_LAST, + &authed.email, ) .execute(&mut *tx) .await?; tx.commit().await?; - Ok(StatusCode::CREATED) + Ok(()) } -pub async fn update_payload( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - args: PushArgsOwned, -) -> Result { - let mut tx = db.begin().await?; - +async fn ping_config( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, trigger_kind, runnable_kind, path)): Path<( + String, + TriggerKind, + RunnableKind, + StripPath, + )>, +) -> Result<()> { + let mut tx = user_db.begin(&authed).await?; sqlx::query!( - " - UPDATE capture - SET payload = $3 - WHERE workspace_id = $1 - AND path = $2 - ", + "UPDATE capture_config SET last_client_ping = now() WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4", &w_id, &path.to_path(), - Json(PushArgs { args: &args.args, extra: args.extra }) as Json, + matches!(runnable_kind, RunnableKind::Flow), + trigger_kind as TriggerKind, ) .execute(&mut *tx) .await?; - tx.commit().await?; + Ok(()) +} - Ok(StatusCode::NO_CONTENT) +#[derive(Serialize, Deserialize)] +struct Capture { + id: i64, + created_at: chrono::DateTime, + trigger_kind: TriggerKind, + payload: SqlxJson>, + trigger_extra: Option>>, } -#[derive(sqlx::FromRow)] -struct Payload { - payload: sqlx::types::Json>, +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum RunnableKind { + Script, + Flow, } -pub async fn get_payload( + +async fn list_captures( authed: ApiAuthed, Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { + Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, +) -> JsonResult> { let mut tx = user_db.begin(&authed).await?; - let payload = sqlx::query_as::<_, Payload>( - " - SELECT payload - FROM capture + let captures = sqlx::query_as!( + Capture, + r#"SELECT id, created_at, trigger_kind as "trigger_kind: _", payload as "payload: _", trigger_extra as "trigger_extra: _" + FROM capture WHERE workspace_id = $1 - AND path = $2 - ", + AND path = $2 AND is_flow = $3 + ORDER BY created_at DESC"#, + &w_id, + &path.to_path(), + matches!(runnable_kind, RunnableKind::Flow), ) - .bind(&w_id) - .bind(&path.to_path()) - .fetch_optional(&mut *tx) + .fetch_all(&mut *tx) .await?; tx.commit().await?; - not_found_if_none(payload.map(|x| x.payload.0), "capture", path.to_path()).map(axum::Json) + Ok(Json(captures)) +} + +async fn delete_capture( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((_, id)): Path<(String, i64)>, +) -> Result<()> { + let mut tx = user_db.begin(&authed).await?; + sqlx::query!("DELETE FROM capture WHERE id = $1", id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +#[derive(Serialize, Deserialize)] +struct ActiveCaptureOwner { + owner: String, + email: String, +} + +pub async fn get_active_capture_owner_and_email( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + kind: &TriggerKind, +) -> Result<(String, String)> { + let capture_config = sqlx::query_as!( + ActiveCaptureOwner, + "SELECT owner, email + FROM capture_config + WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'", + &w_id, + &path, + is_flow, + kind as &TriggerKind, + ) + .fetch_optional(db) + .await?; + + let capture_config = not_found_if_none( + capture_config, + &format!("capture config for {} trigger", kind), + path, + )?; + + Ok((capture_config.owner, capture_config.email)) +} + +async fn get_capture_trigger_config_and_owner( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + kind: &TriggerKind, +) -> Result<(T, String)> { + #[derive(Deserialize)] + struct CaptureTriggerConfigAndOwner { + trigger_config: Option>>, + owner: String, + } + + let capture_config = sqlx::query_as!( + CaptureTriggerConfigAndOwner, + r#"SELECT trigger_config as "trigger_config: _", owner + FROM capture_config + WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'"#, + &w_id, + &path, + is_flow, + kind as &TriggerKind, + ) + .fetch_optional(db) + .await?; + + let capture_config = not_found_if_none( + capture_config, + &format!("capture config for {} trigger", kind), + path, + )?; + + let trigger_config = not_found_if_none( + capture_config.trigger_config, + &format!("capture {} trigger config", kind), + path, + )?; + + Ok(( + serde_json::from_str(trigger_config.get()).map_err(|e| { + Error::InternalErr(format!( + "error parsing capture config for {} trigger: {}", + kind, e + )) + })?, + capture_config.owner, + )) +} + +async fn clear_captures_history(db: &DB, w_id: &str) -> Result<()> { + if *CLOUD_HOSTED { + /* Retain only KEEP_LAST most recent captures in this workspace. */ + sqlx::query!( + "DELETE FROM capture + WHERE workspace_id = $1 + AND created_at <= + ( + SELECT created_at + FROM capture + WHERE workspace_id = $1 + ORDER BY created_at DESC + OFFSET $2 + LIMIT 1 + )", + &w_id, + KEEP_LAST, + ) + .execute(db) + .await?; + } + Ok(()) +} + +pub async fn insert_capture_payload( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + trigger_kind: &TriggerKind, + payload: PushArgsOwned, + trigger_extra: Option>, + owner: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + &w_id, + path, + is_flow, + trigger_kind as &TriggerKind, + SqlxJson(to_raw_value(&PushArgs { + args: &payload.args, + extra: payload.extra + })) as SqlxJson>, + trigger_extra.map(SqlxJson) as Option>>, + owner, + ) + .execute(db) + .await?; + + clear_captures_history(db, &w_id).await?; + + Ok(()) +} + +async fn webhook_payload( + Extension(db): Extension, + Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, + args: PushArgsOwned, +) -> Result { + let (owner, _) = get_active_capture_owner_and_email( + &db, + &w_id, + &path.to_path(), + matches!(runnable_kind, RunnableKind::Flow), + &TriggerKind::Webhook, + ) + .await?; + + insert_capture_payload( + &db, + &w_id, + &path.to_path(), + matches!(runnable_kind, RunnableKind::Flow), + &TriggerKind::Webhook, + args, + Some(to_raw_value(&serde_json::json!({ + "wm_trigger": { + "kind": "webhook", + } + }))), + &owner, + ) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Serialize, Deserialize)] +struct HttpTriggerConfig { + route_path: String, +} + +async fn http_payload( + Extension(db): Extension, + Path((w_id, kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>, + Query(query): Query>, + method: http::Method, + headers: HeaderMap, + args: PushArgsOwned, +) -> Result { + let route_path = route_path.to_path(); + + let (http_trigger_config, owner): (HttpTriggerConfig, _) = + get_capture_trigger_config_and_owner( + &db, + &w_id, + &path, + matches!(kind, RunnableKind::Flow), + &TriggerKind::Http, + ) + .await?; + + let mut router = matchit::Router::new(); + router.insert(&http_trigger_config.route_path, ()).ok(); + let match_ = router.at(route_path).ok(); + + let match_ = not_found_if_none(match_, "capture http trigger", &route_path)?; + + let matchit::Match { params, .. } = match_; + + let params: HashMap = params + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let extra: HashMap> = HashMap::from_iter(vec![( + "wm_trigger".to_string(), + build_http_trigger_extra( + &http_trigger_config.route_path, + route_path, + &method, + ¶ms, + &query, + &headers, + ) + .await, + )]); + + insert_capture_payload( + &db, + &w_id, + &path, + matches!(kind, RunnableKind::Flow), + &TriggerKind::Http, + args, + Some(to_raw_value(&extra)), + &owner, + ) + .await?; + + Ok(StatusCode::NO_CONTENT) } diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index feeb9977a8822..4eee704ccd958 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -514,6 +514,32 @@ async fn get_http_route_trigger( Ok((trigger, route_path.0, params, authed)) } +pub async fn build_http_trigger_extra( + route_path: &str, + called_path: &str, + method: &http::Method, + params: &HashMap, + query: &HashMap, + headers: &HeaderMap, +) -> Box { + let headers = headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect::>(); + + to_raw_value(&serde_json::json!({ + "kind": "http", + "http": { + "route": route_path, + "path": called_path, + "method": method.to_string().to_lowercase(), + "params": params, + "query": query, + "headers": headers + }, + })) +} + async fn route_job( Extension(db): Extension, Extension(user_db): Extension, @@ -615,24 +641,18 @@ async fn route_job( } } - let headers = headers - .iter() - .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) - .collect::>(); let extra = args.extra.get_or_insert_with(HashMap::new); extra.insert( "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": "http", - "http": { - "route": trigger.route_path, - "path": called_path, - "method": method.to_string().to_lowercase(), - "params": params, - "query": query, - "headers": headers - }, - })), + build_http_trigger_extra( + &trigger.route_path, + &called_path, + &method, + ¶ms, + &query, + &headers, + ) + .await, ); let http_method = http::Method::from(trigger.http_method); diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index ba1ab8df59bed..a5e409420a881 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -39,11 +39,12 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::{ALL_TAGS, CLOUD_HOSTED}; -use windmill_common::{BASE_URL, INSTANCE_NAME, utils::GIT_VERSION}; +use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME}; use crate::scim_ee::has_scim_token; use windmill_common::error::AppError; +mod ai; mod apps; mod audit; mod capture; @@ -62,7 +63,6 @@ mod http_triggers; mod indexer_ee; mod inputs; mod integration; -mod ai; #[cfg(feature = "parquet")] mod job_helpers_ee; @@ -97,7 +97,6 @@ mod workspaces_ee; pub const DEFAULT_BODY_LIMIT: usize = 2097152 * 100; // 200MB - lazy_static::lazy_static! { pub static ref REQUEST_SIZE_LIMIT: Arc> = Arc::new(RwLock::new(DEFAULT_BODY_LIMIT)); @@ -374,7 +373,7 @@ pub async fn run_server( ) .nest( "/w/:workspace_id/capture_u", - capture::global_service().layer(cors.clone()), + capture::workspaced_unauthed_service().layer(cors.clone()), ) .nest( "/auth", diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index d0fd2fa8cfa59..cccb5148d4592 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -15,6 +15,7 @@ use serde::{ use serde_json::{value::RawValue, Value}; use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::prelude::FromRow; +use sqlx::types::Json as SqlxJson; use std::{collections::HashMap, fmt}; use tokio::net::TcpStream; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; @@ -23,20 +24,21 @@ use windmill_audit::{audit_ee::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{self, to_anyhow, JsonResult}, - utils::{ - not_found_if_none, paginate, report_critical_error, Pagination, StripPath, - }, + utils::{not_found_if_none, paginate, report_critical_error, Pagination, StripPath}, worker::{to_raw_value, CLOUD_HOSTED}, INSTANCE_NAME, }; use windmill_queue::PushArgsOwned; use crate::{ + capture::{insert_capture_payload, TriggerKind}, db::{ApiAuthed, DB}, jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, users::fetch_api_authed, }; +use std::borrow::Cow; + pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_websocket_trigger)) @@ -61,14 +63,14 @@ struct NewWebsocketTrigger { } #[derive(Deserialize)] -struct JsonFilter { +pub struct JsonFilter { key: String, value: serde_json::Value, } #[derive(Deserialize)] #[serde(untagged)] -enum Filter { +pub enum Filter { JsonFilter(JsonFilter), } @@ -95,9 +97,9 @@ pub struct WebsocketTrigger { extra_perms: serde_json::Value, error: Option, enabled: bool, - filters: Vec>>, - initial_messages: Vec>>, - url_runnable_args: sqlx::types::Json>, + filters: Vec>>, + initial_messages: Vec>>, + url_runnable_args: SqlxJson>, } #[derive(Deserialize)] @@ -192,12 +194,8 @@ async fn create_websocket_trigger( let mut tx = user_db.begin(&authed).await?; - let filters = ct.filters.into_iter().map(sqlx::types::Json).collect_vec(); - let initial_messages = ct - .initial_messages - .into_iter() - .map(sqlx::types::Json) - .collect_vec(); + let filters = ct.filters.into_iter().map(SqlxJson).collect_vec(); + let initial_messages = ct.initial_messages.into_iter().map(SqlxJson).collect_vec(); sqlx::query_as::<_, WebsocketTrigger>( "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, initial_messages, url_runnable_args, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now()) RETURNING *", ) @@ -209,7 +207,7 @@ async fn create_websocket_trigger( .bind(ct.enabled.unwrap_or(true)) .bind(filters.as_slice()) .bind(initial_messages.as_slice()) - .bind(sqlx::types::Json(ct.url_runnable_args)) + .bind(SqlxJson(ct.url_runnable_args)) .bind(&authed.username) .bind(&authed.email) .fetch_one(&mut *tx).await?; @@ -239,12 +237,8 @@ async fn update_websocket_trigger( let path = path.to_path(); let mut tx = user_db.begin(&authed).await?; - let filters = ct.filters.into_iter().map(sqlx::types::Json).collect_vec(); - let initial_messages = ct - .initial_messages - .into_iter() - .map(sqlx::types::Json) - .collect_vec(); + let filters = ct.filters.into_iter().map(SqlxJson).collect_vec(); + let initial_messages = ct.initial_messages.into_iter().map(SqlxJson).collect_vec(); // important to update server_id, last_server_ping and error to NULL to stop current websocket listener sqlx::query!( @@ -254,9 +248,9 @@ async fn update_websocket_trigger( ct.script_path, ct.path, ct.is_flow, - filters.as_slice() as &[sqlx::types::Json>], - initial_messages.as_slice() as &[sqlx::types::Json>], - sqlx::types::Json(ct.url_runnable_args) as sqlx::types::Json>, + filters.as_slice() as &[SqlxJson>], + initial_messages.as_slice() as &[SqlxJson>], + SqlxJson(ct.url_runnable_args) as SqlxJson>, &authed.username, &authed.email, w_id, @@ -389,13 +383,31 @@ async fn listen_to_unlistened_websockets( Ok(mut triggers) => { triggers.shuffle(&mut rand::thread_rng()); for trigger in triggers { - maybe_listen_to_websocket(trigger, db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; + trigger.maybe_listen_to_websocket(db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; } } Err(err) => { tracing::error!("Error fetching websocket triggers: {:?}", err); } }; + + match sqlx::query_as!( + CaptureConfigForWebsocket, + r#"SELECT path, is_flow, workspace_id, trigger_config as "trigger_config!: _", owner FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')"# + ) + .fetch_all(db) + .await + { + Ok(mut captures) => { + captures.shuffle(&mut rand::thread_rng()); + for capture in captures { + capture.maybe_listen_to_websocket(db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; + } + } + Err(err) => { + tracing::error!("Error fetching capture websocket triggers: {:?}", err); + } + } } pub async fn start_websockets( @@ -419,31 +431,6 @@ pub async fn start_websockets( }); } -async fn maybe_listen_to_websocket( - ws_trigger: WebsocketTrigger, - db: DB, - rsmq: Option, - killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now() WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - *INSTANCE_NAME, - ws_trigger.workspace_id, - ws_trigger.path, - ).fetch_optional(&db).await { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(ws_trigger, db, rsmq, killpill_rx)); - } else { - tracing::info!("Websocket {} already being listened to", ws_trigger.url); - } - }, - Err(err) => { - tracing::error!("Error acquiring lock for websocket {}: {:?}", ws_trigger.path, err); - } - }; -} - struct SupersetVisitor<'a> { key: &'a str, value_to_check: &'a Value, @@ -574,7 +561,7 @@ async fn wait_runnable_result( #[derive(sqlx::FromRow)] struct RawResult { - result: Option>>, + result: Option>>, success: bool, } @@ -612,218 +599,372 @@ async fn wait_runnable_result( } } -async fn send_initial_messages( - ws_trigger: &WebsocketTrigger, - mut writer: SplitSink>, Message>, - db: &DB, - rsmq: Option, -) -> error::Result<()> { - let initial_messages: Vec = ws_trigger - .initial_messages - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); - - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() +async fn loop_ping(db: &DB, ws: &WebsocketEnum, error: Option<&str>) -> () { + loop { + if let None = ws.update_ping(db, error).await { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } +} + +impl WebsocketTrigger { + async fn maybe_listen_to_websocket( + self, + db: DB, + rsmq: Option, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> () { + match sqlx::query_scalar!( + "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now() WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + *INSTANCE_NAME, + self.workspace_id, + self.path, + ).fetch_optional(&db).await { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn(listen_to_websocket(WebsocketEnum::Trigger(self), db, rsmq, killpill_rx)); } else { - msg - }; - tracing::info!( - "Sending raw message initial message to websocket {}: {}", - ws_trigger.url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; + tracing::info!("Websocket {} already being listened to", self.url); + } + }, + Err(err) => { + tracing::error!("Error acquiring lock for websocket {}: {:?}", self.path, err); } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running runnable {path} (is_flow: {is_flow}) for initial message to websocket {}", - ws_trigger.url, - ); - - let result = wait_runnable_result( - path.clone(), - is_flow, - &args, - ws_trigger, - "init".to_string(), - db, - rsmq.clone(), - ) - .await?; + }; + } - tracing::info!( - "Sending runnable {path} (is_flow: {is_flow}) result to websocket {}", - ws_trigger.url - ); + async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { + match sqlx::query_scalar!( + "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", + error, + self.workspace_id, + self.path, + *INSTANCE_NAME + ).fetch_optional(db).await { + Ok(updated) => { + if updated.flatten().is_none() { + tracing::info!("Websocket {} changed, disabled, or deleted, stopping...", self.url); + return None; + } + }, + Err(err) => { + tracing::warn!("Error updating ping of websocket {}: {:?}", self.url, err); + } + }; - let result = if result.starts_with("\"") && result.ends_with("\"") { - result[1..result.len() - 1].to_string() - } else { - result - }; - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!("Failed to send runnable {path} (is_flow: {is_flow}) result") - })?; + Some(()) + } + + async fn disable_with_error(&self, db: &DB, error: String) -> () { + match sqlx::query!( + "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", + error, + self.workspace_id, + self.path, + ) + .execute(db).await { + Ok(_) => { + report_critical_error(format!("Disabling websocket {} because of error: {}", self.url, error), db.clone(), Some(&self.workspace_id), None).await; + }, + Err(disable_err) => { + report_critical_error( + format!("Could not disable websocket {} with err {}, disabling because of error {}", self.path, disable_err, error), + db.clone(), + Some(&self.workspace_id), + None, + ).await; } } } - Ok(()) + async fn get_url_from_runnable( + &self, + path: &str, + is_flow: bool, + db: &DB, + rsmq: Option, + ) -> error::Result { + tracing::info!("Running runnable {path} (is_flow: {is_flow}) to get websocket URL",); + + let result = wait_runnable_result( + path.to_string(), + is_flow, + &self.url_runnable_args.0, + self, + "url".to_string(), + db, + rsmq, + ) + .await?; + + if result.starts_with("\"") && result.ends_with("\"") { + Ok(result[1..result.len() - 1].to_string()) + } else { + Err( + anyhow::anyhow!("Runnable {path} (is_flow: {is_flow}) did not return a string") + .into(), + ) + } + } + + async fn send_initial_messages( + &self, + mut writer: SplitSink>, Message>, + db: &DB, + rsmq: Option, + ) -> error::Result<()> { + let initial_messages: Vec = self + .initial_messages + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); + + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to websocket {}: {}", + self.url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) + .await + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running runnable {path} (is_flow: {is_flow}) for initial message to websocket {}", + self.url, + ); + + let result = wait_runnable_result( + path.clone(), + is_flow, + &args, + self, + "init".to_string(), + db, + rsmq.clone(), + ) + .await?; + + tracing::info!( + "Sending runnable {path} (is_flow: {is_flow}) result to websocket {}", + self.url + ); + + let result = if result.starts_with("\"") && result.ends_with("\"") { + result[1..result.len() - 1].to_string() + } else { + result + }; + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| { + format!("Failed to send runnable {path} (is_flow: {is_flow}) result") + })?; + } + } + } + + Ok(()) + } + + async fn handle( + &self, + db: &DB, + rsmq: Option, + args: PushArgsOwned, + ) -> () { + if let Err(err) = run_job(db, rsmq, self, args).await { + report_critical_error( + format!( + "Failed to trigger job from websocket {}: {:?}", + self.url, err + ), + db.clone(), + Some(&self.workspace_id), + None, + ) + .await; + }; + } +} + +#[derive(Deserialize)] +struct WebsocketTriggerConfig { + url: String, } -async fn get_url_from_runnable( - path: &str, +#[derive(Deserialize)] +struct CaptureConfigForWebsocket { + trigger_config: SqlxJson, + path: String, is_flow: bool, - ws_trigger: &WebsocketTrigger, - db: &DB, - rsmq: Option, -) -> error::Result { - tracing::info!("Running runnable {path} (is_flow: {is_flow}) to get websocket URL",); - - let result = wait_runnable_result( - path.to_string(), - is_flow, - &ws_trigger.url_runnable_args.0, - ws_trigger, - "url".to_string(), - db, - rsmq, - ) - .await?; + workspace_id: String, + owner: String, +} - if result.starts_with("\"") && result.ends_with("\"") { - Ok(result[1..result.len() - 1].to_string()) - } else { - Err(anyhow::anyhow!("Runnable {path} (is_flow: {is_flow}) did not return a string").into()) +impl CaptureConfigForWebsocket { + async fn maybe_listen_to_websocket( + self, + db: DB, + rsmq: Option, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> () { + match sqlx::query_scalar!( + "UPDATE capture_config SET server_id = $1, last_server_ping = now() WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + *INSTANCE_NAME, + self.workspace_id, + self.path, + self.is_flow, + ).fetch_optional(&db).await { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn(listen_to_websocket(WebsocketEnum::Capture(self), db, rsmq, killpill_rx)); + } else { + tracing::info!("Websocket {} already being listened to", self.trigger_config.url); + } + }, + Err(err) => { + tracing::error!("Error acquiring lock for capture websocket {}: {:?}", self.path, err); + } + }; } -} -async fn update_ping(db: &DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", + async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { + match sqlx::query_scalar!( + "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", error, - ws_trigger.workspace_id, - ws_trigger.path, + self.workspace_id, + self.path, + self.is_flow, *INSTANCE_NAME ).fetch_optional(db).await { Ok(updated) => { if updated.flatten().is_none() { - tracing::info!("Websocket {} changed, disabled, or deleted, stopping...", ws_trigger.url); + tracing::info!("Websocket capture {} changed, disabled, or deleted, stopping...", self.trigger_config.url); return None; } }, - Err(err) => { - tracing::warn!("Error updating ping of websocket {}: {:?}", ws_trigger.url, err); - } - }; + Err(err) => { + tracing::warn!("Error updating ping of capture websocket {}: {:?}", self.trigger_config.url, err); + } + }; - Some(()) -} + Some(()) + } -async fn loop_ping(db: &DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> () { - loop { - if let None = update_ping(db, ws_trigger, error).await { - return; + async fn handle(&self, db: &DB, args: PushArgsOwned) -> () { + if let Err(err) = insert_capture_payload( + db, + &self.workspace_id, + &self.path, + self.is_flow, + &TriggerKind::Websocket, + PushArgsOwned { args: args.args, extra: None }, + args.extra.as_ref().map(to_raw_value), + &self.owner, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; } } -async fn disable_with_error(db: &DB, ws_trigger: &WebsocketTrigger, error: String) { - match sqlx::query!( - "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - error, - ws_trigger.workspace_id, - ws_trigger.path, - ) - .execute(db).await { - Ok(_) => { - report_critical_error(format!("Disabling websocket {} because of error: {}", ws_trigger.url, error), db.clone(), Some(&ws_trigger.workspace_id), None).await; - }, - Err(disable_err) => { - report_critical_error( - format!("Could not disable websocket {} with err {}, disabling because of error {}", ws_trigger.path, disable_err, error), - db.clone(), - Some(&ws_trigger.workspace_id), - None, - ).await; +enum WebsocketEnum { + Trigger(WebsocketTrigger), + Capture(CaptureConfigForWebsocket), +} + +impl WebsocketEnum { + async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { + match self { + WebsocketEnum::Trigger(ws) => ws.update_ping(db, error).await, + WebsocketEnum::Capture(capture) => capture.update_ping(db, error).await, } } } async fn listen_to_websocket( - ws_trigger: WebsocketTrigger, + ws: WebsocketEnum, db: DB, rsmq: Option, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { - if let None = update_ping(&db, &ws_trigger, Some("Connecting...")).await { + if let None = ws.update_ping(&db, Some("Connecting")).await { return; } - let url = ws_trigger.url.as_str(); + let url = match &ws { + WebsocketEnum::Trigger(ws_trigger) => &ws_trigger.url, + WebsocketEnum::Capture(capture) => &capture.trigger_config.url, + }; - let filters: Vec = ws_trigger - .filters - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); + let filters: Vec = match &ws { + WebsocketEnum::Trigger(ws_trigger) => ws_trigger + .filters + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(), + WebsocketEnum::Capture(_) => vec![], + }; loop { - let connect_url = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - let path = url.splitn(2, ':').nth(1).unwrap(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws_trigger, Some( - "Waiting on runnable to return websocket URL..." - )) => { - return; - }, - url_result = get_url_from_runnable(path, url.starts_with("$flow:"), &ws_trigger, &db, rsmq.clone()) => match url_result { - Ok(url) => url, - Err(err) => { - disable_with_error( + let connect_url: Cow = match &ws { + WebsocketEnum::Trigger(ws_trigger) => { + if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + let path = url.splitn(2, ':').nth(1).unwrap(); + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + }, + _ = loop_ping(&db, &ws, Some( + "Waiting on runnable to return websocket URL..." + )) => { + return; + }, + url_result = ws_trigger.get_url_from_runnable(path, url.starts_with("$flow:"), &db, rsmq.clone()) => match url_result { + Ok(url) => Cow::Owned(url), + Err(err) => { + ws_trigger.disable_with_error(&db, format!( + "Error getting websocket URL from runnable after 5 tries: {:?}", + err + ), + ) + .await; + return; + } + }, + } + } else { + ws_trigger + .disable_with_error( &db, - &ws_trigger, - format!( - "Error getting websocket URL from runnable after 5 tries: {:?}", - err - ), + format!("Invalid websocket runnable path: {}", url), ) .await; - return; - } - }, + return; + } + } else { + Cow::Borrowed(url) } - } else { - disable_with_error( - &db, - &ws_trigger, - format!("Invalid websocket runnable path: {}", url), - ) - .await; - return; } - } else { - url.to_string() + WebsocketEnum::Capture(capture) => Cow::Borrowed(&capture.trigger_config.url), }; tokio::select! { @@ -831,14 +972,14 @@ async fn listen_to_websocket( _ = killpill_rx.recv() => { return; }, - _ = loop_ping(&db, &ws_trigger, Some("Connecting...")) => { + _ = loop_ping(&db, &ws, Some("Connecting...")) => { return; }, - connection = connect_async(connect_url) => { + connection = connect_async(connect_url.as_ref()) => { match connection { Ok((ws_stream, _)) => { tracing::info!("Listening to websocket {}", url); - if let None = update_ping(&db, &ws_trigger, None).await { + if let None = ws.update_ping(&db, None).await { return; } let (writer, mut reader) = ws_stream.split(); @@ -850,11 +991,18 @@ async fn listen_to_websocket( return; } _ = async { - if let Err(err) = send_initial_messages(&ws_trigger, writer, &db, rsmq.clone()).await { - disable_with_error(&db, &ws_trigger, format!("Error sending initial messages: {:?}", err)).await; - } else { - // if initial messages sent successfully, wait forever - futures::future::pending::<()>().await; + match &ws { + WebsocketEnum::Trigger(ws_trigger) => { + if let Err(err) = ws_trigger.send_initial_messages(writer, &db, rsmq.clone()).await { + ws_trigger.disable_with_error(&db, format!("Error sending initial messages: {:?}", err)).await; + } else { + // if initial messages sent successfully, wait forever + futures::future::pending::<()>().await; + } + }, + WebsocketEnum::Capture(_) => { + futures::future::pending::<()>().await; + } } } => { // was disabled => exit @@ -867,7 +1015,7 @@ async fn listen_to_websocket( msg = reader.next() => { if let Some(msg) = msg { if last_ping.elapsed() > tokio::time::Duration::from_secs(5) { - if let None = update_ping(&db, &ws_trigger, None).await { + if let None = ws.update_ping(&db, None).await { return; } last_ping = tokio::time::Instant::now(); @@ -897,9 +1045,22 @@ async fn listen_to_websocket( } } if should_handle { - if let Err(err) = run_job(&db, rsmq.clone(), &ws_trigger, text).await { - report_critical_error(format!("Failed to trigger job from websocket {}: {:?}", ws_trigger.url, err), db.clone(), Some(&ws_trigger.workspace_id), None).await; - }; + + let args = HashMap::from([("msg".to_string(), to_raw_value(&text))]); + let extra = Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({"kind": "websocket", "websocket": { "url": url }})), + )])); + + let args = PushArgsOwned { args, extra }; + match &ws { + WebsocketEnum::Trigger(ws_trigger) => { + ws_trigger.handle(&db, rsmq.clone(), args).await; + }, + WebsocketEnum::Capture(capture) => { + capture.handle(&db, args).await; + } + } } }, _ => {} @@ -911,9 +1072,7 @@ async fn listen_to_websocket( } } else { tracing::error!("Websocket {} closed", url); - if let None = - update_ping(&db, &ws_trigger, Some("Websocket closed")).await - { + if let None = ws.update_ping(&db, Some("Websocket closed")).await { return; } tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; @@ -921,7 +1080,7 @@ async fn listen_to_websocket( } }, _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => { - if let None = update_ping(&db, &ws_trigger, None).await { + if let None = ws.update_ping(&db, None).await { return; } last_ping = tokio::time::Instant::now(); @@ -935,9 +1094,7 @@ async fn listen_to_websocket( } Err(err) => { tracing::error!("Error connecting to websocket {}: {:?}", url, err); - if let None = - update_ping(&db, &ws_trigger, Some(err.to_string().as_str())).await - { + if let None = ws.update_ping(&db, Some(err.to_string().as_str())).await { return; } tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; @@ -952,17 +1109,8 @@ async fn run_job( db: &DB, rsmq: Option, trigger: &WebsocketTrigger, - msg: String, + args: PushArgsOwned, ) -> anyhow::Result<()> { - let args = PushArgsOwned { - args: HashMap::from([("msg".to_string(), to_raw_value(&msg))]), - extra: Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value( - &serde_json::json!({"kind": "websocket", "websocket": { "url": trigger.url }}), - ), - )])), - }; let label_prefix = Some(format!("ws-{}-", trigger.path)); let authed = fetch_api_authed( diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index e6bd21e40b578..d87268aaf8229 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,6 +24,11 @@ export let newResource: boolean = false export let hidePath: boolean = false export let watchChanges: boolean = false + export let defaultValues: Record | undefined = undefined + + $: if (defaultValues && Object.keys(defaultValues).length > 0) { + args = defaultValues + } let isValid = true let jsonError = '' diff --git a/frontend/src/lib/components/ResourceEditorDrawer.svelte b/frontend/src/lib/components/ResourceEditorDrawer.svelte index 114e2f32f6609..b7c80ec840501 100644 --- a/frontend/src/lib/components/ResourceEditorDrawer.svelte +++ b/frontend/src/lib/components/ResourceEditorDrawer.svelte @@ -8,6 +8,7 @@ let drawer: Drawer let canSave = true let resource_type: string | undefined = undefined + let defaultValues: Record | undefined = undefined let resourceEditor: { editResource: () => void } | undefined = undefined @@ -21,10 +22,14 @@ drawer.openDrawer?.() } - export async function initNew(resourceType: string): Promise { + export async function initNew( + resourceType: string, + nDefaultValues?: Record + ): Promise { newResource = true path = undefined resource_type = resourceType + defaultValues = nDefaultValues drawer.openDrawer?.() } @@ -38,6 +43,7 @@ {newResource} {path} {resource_type} + {defaultValues} on:refresh bind:this={resourceEditor} bind:canSave diff --git a/frontend/src/lib/components/ResourcePicker.svelte b/frontend/src/lib/components/ResourcePicker.svelte index 4942ec3014271..b2c9ac6807295 100644 --- a/frontend/src/lib/components/ResourcePicker.svelte +++ b/frontend/src/lib/components/ResourcePicker.svelte @@ -1,7 +1,7 @@
- {#if showTabs} -
- - Main - Preprocessor - -
- {/if} -
- {#if testIsLoading} - - {:else} - - {/if} +
+ + Main + {#if hasPreprocessor} +
+ Preprocessor +
+ {/if} + Capture +
- - -
-
- { + argsRender++ + console.log(e.detail.args) + args = e.detail.args ?? {} + selectedTab = e.detail.kind + }} + /> + {:else} +
+ {#if testIsLoading} + + {:else} + + {/if} +
+ + +
+
+ {#key argsRender} + + {/key} +
-
- - - - {#if scriptProgress} - - - {/if} - - - + + + + {#if scriptProgress} + + + {/if} + + + + {/if}
diff --git a/frontend/src/lib/components/details/CopyableCodeBlock.svelte b/frontend/src/lib/components/details/CopyableCodeBlock.svelte new file mode 100644 index 0000000000000..6468ca127cf72 --- /dev/null +++ b/frontend/src/lib/components/details/CopyableCodeBlock.svelte @@ -0,0 +1,22 @@ + + + + +
{ + e.preventDefault() + copyToClipboard(code) + }} +> + + +
diff --git a/frontend/src/lib/components/triggers.ts b/frontend/src/lib/components/triggers.ts index efc8c8a901395..d7c0158c6a221 100644 --- a/frontend/src/lib/components/triggers.ts +++ b/frontend/src/lib/components/triggers.ts @@ -1,4 +1,4 @@ -import type { TriggersCount } from '$lib/gen' +import type { CaptureTriggerKind, TriggersCount } from '$lib/gen' import type { Writable } from 'svelte/store' export type ScheduleTrigger = { @@ -10,19 +10,11 @@ export type ScheduleTrigger = { } export type TriggerContext = { - selectedTrigger: Writable< - | 'webhooks' - | 'emails' - | 'schedules' - | 'cli' - | 'routes' - | 'websockets' - | 'scheduledPoll' - | 'kafka' - > + selectedTrigger: Writable primarySchedule: Writable triggersCount: Writable simplifiedPoll: Writable + defaultValues?: Writable | undefined> } export function setScheduledPollSchedule( @@ -45,3 +37,30 @@ export function setScheduledPollSchedule( } }) } + +export type TriggerKind = + | 'webhooks' + | 'emails' + | 'schedules' + | 'cli' + | 'routes' + | 'websockets' + | 'scheduledPoll' + | 'kafka' + +export function captureTriggerKindToTriggerKind(kind: CaptureTriggerKind): TriggerKind { + switch (kind) { + case 'webhook': + return 'webhooks' + case 'email': + return 'emails' + case 'http': + return 'routes' + case 'websocket': + return 'websockets' + case 'kafka': + return 'kafka' + default: + throw new Error(`Unknown CaptureTriggerKind: ${kind}`) + } +} diff --git a/frontend/src/lib/components/triggers/CapturePanel.svelte b/frontend/src/lib/components/triggers/CapturePanel.svelte new file mode 100644 index 0000000000000..71d8b6b6a6803 --- /dev/null +++ b/frontend/src/lib/components/triggers/CapturePanel.svelte @@ -0,0 +1,514 @@ + + +
+
+ + + + + + + + +
+ {#if (selected === 'websocket' || selected === 'kafka') && config && active} + {@const serverEnabled = getServerEnabled(config)} +
+ {#if serverEnabled} + + + + +
Websocket is connected
+ + {:else} + + + + + +
+ Websocket is not connected{config.error ? ': ' + config.error : ''} +
+ + {/if} +
+ {/if} + + +
+
+ + {#if selected in schemas} + + {/if} + + {#if selected === 'webhook'} + + + {:else if selected === 'http'} + + + {:else if selected === 'email'} + + {/if} + + +
diff --git a/frontend/src/lib/components/triggers/KafkaTriggerEditor.svelte b/frontend/src/lib/components/triggers/KafkaTriggerEditor.svelte index 33083d7ec62b5..4044225223307 100644 --- a/frontend/src/lib/components/triggers/KafkaTriggerEditor.svelte +++ b/frontend/src/lib/components/triggers/KafkaTriggerEditor.svelte @@ -9,10 +9,14 @@ drawer?.openEdit(ePath, isFlow) } - export async function openNew(is_flow: boolean, initial_script_path?: string) { + export async function openNew( + is_flow: boolean, + initial_script_path?: string, + defaultValues?: Record + ) { open = true await tick() - drawer?.openNew(is_flow, initial_script_path) + drawer?.openNew(is_flow, initial_script_path, defaultValues) } let drawer: KafkaTriggerEditorInner diff --git a/frontend/src/lib/components/triggers/KafkaTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/KafkaTriggerEditorInner.svelte index c65bdbc32c253..42f69aa896402 100644 --- a/frontend/src/lib/components/triggers/KafkaTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/KafkaTriggerEditorInner.svelte @@ -34,6 +34,7 @@ let dirtyPath = false let can_write = true let drawerLoading = true + let defaultValues: Record | undefined = undefined const dispatch = createEventDispatcher() @@ -56,7 +57,11 @@ } } - export async function openNew(nis_flow: boolean, fixedScriptPath_?: string) { + export async function openNew( + nis_flow: boolean, + fixedScriptPath_?: string, + nDefaultValues?: Record + ) { drawerLoading = true try { drawer?.openDrawer() @@ -64,8 +69,8 @@ edit = false itemKind = nis_flow ? 'flow' : 'script' kafka_resource_path = '' - group_id = '' - topics = [''] + group_id = nDefaultValues?.group_id ?? '' + topics = nDefaultValues?.topics ?? [''] dirtyGroupId = false initialScriptPath = '' fixedScriptPath = fixedScriptPath_ ?? '' @@ -73,6 +78,7 @@ path = '' initialPath = '' dirtyPath = false + defaultValues = nDefaultValues } finally { drawerLoading = false } @@ -219,7 +225,11 @@ Resource
- +
diff --git a/frontend/src/lib/components/flows/content/CapturePayload.svelte b/frontend/src/lib/components/flows/content/CapturePayload.svelte deleted file mode 100644 index 65fa03002fc13..0000000000000 --- a/frontend/src/lib/components/flows/content/CapturePayload.svelte +++ /dev/null @@ -1,125 +0,0 @@ - - - { - startCapturePoint() - interval = setInterval(() => { - getCaptureInput() - }, 1000) - }} - on:close={() => interval && clearInterval(interval)} -> - - Send a payload at: -

CURL example

- -
- 
{`curl -X POST ${hostname}/api/w/${$workspaceStore}/capture_u/${$pathStore} \\
-   -H 'Content-Type: application/json' \\
-   -d '{"foo": 42}'`}
-
-
- Listening for new requests - - - - -
-
- -
- - - - -

Derived schema

-
- -
-

Test args

- - - diff --git a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte index deb9c6dd1749c..cd5e9bc4b8824 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte @@ -12,6 +12,7 @@ import { initFlowStepWarnings } from '../utils' import { dfs } from '../dfs' import FlowPreprocessorModule from './FlowPreprocessorModule.svelte' + import type { TriggerContext } from '$lib/components/triggers' export let noEditor = false export let enableAi = false @@ -21,6 +22,7 @@ const { selectedId, flowStore, flowStateStore, flowInputsStore, pathStore, initialPath } = getContext('FlowEditorContext') + const { selectedTrigger, defaultValues } = getContext('TriggerContext') function checkDup(modules: FlowModule[]): string | undefined { let seenModules: string[] = [] for (const m of modules) { @@ -60,7 +62,16 @@ {#if $selectedId?.startsWith('settings')} {:else if $selectedId === 'Input'} - + { + $selectedId = 'triggers' + selectedTrigger.set(ev.detail.kind) + defaultValues.set(ev.detail.config) + }} + on:applyArgs + /> {:else if $selectedId === 'Result'}

The result of the flow will be the result of the last node.

{:else if $selectedId === 'constants'} diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index cb7ed9ad80e83..0834e03ab2127 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -4,7 +4,6 @@ import FlowCard from '../common/FlowCard.svelte' import { copyFirstStepSchema } from '../flowStore' import type { FlowEditorContext } from '../types' - import CapturePayload from './CapturePayload.svelte' import Drawer from '$lib/components/common/drawer/Drawer.svelte' import SimpleEditor from '$lib/components/SimpleEditor.svelte' import { convert } from '@redocly/json-to-json-schema' @@ -13,14 +12,17 @@ import EditableSchemaForm from '$lib/components/EditableSchemaForm.svelte' import AddProperty from '$lib/components/schema/AddProperty.svelte' import FlowInputViewer from '$lib/components/FlowInputViewer.svelte' + import Tabs from '$lib/components/common/tabs/Tabs.svelte' + import Tab from '$lib/components/common/tabs/Tab.svelte' + import CapturePanel from '$lib/components/triggers/CapturePanel.svelte' + import { insertNewPreprocessorModule } from '../flowStateUtils' export let noEditor: boolean export let disabled: boolean - const { flowStore, flowStateStore, previewArgs, initialPath } = + const { flowStore, flowStateStore, previewArgs, initialPath, pathStore, selectedId } = getContext('FlowEditorContext') - let capturePayload: CapturePayload let inputLibraryDrawer: Drawer let jsonPayload: Drawer let pendingJson: string @@ -39,73 +41,95 @@ jsonPayload.closeDrawer() } const yOffset = 191 - - + let tabSelected = 'input' + {#if !disabled} -
-
Copy input's schema from
- + + +
+
+ { + $flowStore = $flowStore + }} + /> +
+ + { + addProperty?.openDrawer(e.detail) }} - > - A request - - - - - -
- { - $flowStore = $flowStore + on:updateSchema={(e) => { + const { schema, redirect } = e.detail + $flowStore.schema = schema + if (redirect) { + tabSelected = 'input' + } }} /> -
- - { - addProperty?.openDrawer(e.detail) - }} - on:delete={(e) => { - addProperty?.handleDeleteArgument([e.detail]) - }} - offset={yOffset} - displayWebhookWarning - /> + {/if} {:else}
diff --git a/frontend/src/lib/components/flows/content/FlowPathViewer.svelte b/frontend/src/lib/components/flows/content/FlowPathViewer.svelte index 40af045bdfa1c..b80df515ede57 100644 --- a/frontend/src/lib/components/flows/content/FlowPathViewer.svelte +++ b/frontend/src/lib/components/flows/content/FlowPathViewer.svelte @@ -21,7 +21,8 @@ primarySchedule: primaryScheduleStore, selectedTrigger: selectedTriggerStore, triggersCount: triggersCount, - simplifiedPoll: writable(false) + simplifiedPoll: writable(false), + defaultValues: writable(undefined) }) async function loadFlow(path: string) { diff --git a/frontend/src/lib/components/flows/flowStateUtils.ts b/frontend/src/lib/components/flows/flowStateUtils.ts index 646506514524c..e958adac22e55 100644 --- a/frontend/src/lib/components/flows/flowStateUtils.ts +++ b/frontend/src/lib/components/flows/flowStateUtils.ts @@ -18,6 +18,7 @@ import { NEVER_TESTED_THIS_FAR } from './models' import { loadSchemaFromModule } from './flowInfers' import { nextId } from './flowModuleNextId' import { findNextAvailablePath } from '$lib/path' +import type { ExtendedOpenFlow } from './types' export async function loadFlowModuleState(flowModule: FlowModule): Promise { try { @@ -74,7 +75,7 @@ export async function pickFlow( export async function createInlineScriptModule( language: RawScript['language'], kind: Script['kind'], - subkind: 'pgsql' | 'flow', + subkind: 'pgsql' | 'flow' | 'preprocessor', id: string, summary?: string ): Promise<[FlowModule, FlowModuleState]> { @@ -292,3 +293,40 @@ export function sliceModules( return m }) } + +export async function insertNewPreprocessorModule( + flowStore: Writable, + flowStateStore: Writable, + inlineScript?: { + language: RawScript['language'] + subkind: 'preprocessor' + }, + wsScript?: { path: string; summary: string; hash: string | undefined } +) { + var module: FlowModule = { + id: 'preprocessor', + value: { type: 'identity' } + } + var state = emptyFlowModuleState() + + if (inlineScript) { + ;[module, state] = await createInlineScriptModule( + inlineScript.language, + 'script', + inlineScript.subkind, + 'preprocessor' + ) + } else if (wsScript) { + ;[module, state] = await pickScript(wsScript.path, wsScript.summary, module.id, wsScript.hash) + } + + flowStore.update((fs) => { + fs.value.preprocessor_module = module + return fs + }) + + flowStateStore.update((fss) => { + fss[module.id] = state + return fss + }) +} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 8e53a1030c992..a34a2806dec30 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -10,7 +10,8 @@ deleteFlowStateById, emptyModule, pickScript, - pickFlow + pickFlow, + insertNewPreprocessorModule } from '$lib/components/flows/flowStateUtils' import type { FlowModule, RawScript, Script } from '$lib/gen' import { emptyFlowModuleState, initFlowStepWarnings } from '../utils' @@ -133,34 +134,6 @@ return modules } - async function insertNewPreprocessorModule( - inlineScript?: { - language: RawScript['language'] - subkind: 'pgsql' | 'flow' - }, - wsScript?: { path: string; summary: string; hash: string | undefined } - ) { - var module: FlowModule = { - id: 'preprocessor', - value: { type: 'identity' } - } - var state = emptyFlowModuleState() - - if (inlineScript) { - ;[module, state] = await createInlineScriptModule( - inlineScript.language, - 'script', - inlineScript.subkind, - 'preprocessor' - ) - } else if (wsScript) { - ;[module, state] = await pickScript(wsScript.path, wsScript.summary, module.id, wsScript.hash) - } - - $flowStore.value.preprocessor_module = module - $flowStateStore[module.id] = state - } - function removeAtId(modules: FlowModule[], id: string): FlowModule[] { const index = modules.findIndex((mod) => mod.id == id) if (index != -1) { @@ -389,7 +362,12 @@ $moving = undefined } else { if (detail.detail === 'preprocessor') { - insertNewPreprocessorModule(detail.inlineScript, detail.script) + insertNewPreprocessorModule( + flowStore, + flowStateStore, + detail.inlineScript, + detail.script + ) $selectedId = 'preprocessor' } else { const index = detail.index ?? 0 diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte index c17bea1026030..3eab0f4d7a029 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte @@ -22,7 +22,7 @@
- + {#if newItem || cloudDisabled} + +
+ {#if cloudDisabled} + + {capitalize(selected)} triggers are disabled in the multi-tenant cloud. + + {:else} + {#if selected in schemas} + {#key selected} + + {/key} + {/if} - {#if selected in schemas} - - {/if} - - {#if selected === 'webhook'} - - + - {:else if selected === 'http'} - - + {:else if selected === 'http'} + + - {:else if selected === 'email'} - - {/if} + language={bash} + /> + + {:else if selected === 'email'} + + {/if} - + {/if} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 9f78127b45347..d3733efa99ca6 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -511,7 +511,7 @@ export async function main(approver?: string) { // add a form in Advanced - Suspend // all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` -const BUN_PREPROCESSOR_MODULE_CODE = ` +export const BUN_PREPROCESSOR_MODULE_CODE = ` export async function preprocessor( wm_trigger: { kind: 'http' | 'email' | 'webhook' | 'websocket' | 'kafka', @@ -535,7 +535,7 @@ export async function preprocessor( /* your other args */ ) { return { - // return the args to be passed to the flow + // return the args to be passed to the runnable } } ` @@ -564,7 +564,7 @@ export async function preprocessor( /* your other args */ ) { return { - // return the args to be passed to the flow + // return the args to be passed to the runnable } } ` @@ -596,7 +596,7 @@ def main(): # add a form in Advanced - Suspend # all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` -const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal +export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal class Http(TypedDict): route: str # The route path, e.g. "/users/:id" @@ -625,7 +625,7 @@ def preprocessor( # your other args ): return { - # return the args to be passed to the flow + # return the args to be passed to the runnable } ` diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index f7d83181339ef..1a8be27726fd3 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -585,8 +585,8 @@ export function addWhitespaceBeforeCapitals(word?: string): string { return word.replace(/([A-Z])/g, ' $1').trim() } -export function isObject(obj: any) { - return obj != null && typeof obj === 'object' +export function isObject(obj: any): obj is Record { + return obj != null && typeof obj === 'object' && !Array.isArray(obj) } export function debounce(func: (...args: any[]) => any, wait: number) { @@ -877,11 +877,10 @@ function replaceFalseWithUndefinedRec(obj: any) { // If the value is false, replace it with undefined if (obj[key] === false) { // delete obj[key]; - obj[key] = undefined; - + obj[key] = undefined } else { // If the value is an object, call the function recursively - replaceFalseWithUndefinedRec(obj[key]); + replaceFalseWithUndefinedRec(obj[key]) } } } diff --git a/frontend/src/routes/flows/dev/+page.svelte b/frontend/src/routes/flows/dev/+page.svelte index cb984d1095f04..9e0b005da64f4 100644 --- a/frontend/src/routes/flows/dev/+page.svelte +++ b/frontend/src/routes/flows/dev/+page.svelte @@ -82,7 +82,8 @@ primarySchedule: primaryScheduleStore, selectedTrigger: selectedTriggerStore, triggersCount: triggersCount, - simplifiedPoll: writable(false) + simplifiedPoll: writable(false), + defaultValues: writable(undefined) }) setContext('FlowEditorContext', { @@ -226,6 +227,8 @@ console.error('issue parsing new change:', code, e) } } + + let flowPreviewButtons: FlowPreviewButtons @@ -251,7 +254,7 @@
- +
@@ -268,7 +271,18 @@ {/if} - + { + if (ev.detail.kind === 'preprocessor') { + $testStepStore['preprocessor'] = ev.detail.args ?? {} + $selectedIdStore = 'preprocessor' + } else { + $previewArgsStore = ev.detail.args ?? {} + flowPreviewButtons?.openPreview() + } + }} + /> From a27f662d2a91e8e93a3640b2d0b30d2a7891bbeb Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 28 Nov 2024 17:49:29 +0100 Subject: [PATCH 03/60] fix: build --- backend/windmill-api/src/capture.rs | 7 ++- backend/windmill-api/src/kafka_triggers_ee.rs | 4 ++ backend/windmill-api/src/lib.rs | 3 -- .../windmill-api/src/websocket_triggers.rs | 53 +++++-------------- 4 files changed, 22 insertions(+), 45 deletions(-) diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index f819e0d0e116d..04f7c87df51dd 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -26,10 +26,11 @@ use windmill_common::{ }; use windmill_queue::{PushArgs, PushArgsOwned}; +#[cfg(all(feature = "enterprise", feature = "kafka"))] +use crate::kafka_triggers_ee::KafkaResourceSecurity; use crate::{ db::{ApiAuthed, DB}, http_triggers::build_http_trigger_extra, - kafka_triggers_ee::KafkaResourceSecurity, }; const KEEP_LAST: i64 = 20; @@ -87,6 +88,7 @@ struct HttpTriggerConfig { route_path: String, } +#[cfg(all(feature = "enterprise", feature = "kafka"))] #[derive(Serialize, Deserialize)] pub struct KafkaTriggerConfig { pub brokers: Vec, @@ -104,8 +106,9 @@ pub struct WebsocketTriggerConfig { #[serde(untagged)] enum TriggerConfig { Http(HttpTriggerConfig), - Kafka(KafkaTriggerConfig), Websocket(WebsocketTriggerConfig), + #[cfg(all(feature = "enterprise", feature = "kafka"))] + Kafka(KafkaTriggerConfig), } #[derive(Serialize, Deserialize)] diff --git a/backend/windmill-api/src/kafka_triggers_ee.rs b/backend/windmill-api/src/kafka_triggers_ee.rs index 3a03ebe864ed5..e3de5e6042c9c 100644 --- a/backend/windmill-api/src/kafka_triggers_ee.rs +++ b/backend/windmill-api/src/kafka_triggers_ee.rs @@ -1,5 +1,9 @@ use crate::db::DB; use axum::Router; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct KafkaResourceSecurity {} pub fn workspaced_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index f5b2b7e89806b..4f35de7fd3b84 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -222,9 +222,6 @@ pub async fn run_server( } } - // #[cfg(feature = "kafka")] - // start_listening().await; - let job_helpers_service = { #[cfg(feature = "parquet")] { diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 3580b1d440f05..b8e6f2787a5a6 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -369,7 +369,6 @@ async fn exists_websocket_trigger( async fn listen_to_unlistened_websockets( db: &DB, - rsmq: &Option, killpill_rx: &tokio::sync::broadcast::Receiver<()>, ) -> () { match sqlx::query_as::<_, WebsocketTrigger>( @@ -383,7 +382,7 @@ async fn listen_to_unlistened_websockets( Ok(mut triggers) => { triggers.shuffle(&mut rand::thread_rng()); for trigger in triggers { - trigger.maybe_listen_to_websocket(db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; + trigger.maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()).await; } } Err(err) => { @@ -401,7 +400,7 @@ async fn listen_to_unlistened_websockets( Ok(mut captures) => { captures.shuffle(&mut rand::thread_rng()); for capture in captures { - capture.maybe_listen_to_websocket(db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; + capture.maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()).await; } } Err(err) => { @@ -410,13 +409,9 @@ async fn listen_to_unlistened_websockets( } } -pub async fn start_websockets( - db: DB, - rsmq: Option, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { +pub async fn start_websockets(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { tokio::spawn(async move { - listen_to_unlistened_websockets(&db, &rsmq, &killpill_rx).await; + listen_to_unlistened_websockets(&db, &killpill_rx).await; loop { tokio::select! { biased; @@ -424,7 +419,7 @@ pub async fn start_websockets( return; } _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_websockets(&db, &rsmq, &killpill_rx).await; + listen_to_unlistened_websockets(&db, &killpill_rx).await; } } } @@ -504,7 +499,6 @@ async fn wait_runnable_result( ws_trigger: &WebsocketTrigger, username_override: String, db: &DB, - rsmq: Option, ) -> error::Result { let user_db = UserDB::new(db.clone()); let authed = fetch_api_authed( @@ -526,7 +520,6 @@ async fn wait_runnable_result( authed, db.clone(), user_db, - rsmq.clone(), ws_trigger.workspace_id.clone(), StripPath(path.clone()), RunJobQuery::default(), @@ -539,7 +532,6 @@ async fn wait_runnable_result( authed, db.clone(), user_db, - rsmq.clone(), ws_trigger.workspace_id.clone(), StripPath(path.clone()), RunJobQuery::default(), @@ -612,7 +604,6 @@ impl WebsocketTrigger { async fn maybe_listen_to_websocket( self, db: DB, - rsmq: Option, killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { match sqlx::query_scalar!( @@ -623,7 +614,7 @@ impl WebsocketTrigger { ).fetch_optional(&db).await { Ok(has_lock) => { if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Trigger(self), db, rsmq, killpill_rx)); + tokio::spawn(listen_to_websocket(WebsocketEnum::Trigger(self), db, killpill_rx)); } else { tracing::info!("Websocket {} already being listened to", self.url); } @@ -683,7 +674,6 @@ impl WebsocketTrigger { path: &str, is_flow: bool, db: &DB, - rsmq: Option, ) -> error::Result { tracing::info!("Running runnable {path} (is_flow: {is_flow}) to get websocket URL",); @@ -694,7 +684,6 @@ impl WebsocketTrigger { self, "url".to_string(), db, - rsmq, ) .await?; @@ -712,7 +701,6 @@ impl WebsocketTrigger { &self, mut writer: SplitSink>, Message>, db: &DB, - rsmq: Option, ) -> error::Result<()> { let initial_messages: Vec = self .initial_messages @@ -752,7 +740,6 @@ impl WebsocketTrigger { self, "init".to_string(), db, - rsmq.clone(), ) .await?; @@ -781,13 +768,8 @@ impl WebsocketTrigger { Ok(()) } - async fn handle( - &self, - db: &DB, - rsmq: Option, - args: PushArgsOwned, - ) -> () { - if let Err(err) = run_job(db, rsmq, self, args).await { + async fn handle(&self, db: &DB, args: PushArgsOwned) -> () { + if let Err(err) = run_job(db, self, args).await { report_critical_error( format!( "Failed to trigger job from websocket {}: {:?}", @@ -815,7 +797,6 @@ impl CaptureConfigForWebsocket { async fn maybe_listen_to_websocket( self, db: DB, - rsmq: Option, killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { match sqlx::query_scalar!( @@ -827,7 +808,7 @@ impl CaptureConfigForWebsocket { ).fetch_optional(&db).await { Ok(has_lock) => { if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Capture(self), db, rsmq, killpill_rx)); + tokio::spawn(listen_to_websocket(WebsocketEnum::Capture(self), db, killpill_rx)); } else { tracing::info!("Websocket {} already being listened to", self.trigger_config.url); } @@ -896,7 +877,6 @@ impl WebsocketEnum { async fn listen_to_websocket( ws: WebsocketEnum, db: DB, - rsmq: Option, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { if let None = ws.update_ping(&db, Some("Connecting")).await { @@ -933,7 +913,7 @@ async fn listen_to_websocket( )) => { return; }, - url_result = ws_trigger.get_url_from_runnable(path, url.starts_with("$flow:"), &db, rsmq.clone()) => match url_result { + url_result = ws_trigger.get_url_from_runnable(path, url.starts_with("$flow:"), &db) => match url_result { Ok(url) => Cow::Owned(url), Err(err) => { ws_trigger.disable_with_error(&db, format!( @@ -988,7 +968,7 @@ async fn listen_to_websocket( _ = async { match &ws { WebsocketEnum::Trigger(ws_trigger) => { - if let Err(err) = ws_trigger.send_initial_messages(writer, &db, rsmq.clone()).await { + if let Err(err) = ws_trigger.send_initial_messages(writer, &db).await { ws_trigger.disable_with_error(&db, format!("Error sending initial messages: {:?}", err)).await; } else { // if initial messages sent successfully, wait forever @@ -1050,7 +1030,7 @@ async fn listen_to_websocket( let args = PushArgsOwned { args, extra }; match &ws { WebsocketEnum::Trigger(ws_trigger) => { - ws_trigger.handle(&db, rsmq.clone(), args).await; + ws_trigger.handle(&db, args).await; }, WebsocketEnum::Capture(capture) => { capture.handle(&db, args).await; @@ -1100,12 +1080,7 @@ async fn listen_to_websocket( } } -async fn run_job( - db: &DB, - rsmq: Option, - trigger: &WebsocketTrigger, - args: PushArgsOwned, -) -> anyhow::Result<()> { +async fn run_job(db: &DB, trigger: &WebsocketTrigger, args: PushArgsOwned) -> anyhow::Result<()> { let label_prefix = Some(format!("ws-{}-", trigger.path)); let authed = fetch_api_authed( @@ -1126,7 +1101,6 @@ async fn run_job( authed, db.clone(), user_db, - rsmq, trigger.workspace_id.clone(), StripPath(trigger.script_path.to_owned()), run_query, @@ -1139,7 +1113,6 @@ async fn run_job( authed, db.clone(), user_db, - rsmq, trigger.workspace_id.clone(), StripPath(trigger.script_path.to_owned()), run_query, From b5f72b2746d120da6b634ed1d75345a674179fe9 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 28 Nov 2024 18:41:53 +0100 Subject: [PATCH 04/60] fix sqlx --- ...74e5ef0aaa2505facbea8c764003dfc8fffb1.json | 26 ++++++++ ...254c783fc34b617c8a9a95a0eb0cda535dab5.json | 33 ++++++++++ ...ffca075a0550eada87df7162c5037164ad6bf.json | 16 ----- ...591b28cbf9ea8f61d379b84ee6e14c033035d.json | 17 ++++++ ...770e6341624e98117d21b9f01e68b4e0ce033.json | 14 +++++ ...bd0de7e03c539ee046955543d9693551246f7.json | 14 +++++ ...290278dff07f689d8f192aea3c5b71a6785c2.json | 61 +++++++++++++++++++ ...451217ccb9f1bc35b1ad6e10d16bc19c41447.json | 44 +++++++++++++ ...4e44ef5a0f082bdde854900064325adc4dd77.json | 15 +++++ ...120444af65b1dc43a234821cde5bf6bf8b74f.json | 27 -------- ...e6e3ae6c5add6ca02414140adb724120a6800.json | 16 ----- ...677774aa237508d5610714efd2e9b8b93c7b8.json | 55 +++++++++++++++++ ...d3ef34fa257e794e8886c2234317fcdc1c454.json | 44 +++++++++++++ ...0da43239c9a5aaea41c9aed7ed33a6219a534.json | 30 +++++++++ ...96ab6b08def5c93c2744435d211abdfd3e2d5.json | 44 +++++++++++++ ...bceffac637548841897341672da427a9140fc.json | 26 ++++++++ ...f59c8cff67bb1ec3926680fb691cc3573738a.json | 16 ----- ...3ab5a528b0468715238e17ffed7d75e9c0c5c.json | 25 ++++++++ ...c90ac1c18b5df7aab6a143a328a6bddc6ad32.json | 25 ++++++++ ...a74407b78378d66d8a089407998074059e79b.json | 33 ++++++++++ ...2ee3619b5818bec3600cca3dd49ce0dd4bfe7.json | 44 +++++++++++++ 21 files changed, 550 insertions(+), 75 deletions(-) create mode 100644 backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json create mode 100644 backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json delete mode 100644 backend/.sqlx/query-0a9a191273c735c41d56ea46a39ffca075a0550eada87df7162c5037164ad6bf.json create mode 100644 backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json create mode 100644 backend/.sqlx/query-3c9fc4d8579767f3ce7c3633fca770e6341624e98117d21b9f01e68b4e0ce033.json create mode 100644 backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json create mode 100644 backend/.sqlx/query-640897c5a33e87e1dfd66f9c642290278dff07f689d8f192aea3c5b71a6785c2.json create mode 100644 backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json create mode 100644 backend/.sqlx/query-97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77.json delete mode 100644 backend/.sqlx/query-aa98ab0e4b9a0eb41a804ab047a120444af65b1dc43a234821cde5bf6bf8b74f.json delete mode 100644 backend/.sqlx/query-b9468b9e16f55db11b33d8e9793e6e3ae6c5add6ca02414140adb724120a6800.json create mode 100644 backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json create mode 100644 backend/.sqlx/query-c3ba14914756231efd60e9dd3aad3ef34fa257e794e8886c2234317fcdc1c454.json create mode 100644 backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json create mode 100644 backend/.sqlx/query-d0fdc2c1ffd3c0fa0b973e8ac6396ab6b08def5c93c2744435d211abdfd3e2d5.json create mode 100644 backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json delete mode 100644 backend/.sqlx/query-e02b99525cb1f8737acfec86809f59c8cff67bb1ec3926680fb691cc3573738a.json create mode 100644 backend/.sqlx/query-e86295e181a82823ffce8234d413ab5a528b0468715238e17ffed7d75e9c0c5c.json create mode 100644 backend/.sqlx/query-ee9adcbf82d3f62088a38ff65e8c90ac1c18b5df7aab6a143a328a6bddc6ad32.json create mode 100644 backend/.sqlx/query-ef299490c4674c4c76e18d84620a74407b78378d66d8a089407998074059e79b.json create mode 100644 backend/.sqlx/query-f8fc1c07c70ccf4ca46540967412ee3619b5818bec3600cca3dd49ce0dd4bfe7.json diff --git a/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json b/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json new file mode 100644 index 0000000000000..8fce6ae729d8d --- /dev/null +++ b/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1" +} diff --git a/backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json b/backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json new file mode 100644 index 0000000000000..a86748af87145 --- /dev/null +++ b/backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO capture (workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5" +} diff --git a/backend/.sqlx/query-0a9a191273c735c41d56ea46a39ffca075a0550eada87df7162c5037164ad6bf.json b/backend/.sqlx/query-0a9a191273c735c41d56ea46a39ffca075a0550eada87df7162c5037164ad6bf.json deleted file mode 100644 index de6e87bffd8ff..0000000000000 --- a/backend/.sqlx/query-0a9a191273c735c41d56ea46a39ffca075a0550eada87df7162c5037164ad6bf.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO capture\n (workspace_id, path, created_by)\n VALUES ($1, $2, $3)\n ON CONFLICT (workspace_id, path)\n DO UPDATE SET created_at = now()\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "0a9a191273c735c41d56ea46a39ffca075a0550eada87df7162c5037164ad6bf" -} diff --git a/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json b/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json new file mode 100644 index 0000000000000..65b1b24efb7d8 --- /dev/null +++ b/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d" +} diff --git a/backend/.sqlx/query-3c9fc4d8579767f3ce7c3633fca770e6341624e98117d21b9f01e68b4e0ce033.json b/backend/.sqlx/query-3c9fc4d8579767f3ce7c3633fca770e6341624e98117d21b9f01e68b4e0ce033.json new file mode 100644 index 0000000000000..7b0d8aa4b0353 --- /dev/null +++ b/backend/.sqlx/query-3c9fc4d8579767f3ce7c3633fca770e6341624e98117d21b9f01e68b4e0ce033.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM capture_config WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "3c9fc4d8579767f3ce7c3633fca770e6341624e98117d21b9f01e68b4e0ce033" +} diff --git a/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json b/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json new file mode 100644 index 0000000000000..0df0167c42d2b --- /dev/null +++ b/backend/.sqlx/query-41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM capture WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "41e557e1b63b13c9fcc195901c0bd0de7e03c539ee046955543d9693551246f7" +} diff --git a/backend/.sqlx/query-640897c5a33e87e1dfd66f9c642290278dff07f689d8f192aea3c5b71a6785c2.json b/backend/.sqlx/query-640897c5a33e87e1dfd66f9c642290278dff07f689d8f192aea3c5b71a6785c2.json new file mode 100644 index 0000000000000..c91cef65bd7c3 --- /dev/null +++ b/backend/.sqlx/query-640897c5a33e87e1dfd66f9c642290278dff07f689d8f192aea3c5b71a6785c2.json @@ -0,0 +1,61 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, created_at, trigger_kind as \"trigger_kind: _\", payload as \"payload: _\", trigger_extra as \"trigger_extra: _\"\n FROM capture\n WHERE workspace_id = $1\n AND path = $2 AND is_flow = $3\n ORDER BY created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "payload: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "trigger_extra: _", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "640897c5a33e87e1dfd66f9c642290278dff07f689d8f192aea3c5b71a6785c2" +} diff --git a/backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json b/backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json new file mode 100644 index 0000000000000..1f23430419f27 --- /dev/null +++ b/backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT owner, email\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + } + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447" +} diff --git a/backend/.sqlx/query-97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77.json b/backend/.sqlx/query-97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77.json new file mode 100644 index 0000000000000..2014d9e6e495b --- /dev/null +++ b/backend/.sqlx/query-97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM capture\n WHERE workspace_id = $1\n AND created_at <=\n (\n SELECT created_at\n FROM capture\n WHERE workspace_id = $1\n ORDER BY created_at DESC\n OFFSET $2\n LIMIT 1\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "97942578df746c8c8103b403cfc4e44ef5a0f082bdde854900064325adc4dd77" +} diff --git a/backend/.sqlx/query-aa98ab0e4b9a0eb41a804ab047a120444af65b1dc43a234821cde5bf6bf8b74f.json b/backend/.sqlx/query-aa98ab0e4b9a0eb41a804ab047a120444af65b1dc43a234821cde5bf6bf8b74f.json deleted file mode 100644 index d70e81e786391..0000000000000 --- a/backend/.sqlx/query-aa98ab0e4b9a0eb41a804ab047a120444af65b1dc43a234821cde5bf6bf8b74f.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH existing AS (\n SELECT id FROM flow_node\n WHERE hash = $1 AND path = $2 AND workspace_id = $3 AND code = $4 AND lock = $5 AND flow = $6\n LIMIT 1\n ),\n inserted AS (\n INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n RETURNING id\n )\n SELECT id FROM existing\n UNION ALL\n SELECT id FROM inserted\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Text", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "aa98ab0e4b9a0eb41a804ab047a120444af65b1dc43a234821cde5bf6bf8b74f" -} diff --git a/backend/.sqlx/query-b9468b9e16f55db11b33d8e9793e6e3ae6c5add6ca02414140adb724120a6800.json b/backend/.sqlx/query-b9468b9e16f55db11b33d8e9793e6e3ae6c5add6ca02414140adb724120a6800.json deleted file mode 100644 index 8f9f3dfc89c7d..0000000000000 --- a/backend/.sqlx/query-b9468b9e16f55db11b33d8e9793e6e3ae6c5add6ca02414140adb724120a6800.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE capture\n SET payload = $3\n WHERE workspace_id = $1\n AND path = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "b9468b9e16f55db11b33d8e9793e6e3ae6c5add6ca02414140adb724120a6800" -} diff --git a/backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json b/backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json new file mode 100644 index 0000000000000..211e8fa8abd89 --- /dev/null +++ b/backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_config as \"trigger_config: _\", trigger_kind as \"trigger_kind: _\", error, last_server_ping\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "trigger_kind: _", + "type_info": { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "last_server_ping", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + true, + false, + true, + true + ] + }, + "hash": "c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8" +} diff --git a/backend/.sqlx/query-c3ba14914756231efd60e9dd3aad3ef34fa257e794e8886c2234317fcdc1c454.json b/backend/.sqlx/query-c3ba14914756231efd60e9dd3aad3ef34fa257e794e8886c2234317fcdc1c454.json new file mode 100644 index 0000000000000..04586025c6ab5 --- /dev/null +++ b/backend/.sqlx/query-c3ba14914756231efd60e9dd3aad3ef34fa257e794e8886c2234317fcdc1c454.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trigger_config as \"trigger_config: _\", owner\n FROM capture_config\n WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4 AND last_client_ping > NOW() - INTERVAL '10 seconds'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trigger_config: _", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + } + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "c3ba14914756231efd60e9dd3aad3ef34fa257e794e8886c2234317fcdc1c454" +} diff --git a/backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json b/backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json new file mode 100644 index 0000000000000..52bb869ca651b --- /dev/null +++ b/backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET last_client_ping = now() WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534" +} diff --git a/backend/.sqlx/query-d0fdc2c1ffd3c0fa0b973e8ac6396ab6b08def5c93c2744435d211abdfd3e2d5.json b/backend/.sqlx/query-d0fdc2c1ffd3c0fa0b973e8ac6396ab6b08def5c93c2744435d211abdfd3e2d5.json new file mode 100644 index 0000000000000..6817a3faf4f21 --- /dev/null +++ b/backend/.sqlx/query-d0fdc2c1ffd3c0fa0b973e8ac6396ab6b08def5c93c2744435d211abdfd3e2d5.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "trigger_config!: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + false + ] + }, + "hash": "d0fdc2c1ffd3c0fa0b973e8ac6396ab6b08def5c93c2744435d211abdfd3e2d5" +} diff --git a/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json b/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json new file mode 100644 index 0000000000000..3ccfcc95820d0 --- /dev/null +++ b/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc" +} diff --git a/backend/.sqlx/query-e02b99525cb1f8737acfec86809f59c8cff67bb1ec3926680fb691cc3573738a.json b/backend/.sqlx/query-e02b99525cb1f8737acfec86809f59c8cff67bb1ec3926680fb691cc3573738a.json deleted file mode 100644 index d600d8069a8ba..0000000000000 --- a/backend/.sqlx/query-e02b99525cb1f8737acfec86809f59c8cff67bb1ec3926680fb691cc3573738a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM capture\n WHERE workspace_id = $1\n AND created_by = $2\n AND created_at <=\n ( SELECT created_at\n FROM capture\n WHERE workspace_id = $1\n AND created_by = $2\n ORDER BY created_at DESC\n OFFSET $3\n LIMIT 1 )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "e02b99525cb1f8737acfec86809f59c8cff67bb1ec3926680fb691cc3573738a" -} diff --git a/backend/.sqlx/query-e86295e181a82823ffce8234d413ab5a528b0468715238e17ffed7d75e9c0c5c.json b/backend/.sqlx/query-e86295e181a82823ffce8234d413ab5a528b0468715238e17ffed7d75e9c0c5c.json new file mode 100644 index 0000000000000..42f6b47554523 --- /dev/null +++ b/backend/.sqlx/query-e86295e181a82823ffce8234d413ab5a528b0468715238e17ffed7d75e9c0c5c.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now() WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e86295e181a82823ffce8234d413ab5a528b0468715238e17ffed7d75e9c0c5c" +} diff --git a/backend/.sqlx/query-ee9adcbf82d3f62088a38ff65e8c90ac1c18b5df7aab6a143a328a6bddc6ad32.json b/backend/.sqlx/query-ee9adcbf82d3f62088a38ff65e8c90ac1c18b5df7aab6a143a328a6bddc6ad32.json new file mode 100644 index 0000000000000..1c1179b188463 --- /dev/null +++ b/backend/.sqlx/query-ee9adcbf82d3f62088a38ff65e8c90ac1c18b5df7aab6a143a328a6bddc6ad32.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now() WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ee9adcbf82d3f62088a38ff65e8c90ac1c18b5df7aab6a143a328a6bddc6ad32" +} diff --git a/backend/.sqlx/query-ef299490c4674c4c76e18d84620a74407b78378d66d8a089407998074059e79b.json b/backend/.sqlx/query-ef299490c4674c4c76e18d84620a74407b78378d66d8a089407998074059e79b.json new file mode 100644 index 0000000000000..a3301cfb9a7c9 --- /dev/null +++ b/backend/.sqlx/query-ef299490c4674c4c76e18d84620a74407b78378d66d8a089407998074059e79b.json @@ -0,0 +1,33 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO capture_config\n (workspace_id, path, is_flow, trigger_kind, trigger_config, owner, email)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, path, is_flow, trigger_kind)\n DO UPDATE SET trigger_config = $5, owner = $6, email = $7, server_id = NULL, last_server_ping = NULL, error = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email" + ] + } + } + }, + "Jsonb", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ef299490c4674c4c76e18d84620a74407b78378d66d8a089407998074059e79b" +} diff --git a/backend/.sqlx/query-f8fc1c07c70ccf4ca46540967412ee3619b5818bec3600cca3dd49ce0dd4bfe7.json b/backend/.sqlx/query-f8fc1c07c70ccf4ca46540967412ee3619b5818bec3600cca3dd49ce0dd4bfe7.json new file mode 100644 index 0000000000000..3970e4b776b7a --- /dev/null +++ b/backend/.sqlx/query-f8fc1c07c70ccf4ca46540967412ee3619b5818bec3600cca3dd49ce0dd4bfe7.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner FROM capture_config WHERE trigger_kind = 'kafka' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "trigger_config!: _", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + true, + false + ] + }, + "hash": "f8fc1c07c70ccf4ca46540967412ee3619b5818bec3600cca3dd49ce0dd4bfe7" +} From c5b96d7118a3de8b7440175b81bca8a1638fa2ac Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 3 Dec 2024 19:33:23 +0000 Subject: [PATCH 05/60] Move Capture WIP --- frontend/src/lib/components/Section.svelte | 73 +-- .../components/triggers/CaptureWrapper.svelte | 554 ++++++++++++++++++ .../triggers/RouteEditorConfigSection.svelte | 166 ++++++ .../triggers/RouteEditorInner.svelte | 123 +--- .../components/triggers/RoutesPanel.svelte | 102 ++-- .../triggers/TriggersEditorSection.svelte | 105 ++++ frontend/tsconfig.json | 2 +- 7 files changed, 930 insertions(+), 195 deletions(-) create mode 100644 frontend/src/lib/components/triggers/CaptureWrapper.svelte create mode 100644 frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte create mode 100644 frontend/src/lib/components/triggers/TriggersEditorSection.svelte diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index 43868d170e27b..1889fc708a8b1 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -10,48 +10,51 @@ export let small: boolean = false export let collapsable: boolean = false - let collapsed: boolean = true + export let collapsed: boolean = true + export let headless: boolean = false
-
-

- {#if collapsable} - + {:else} {label} - - {:else} - {label} - {/if} + {/if} - - {#if tooltip} - {tooltip} - {/if} - {#if eeOnly} - {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
+ + {#if tooltip} + {tooltip} {/if} + {#if eeOnly} + {#if !$enterpriseLicense} +
+ + EE only Enterprise Edition only feature +
+ {/if} + {/if} +

+ + {#if collapsable && collapsed} + {/if} - - - {#if collapsable && collapsed} - - {/if} -
+
+ {/if}
diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte new file mode 100644 index 0000000000000..8654986c6a969 --- /dev/null +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -0,0 +1,554 @@ + + +
+ {#if (captureType === 'websocket' || captureType === 'kafka') && config && active} + {@const serverEnabled = getServerEnabled(config)} +
+
+
+ {#if serverEnabled} + + + + +
Websocket is connected
+ + {:else} + + + + + +
+ Websocket is not connected{config.error ? ': ' + config.error : ''} +
+ + {/if} +
+
+
+ {/if} + {#if cloudDisabled} + + {capitalize(captureType)} triggers are disabled in the multi-tenant cloud. + + {:else} + {#if captureType in schemas && false} + {#key captureType} + + {/key} + {/if} + + {#if captureType === 'webhook'} + + + {:else if captureType === 'http'} + + {:else if captureType === 'email'} + + {/if} + + {#if captureMode} + + {/if} + {/if} +
diff --git a/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte b/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte new file mode 100644 index 0000000000000..892020566f092 --- /dev/null +++ b/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte @@ -0,0 +1,166 @@ + + +
+ {#if !($userStore?.is_admin || $userStore?.is_super_admin)} + + Route endpoints can only be edited by workspace admins + +
+ {/if} +
+ + + + + + + + + +
+
+ + Full endpoint + + { + currentTarget.select() + }} + /> +
+ +
{dirtyRoutePath ? routeError : ''}
+
+ {#if captureMode} + + {/if} +
+
diff --git a/frontend/src/lib/components/triggers/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/RouteEditorInner.svelte index c9e38b1147186..be0c76044d74e 100644 --- a/frontend/src/lib/components/triggers/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/RouteEditorInner.svelte @@ -1,5 +1,5 @@ {#if static_asset_config} @@ -253,76 +222,16 @@ -
- {#if !($userStore?.is_admin || $userStore?.is_super_admin)} - - Route endpoints can only be edited by workspace admins - -
- {/if} -
- - - - - - - - - -
-
- - Full endpoint - - { - currentTarget.select() - }} - /> -
- -
{dirtyRoutePath ? routeError : ''}
-
-
-
+
- import { Button } from '../common' import { userStore, workspaceStore } from '$lib/stores' import { HttpTriggerService, type HttpTrigger } from '$lib/gen' - import { RouteIcon } from 'lucide-svelte' - import Skeleton from '../common/skeleton/Skeleton.svelte' import RouteEditor from './RouteEditor.svelte' import { canWrite } from '$lib/utils' import Alert from '../common/alert/Alert.svelte' import type { TriggerContext } from '../triggers' import { getContext, onMount } from 'svelte' + import Section from '$lib/components/Section.svelte' + import TriggersEditorSection from './TriggersEditorSection.svelte' export let isFlow: boolean export let path: string @@ -58,57 +57,56 @@ bind:this={routeEditor} /> -
- {#if !newItem} - {#if $userStore?.is_admin || $userStore?.is_super_admin} - - {:else} - +
+ { + routeEditor?.openNew(isFlow, path, e.detail.config) + }} + cloudDisabled={false} + captureType="webhook" + {isFlow} + /> +
+ {#if !newItem} + {#if !$userStore?.is_admin && !$userStore?.is_super_admin} + + {/if} {/if} - {/if} - - {#if httpTriggers} - {#if httpTriggers.length == 0} -
No http routes
- {:else} -
- {#each httpTriggers as httpTriggers (httpTriggers.path)} -
-
{httpTriggers.path}
-
- {httpTriggers.http_method.toUpperCase()} /{httpTriggers.route_path} + {#if httpTriggers} + {#if httpTriggers.length == 0} +
No http routes
+ {:else} +
+ {#each httpTriggers as httpTriggers (httpTriggers.path)} +
+
{httpTriggers.path}
+
+ {httpTriggers.http_method.toUpperCase()} /{httpTriggers.route_path} +
+
+ +
-
- -
-
- {/each} -
+ {/each} +
+ {/if} + {:else} + {/if} - {:else} - - {/if} - {#if newItem} - - Deploy the {isFlow ? 'flow' : 'script'} to add http routes. - - {/if} + {#if newItem} + + Deploy the {isFlow ? 'flow' : 'script'} to add http routes. + + {/if} +
diff --git a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte new file mode 100644 index 0000000000000..50470583ca818 --- /dev/null +++ b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte @@ -0,0 +1,105 @@ + + +
+ +
+ +
+ {#if captureMode} + + {/if} + + {#if newItem || cloudDisabled} + +
+
+ + + { + console.log('dbg add preprocessor') + $selectedId = 'preprocessor' + }} + on:updateSchema={(e) => { + const { schema, redirect } = e.detail + $flowStore.schema = schema + if (redirect) { + //tabSelected = 'input' + console.log('dbg redirect') + } + }} + bind:args + {captureMode} + bind:handleCapture + bind:active + /> +
diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index c6e52bf449525..85bb4245adfb3 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -31,6 +31,6 @@ "strict": true, "types": [] }, - "include": ["src/**/*.js", "src/**/*.ts", "src/**/*.d.ts", "src/**/*.svelte"], + "include": ["src/**/*.js", "src/**/*.ts", "src/**/*.d.ts", "src/**/*.svelte", "src/lib/components/triggers/TriggersEditorSectionsvelte"], "extends": "./.svelte-kit/tsconfig.json" } From 8f3007db2ef2478bf9a58775cbb804b7ebe7b178 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 4 Dec 2024 11:34:43 +0000 Subject: [PATCH 06/60] Add capture to webhook and websocket --- frontend/src/lib/components/Label.svelte | 4 +- .../components/triggers/CaptureWrapper.svelte | 30 +- .../components/triggers/RoutesPanel.svelte | 2 +- .../triggers/TriggersEditorSection.svelte | 63 ++- .../triggers/WebhooksConfigSection.svelte | 454 ++++++++++++++++++ .../components/triggers/WebhooksPanel.svelte | 386 +-------------- .../WebsocketEditorConfigSection.svelte | 112 +++++ .../WebsocketTriggerEditorInner.svelte | 97 +--- .../triggers/WebsocketTriggersPanel.svelte | 93 ++-- 9 files changed, 690 insertions(+), 551 deletions(-) create mode 100644 frontend/src/lib/components/triggers/WebhooksConfigSection.svelte create mode 100644 frontend/src/lib/components/triggers/WebsocketEditorConfigSection.svelte diff --git a/frontend/src/lib/components/Label.svelte b/frontend/src/lib/components/Label.svelte index 3d53070c6c8c1..f53a8d9ca24dd 100644 --- a/frontend/src/lib/components/Label.svelte +++ b/frontend/src/lib/components/Label.svelte @@ -1,9 +1,11 @@ -
+
{label} diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte index 8654986c6a969..df7877629b477 100644 --- a/frontend/src/lib/components/triggers/CaptureWrapper.svelte +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -28,6 +28,8 @@ import { convert } from '@redocly/json-to-json-schema' import SchemaViewer from '../SchemaViewer.svelte' import RouteEditorConfigSection from './RouteEditorConfigSection.svelte' + import WebsocketEditorConfigSection from './WebsocketEditorConfigSection.svelte' + import WebhooksConfigSection from './WebhooksConfigSection.svelte' export let isFlow: boolean export let path: string @@ -36,8 +38,8 @@ export let captureType: CaptureTriggerKind = 'webhook' export let captureMode = false export let active = false + export let data: any = {} - $: console.log('dbg isFlow', isFlow) const dispatch = createEventDispatcher<{ openTriggers: { kind: TriggerKind @@ -333,7 +335,6 @@ } export async function handleCapture() { - console.log('dbg handleCapture') if (!active) { await setConfig() capture() @@ -392,19 +393,18 @@ {/key} {/if} - {#if captureType === 'webhook'} - - + {#if captureType === 'websocket'} + + {:else if captureType === 'webhook'} + {:else if captureType === 'http'} {:else if captureType === 'email'} diff --git a/frontend/src/lib/components/triggers/RoutesPanel.svelte b/frontend/src/lib/components/triggers/RoutesPanel.svelte index e073b00910646..e0fe43502b884 100644 --- a/frontend/src/lib/components/triggers/RoutesPanel.svelte +++ b/frontend/src/lib/components/triggers/RoutesPanel.svelte @@ -63,7 +63,7 @@ routeEditor?.openNew(isFlow, path, e.detail.config) }} cloudDisabled={false} - captureType="webhook" + captureType="http" {isFlow} />
diff --git a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte index 50470583ca818..12609e6666e25 100644 --- a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte +++ b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte @@ -13,6 +13,16 @@ export let cloudDisabled: boolean export let captureType: CaptureTriggerKind export let isFlow: boolean = false + export let data: any = {} + export let noSave = false + + const captureTypeLabels: Record = { + http: 'Route', + websocket: 'Websocket', + webhook: 'Webhook', + kafka: 'Kafka', + email: 'Email' + } let args: Record = {} let captureMode = false @@ -28,7 +38,7 @@ let handleCapture: (() => Promise) | undefined -
+
@@ -45,34 +55,36 @@ {/if} - {#if newItem || cloudDisabled} - + {#if !noSave} + {#if newItem || cloudDisabled} + +
@@ -81,7 +93,7 @@
diff --git a/frontend/src/lib/components/triggers/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/WebhooksConfigSection.svelte new file mode 100644 index 0000000000000..bdf295ed33370 --- /dev/null +++ b/frontend/src/lib/components/triggers/WebhooksConfigSection.svelte @@ -0,0 +1,454 @@ + + + { + token = e.detail + triggerTokens?.listTokens() + }} + newTokenWorkspace={$workspaceStore} + newTokenLabel={`webhook-${$userStore?.username ?? 'superadmin'}-${generateRandomString(4)}`} + {scopes} +/> + +
+ {#if SCRIPT_VIEW_SHOW_CREATE_TOKEN_BUTTON} + + {/if} + +
+
+
Request type
+ + + + +
+
+
Call method
+ + + {#if !isFlow} + + {/if} + + + +
+
+
Token configuration
+ + + + +
+
+ + +
+ + REST + {#if SCRIPT_VIEW_SHOW_EXAMPLE_CURL} + Curl + {/if} + Fetch + + + {#key token} + +
+ + + {#if requestType !== 'get_path'} + + {/if} + {#key requestType} + {#key tokenType} + + {/key} + {/key} +
+ + +
+ {#key args} + {#key requestType} + {#key webhookType} + {#key tokenType} + {#key captureMode} +
{ + e.preventDefault() + copyToClipboard(curlCode()) + }} + > + + +
+ {/key} + {/key} + {/key} + {/key} + {/key} +
+ + + {#key args} + {#key requestType} + {#key webhookType} + {#key tokenType} + {#key token} +
{ + e.preventDefault() + copyToClipboard(fetchCode()) + }} + > + + +
+ {/key}{/key}{/key}{/key} + {/key} + + {/key} + + +
+ {#if !captureMode} + + {/if} +
diff --git a/frontend/src/lib/components/triggers/WebhooksPanel.svelte b/frontend/src/lib/components/triggers/WebhooksPanel.svelte index eefd785b47e70..78d948791854c 100644 --- a/frontend/src/lib/components/triggers/WebhooksPanel.svelte +++ b/frontend/src/lib/components/triggers/WebhooksPanel.svelte @@ -1,27 +1,7 @@ - { - token = e.detail - triggerTokens?.listTokens() - }} - newTokenWorkspace={$workspaceStore} - newTokenLabel={`webhook-${$userStore?.username ?? 'superadmin'}-${generateRandomString(4)}`} - {scopes} -/> -
- {#if SCRIPT_VIEW_SHOW_CREATE_TOKEN_BUTTON} - - {/if} - -
-
-
Request type
- - - - -
-
-
Call method
- - - {#if !isFlow} - - {/if} - - - -
-
-
Token configuration
- - - - -
-
- - - - REST - {#if SCRIPT_VIEW_SHOW_EXAMPLE_CURL} - Curl - {/if} - Fetch - - - {#key token} - -
- - - {#if requestType !== 'get_path'} - - {/if} - {#key requestType} - {#key tokenType} - - {/key} - {/key} -
- - -
- {#key args} - {#key requestType} - {#key webhookType} - {#key tokenType} -
{ - e.preventDefault() - copyToClipboard(curlCode()) - }} - > - - -
- {/key} - {/key} - {/key} - {/key} -
- - - {#key args} - {#key requestType} - {#key webhookType} - {#key tokenType} - {#key token} -
{ - e.preventDefault() - copyToClipboard(fetchCode()) - }} - > - - -
- {/key}{/key}{/key}{/key} - {/key} - - {/key} - - - -
- - {#if newItem}
@@ -398,4 +30,6 @@ done` {isFlow ? 'flow' : 'script'}. {/if} + +
diff --git a/frontend/src/lib/components/triggers/WebsocketEditorConfigSection.svelte b/frontend/src/lib/components/triggers/WebsocketEditorConfigSection.svelte new file mode 100644 index 0000000000000..44ce13ee62ac6 --- /dev/null +++ b/frontend/src/lib/components/triggers/WebsocketEditorConfigSection.svelte @@ -0,0 +1,112 @@ + + +
+
+ { + url = ev.detail === 'runnable' ? '$script:' : '' + url_runnable_args = {} + }} + disabled={captureMode} + > + + + +
+ {#if url.startsWith('$')} +
+
+
+
+ Runnable + +
+
+ { + dirtyUrl = true + const { path, itemKind } = ev.detail + url = `$${itemKind}:${path ?? ''}` + }} + /> +
+ {dirtyUrl ? urlError : ''} +
+
+
+ + {#if url.split(':')[1]?.length > 0} + {#if urlRunnableSchema} +

Arguments

+ {#await import('$lib/components/SchemaForm.svelte')} + + {:then Module} + + {/await} + {#if urlRunnableSchema.properties && Object.keys(urlRunnableSchema.properties).length === 0} +
This runnable takes no arguments
+ {/if} + {:else} + + {/if} + {/if} + {:else} +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/triggers/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/WebsocketTriggerEditorInner.svelte index 6afaac01ad6e0..6f777f983ca9f 100644 --- a/frontend/src/lib/components/triggers/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/WebsocketTriggerEditorInner.svelte @@ -24,8 +24,7 @@ import { fade } from 'svelte/transition' import JsonEditor from '../apps/editor/settingsPanel/inputEditor/JsonEditor.svelte' import type { Schema } from '$lib/common' - import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' - import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import WebsocketEditorConfigSection from './WebsocketEditorConfigSection.svelte' let drawer: Drawer let is_flow: boolean = false @@ -316,92 +315,14 @@
-
-
- { - url = ev.detail === 'runnable' ? '$script:' : '' - url_runnable_args = {} - }} - > - - - -
- {#if url.startsWith('$')} -
-
-
-
- Runnable - -
-
- { - dirtyUrl = true - const { path, itemKind } = ev.detail - url = `$${itemKind}:${path ?? ''}` - }} - /> -
- {dirtyUrl ? urlError : ''} -
-
-
- - {#if url.split(':')[1]?.length > 0} - {#if urlRunnableSchema} -

Arguments

- {#await import('$lib/components/SchemaForm.svelte')} - - {:then Module} - - {/await} - {#if urlRunnableSchema.properties && Object.keys(urlRunnableSchema.properties).length === 0} -
This runnable takes no arguments
- {/if} - {:else} - - {/if} - {/if} - {:else} -
- -
- {/if} -
+

diff --git a/frontend/src/lib/components/triggers/WebsocketTriggersPanel.svelte b/frontend/src/lib/components/triggers/WebsocketTriggersPanel.svelte index 66fe3252958e2..6ca12cc56963c 100644 --- a/frontend/src/lib/components/triggers/WebsocketTriggersPanel.svelte +++ b/frontend/src/lib/components/triggers/WebsocketTriggersPanel.svelte @@ -1,8 +1,6 @@ + +{#if connectionInfo} +

+ {#if connectionInfo.status === 'connected'} + + + + + +
{connectionInfo.message ?? ''}
+ + {:else} + + + + + + +
{connectionInfo.message ?? ''}
+ + {/if} +
+{/if} diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte index df7877629b477..eecdb6077d2da 100644 --- a/frontend/src/lib/components/triggers/CaptureWrapper.svelte +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -1,5 +1,5 @@
- {#if (captureType === 'websocket' || captureType === 'kafka') && config && active} - {@const serverEnabled = getServerEnabled(config)} -
-
-
- {#if serverEnabled} - - - - -
Websocket is connected
- - {:else} - - - - - -
- Websocket is not connected{config.error ? ': ' + config.error : ''} -
- - {/if} -
-
-
- {/if} {#if cloudDisabled} {capitalize(captureType)} triggers are disabled in the multi-tenant cloud. diff --git a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte index 12609e6666e25..c3af09b8ccd45 100644 --- a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte +++ b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte @@ -9,7 +9,7 @@ import { type CaptureTriggerKind } from '$lib/gen' import type { FlowEditorContext } from '../flows/types' import Toggle from '../Toggle.svelte' - + import ConnectionIndicator from '$lib/components/common/alert/ConnectionIndicator.svelte' export let cloudDisabled: boolean export let captureType: CaptureTriggerKind export let isFlow: boolean = false @@ -27,6 +27,12 @@ let args: Record = {} let captureMode = false let active = false + let connectionInfo: + | { + status: 'connected' | 'disconnected' | 'error' + message?: string + } + | undefined = undefined const dispatch = createEventDispatcher() @@ -43,6 +49,8 @@
+ + {#if captureMode}
From 8019c3a1d0eba8b8f0b04e72e5f3b3799892cca7 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 4 Dec 2024 14:20:58 +0000 Subject: [PATCH 08/60] change trigger section label --- .../src/lib/components/triggers/TriggersEditorSection.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte index c3af09b8ccd45..c45a6eacd13c8 100644 --- a/frontend/src/lib/components/triggers/TriggersEditorSection.svelte +++ b/frontend/src/lib/components/triggers/TriggersEditorSection.svelte @@ -44,7 +44,7 @@ let handleCapture: (() => Promise) | undefined -
+
From 124eabb9d4887578b3f68c579b5242de5e9d0c2a Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 4 Dec 2024 17:05:17 +0000 Subject: [PATCH 09/60] Add popover capture picker using melt ui --- frontend/package-lock.json | 98 +++++++- frontend/package.json | 4 +- frontend/src/lib/components/Label.svelte | 11 +- .../components/flows/content/FlowInput.svelte | 10 + .../flows/pickers/InputSchemaPicker.svelte | 68 ++++++ .../components/meltComponents/Popover.svelte | 47 ++++ .../components/triggers/CaptureTable.svelte | 209 +++++++++++++++++ .../components/triggers/CaptureWrapper.svelte | 210 ++---------------- frontend/svelte.config.js | 9 +- 9 files changed, 461 insertions(+), 205 deletions(-) create mode 100644 frontend/src/lib/components/flows/pickers/InputSchemaPicker.svelte create mode 100644 frontend/src/lib/components/meltComponents/Popover.svelte create mode 100644 frontend/src/lib/components/triggers/CaptureTable.svelte diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 539b62be83478..d793dadf12bce 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -85,6 +85,8 @@ "devDependencies": { "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", + "@melt-ui/pp": "^0.3.2", + "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", @@ -3554,6 +3556,16 @@ "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", "dev": true }, + "node_modules/@internationalized/date": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/@internationalized/date/-/date-3.6.0.tgz", + "integrity": "sha512-+z6ti+CcJnRlLHok/emGEsWQhe7kfSmEW+/6qCzvKY67YPh7YOBfvc7+/+NXq+zJlbArg30tYpqLjNgcAYv2YQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.0" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", @@ -3666,6 +3678,58 @@ "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.0.tgz", "integrity": "sha512-HPnRdYO0WjFjRTSwO3frz1wKaU649OBFPX3Zo/2WZvuRi6zMiRGui8SnPQiQABgqCf8YikDe5t3HViTVw1WUzA==" }, + "node_modules/@melt-ui/pp": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@melt-ui/pp/-/pp-0.3.2.tgz", + "integrity": "sha512-xKkPvaIAFinklLXcQOpwZ8YSpqAFxykjWf8Y/fSJQwsixV/0rcFs07hJ49hJjPy5vItvw5Qa0uOjzFUbXzBypQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "estree-walker": "^3.0.3", + "magic-string": "^0.30.5" + }, + "peerDependencies": { + "@melt-ui/svelte": ">= 0.29.0", + "svelte": "^3.55.0 || ^4.0.0 || ^5.0.0-next.1" + } + }, + "node_modules/@melt-ui/svelte": { + "version": "0.86.2", + "resolved": "https://registry.npmjs.org/@melt-ui/svelte/-/svelte-0.86.2.tgz", + "integrity": "sha512-wRVN603oIt1aXvx2QRmKqVDJgTScSvr/WJLLokkD8c4QzHgn6pfpPtUKmhV6Dvkk+OY89OG/1Irkd6ouA50Ztw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.3.1", + "@floating-ui/dom": "^1.4.5", + "@internationalized/date": "^3.5.0", + "dequal": "^2.0.3", + "focus-trap": "^7.5.2", + "nanoid": "^5.0.4" + }, + "peerDependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0-next.118" + } + }, + "node_modules/@melt-ui/svelte/node_modules/nanoid": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.9.tgz", + "integrity": "sha512-Aooyr6MXU6HpvvWXKoVoXwKMs/KyVakWwg7xQfv5/S/RIgJMy0Ifa45H9qqYy7pTCszrHzP21Uk4PZq2HpEM8Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.js" + }, + "engines": { + "node": "^18 || >=20" + } + }, "node_modules/@mistralai/mistralai": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.3.0.tgz", @@ -4315,6 +4379,16 @@ "vite": "^5.0.0" } }, + "node_modules/@swc/helpers": { + "version": "0.5.15", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", + "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, "node_modules/@tailwindcss/forms": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.7.tgz", @@ -7292,6 +7366,16 @@ "integrity": "sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ==", "dev": true }, + "node_modules/focus-trap": { + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.6.2.tgz", + "integrity": "sha512-9FhUxK1hVju2+AiQIDJ5Dd//9R2n2RAfJ0qfhF4IHGHgcoEUTMpbTeG/zbEuwaiYXfuAH6XE0/aCyxDdRM+W5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "tabbable": "^6.2.0" + } + }, "node_modules/follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -12829,6 +12913,13 @@ "node": ">= 10" } }, + "node_modules/tabbable": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", + "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", + "dev": true, + "license": "MIT" + }, "node_modules/table": { "version": "6.8.2", "resolved": "https://registry.npmjs.org/table/-/table-6.8.2.tgz", @@ -13112,9 +13203,10 @@ "dev": true }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" }, "node_modules/tsutils": { "version": "3.21.0", diff --git a/frontend/package.json b/frontend/package.json index a6ecec3d4370c..468323f463b80 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,6 +19,8 @@ "devDependencies": { "@floating-ui/core": "^1.3.1", "@hey-api/openapi-ts": "^0.43.0", + "@melt-ui/pp": "^0.3.2", + "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.0", @@ -128,8 +130,8 @@ "ol": "^7.4.0", "openai": "^4.57.2", "p-limit": "^6.1.0", - "pdfjs-dist": "4.8.69", "panzoom": "^9.4.3", + "pdfjs-dist": "4.8.69", "quill": "^1.3.7", "rfc4648": "^1.5.3", "svelte-carousel": "^1.0.25", diff --git a/frontend/src/lib/components/Label.svelte b/frontend/src/lib/components/Label.svelte index f53a8d9ca24dd..24acffca29303 100644 --- a/frontend/src/lib/components/Label.svelte +++ b/frontend/src/lib/components/Label.svelte @@ -3,14 +3,17 @@ export let label: string | undefined = undefined export let primary = false export let disabled = false + export let headless = false
-
- {label} - -
+ {#if !headless} +
+ {label} + +
+ {/if}
diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index 0834e03ab2127..df13a63ed0239 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -16,6 +16,8 @@ import Tab from '$lib/components/common/tabs/Tab.svelte' import CapturePanel from '$lib/components/triggers/CapturePanel.svelte' import { insertNewPreprocessorModule } from '../flowStateUtils' + import Popover from '$lib/components/meltComponents/Popover.svelte' + import InputSchemaPicker from '$lib/components/flows/pickers/InputSchemaPicker.svelte' export let noEditor: boolean export let disabled: boolean @@ -55,6 +57,14 @@ {#if tabSelected === 'input'}
Copy input's schema from
+ + + + + + + + + + + + +
+
+
+ +
+
diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte new file mode 100644 index 0000000000000..4129971cc8a33 --- /dev/null +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -0,0 +1,47 @@ + + + + +{#if open} +
+
+ + {#if closeButton} + + {/if} +
+{/if} + + diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte new file mode 100644 index 0000000000000..4540c017536b0 --- /dev/null +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -0,0 +1,209 @@ + + + diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte index eecdb6077d2da..cd7831b81de43 100644 --- a/frontend/src/lib/components/triggers/CaptureWrapper.svelte +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -1,32 +1,23 @@ diff --git a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte index cd5e9bc4b8824..adb699054a495 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte @@ -22,7 +22,7 @@ const { selectedId, flowStore, flowStateStore, flowInputsStore, pathStore, initialPath } = getContext('FlowEditorContext') - const { selectedTrigger, defaultValues } = getContext('TriggerContext') + const { selectedTrigger, defaultValues, captureOn } = getContext('TriggerContext') function checkDup(modules: FlowModule[]): string | undefined { let seenModules: string[] = [] for (const m of modules) { @@ -66,9 +66,11 @@ {noEditor} disabled={disabledFlowInputs} on:openTriggers={(ev) => { + console.log('dbg openTriggers', ev.detail) $selectedId = 'triggers' selectedTrigger.set(ev.detail.kind) defaultValues.set(ev.detail.config) + captureOn.set(true) }} on:applyArgs /> diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index df13a63ed0239..97f0009ae59f2 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -62,7 +62,7 @@ - +
- +
diff --git a/frontend/src/lib/components/triggers.ts b/frontend/src/lib/components/triggers.ts index a4447b0795e00..e585c7d989621 100644 --- a/frontend/src/lib/components/triggers.ts +++ b/frontend/src/lib/components/triggers.ts @@ -15,6 +15,7 @@ export type TriggerContext = { triggersCount: Writable simplifiedPoll: Writable defaultValues: Writable | undefined> + captureOn: Writable } export function setScheduledPollSchedule( diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte index 4540c017536b0..de0458ceb9acc 100644 --- a/frontend/src/lib/components/triggers/CaptureTable.svelte +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -1,6 +1,6 @@ + /> + + {/each} + {/if} + + +{/if} diff --git a/frontend/src/lib/components/triggers/CaptureWrapper.svelte b/frontend/src/lib/components/triggers/CaptureWrapper.svelte index cd7831b81de43..69e310892ec70 100644 --- a/frontend/src/lib/components/triggers/CaptureWrapper.svelte +++ b/frontend/src/lib/components/triggers/CaptureWrapper.svelte @@ -23,8 +23,7 @@ export let hasPreprocessor: boolean export let canHavePreprocessor: boolean export let captureType: CaptureTriggerKind = 'webhook' - export let captureMode = false - export let active = false + export let captureActive = false export let data: any = {} export let connectionInfo: | { @@ -217,13 +216,14 @@ return acc }, {}) - if ((captureType === 'websocket' || captureType === 'kafka') && active) { + if ((captureType === 'websocket' || captureType === 'kafka') && captureActive) { const config = captureConfigs[captureType] + console.log('dbg config', config) if (config && config.error) { const serverEnabled = getServerEnabled(config) if (!serverEnabled) { sendUserToast('Capture was stopped because of error: ' + config.error, true) - active = false + captureActive = false } } } @@ -233,10 +233,9 @@ let refreshCaptures: () => Promise async function capture() { - console.log('dbg capture') let i = 0 - active = true - while (active) { + captureActive = true + while (captureActive) { if (i % 3 === 0) { await CaptureService.pingCaptureConfig({ workspace: $workspaceStore!, @@ -253,7 +252,7 @@ } function stopAndSetDefaultArgs() { - active = false + captureActive = false if (captureType in captureConfigs) { const triggerConfig = captureConfigs[captureType].trigger_config args = isObject(triggerConfig) ? triggerConfig : {} @@ -270,7 +269,7 @@ $: captureType && stopAndSetDefaultArgs() onDestroy(() => { - active = false + captureActive = false }) function getServerEnabled(config: CaptureConfig) { @@ -281,11 +280,11 @@ } export async function handleCapture() { - if (!active) { + if (!captureActive) { await setConfig() capture() } else { - active = false + captureActive = false } } @@ -297,9 +296,9 @@ function updateConnectionInfo( captureType: CaptureTriggerKind, config: CaptureConfig | undefined, - active: boolean + captureActive: boolean ) { - if ((captureType === 'websocket' || captureType === 'kafka') && config && active) { + if ((captureType === 'websocket' || captureType === 'kafka') && config && captureActive) { const serverEnabled = getServerEnabled(config) const message = serverEnabled ? 'Websocket is connected' @@ -312,7 +311,7 @@ connectionInfo = undefined } } - $: updateConnectionInfo(captureType, config, active) + $: updateConnectionInfo(captureType, config, captureActive)
@@ -328,7 +327,13 @@ {/if} {#if captureType === 'websocket'} - + {:else if captureType === 'webhook'} {:else if captureType === 'http'} - + {:else if captureType === 'email'} {/if} - {#if captureMode} - - {/if} + {/if}
diff --git a/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte b/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte index 892020566f092..31f2cc97e0e5b 100644 --- a/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/RouteEditorConfigSection.svelte @@ -20,7 +20,7 @@ export let can_write: boolean = false export let static_asset_config: { s3: string; storage?: string; filename?: string } | undefined = undefined - export let captureMode = false + export let showCapture = false export let initialRoutePath: string = '' export let isFlow = true export let path: string = '' @@ -69,9 +69,9 @@ $: validateRoute(route_path, http_method) - $: fullRoute = getHttpRoute(route_path, captureMode) + $: fullRoute = getHttpRoute(route_path, showCapture) - $: captureMode && (http_method = 'post') + $: showCapture && (http_method = 'post') export let route_path = '' @@ -119,7 +119,7 @@ disabled={!($userStore?.is_admin || $userStore?.is_super_admin) || !can_write || !!static_asset_config || - captureMode} + showCapture} > @@ -151,7 +151,7 @@ >{dirtyRoutePath ? routeError : ''} - {#if captureMode} + {#if showCapture}