add blueprints

kestra-io · Apr 18, 2024 · 48b32b2 · 48b32b2
1 parent e80ded8
commit 48b32b2
Show file tree

Hide file tree

Showing 20 changed files with 215 additions and 53 deletions.
diff --git a/blueprints/108.yml b/blueprints/108.yml
@@ -4,7 +4,7 @@ namespace: blueprint
 tasks:
   - id: extract
     type: io.kestra.plugin.fs.http.Download
-    uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv
+    uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv
 
   - id: transform
     type: io.kestra.plugin.jdbc.duckdb.Query

diff --git a/blueprints/111.yml b/blueprints/111.yml
@@ -1,13 +1,13 @@
-id: dbtGitDockerDuckDB
+id: dbt_motherduck
 namespace: blueprint
 
 tasks:
-  - id: dbt
+  - id: git
     type: io.kestra.core.tasks.flows.WorkingDirectory
     tasks:
-      - id: cloneRepository
+      - id: clone_repository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
       - id: dbt-build
@@ -16,12 +16,12 @@ tasks:
         docker:
           image: ghcr.io/kestra-io/dbt-duckdb:latest
         profiles: |
-          jaffle_shop:
+          my_dbt_project:
             outputs:
               dev:
                 type: duckdb
                 disable_transactions: true
-                path: md:jaffle_shop?motherduck_token={{secret('MOTHERDUCK_TOKEN')}}
+                path: md:my_dbt_project?motherduck_token={{secret('MOTHERDUCK_TOKEN')}}
                 fixed_retries: 1
                 threads: 16
                 timeout_seconds: 300

diff --git a/blueprints/212.yml b/blueprints/212.yml
@@ -0,0 +1,16 @@
+id: docker_example
+namespace: blueprints
+
+tasks:
+  - id: download
+    type: io.kestra.plugin.fs.http.Download
+    uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv
+
+  - id: docker
+    type: io.kestra.plugin.scripts.shell.Commands
+    inputFiles:
+      data.csv: "{{ outputs.download.uri }}"
+    docker:
+      image: ubuntu:latest
+    commands:
+      - tail -n +2 data.csv | wc -l
diff --git a/blueprints/213.yml b/blueprints/213.yml
@@ -0,0 +1,39 @@
+id: hubspot_to_bigquery
+namespace: blueprints
+
+tasks:
+  - id: sync
+    type: io.kestra.plugin.cloudquery.Sync
+    inputFiles:
+      sa.json: "{{ secret('GCP_SERVICE_ACCOUNT') }}"
+    env:
+      HUBSPOT_APP_TOKEN: "{{ secret('HUBSPOT_API_TOKEN') }}"
+      GOOGLE_APPLICATION_CREDENTIALS: sa.json
+    configs:
+      - kind: destination
+        spec:
+          name: bigquery
+          path: cloudquery/bigquery
+          registry: cloudquery
+          version: "v3.3.16"
+          write_mode: "append"
+          spec:
+            project_id: kestra-prd
+            dataset_id: hubspot
+      - kind: source
+        spec:
+          name: hubspot
+          path: cloudquery/hubspot
+          registry: cloudquery
+          version: "v3.0.18"
+          destinations:
+            - "bigquery"
+          tables:
+            - "*"
+          spec:
+            max_requests_per_second: 5
+
+triggers:
+  - id: schedule
+    type: io.kestra.core.models.triggers.types.Schedule
+    cron: 0 6 * * *
diff --git a/blueprints/214.yml b/blueprints/214.yml
@@ -0,0 +1,30 @@
+id: sqlmesh
+namespace: blueprint
+description: Clone SQLMesh project and run the project, and query with DuckDB
+tasks:
+
+  - id: working_dir
+    type: io.kestra.core.tasks.flows.WorkingDirectory
+    tasks:
+      - id: git_clone
+        type: io.kestra.plugin.git.Clone
+        url: https://github.com/TobikoData/sqlmesh-examples.git
+        branch: main
+
+      - id: sqlmesh
+        type: io.kestra.plugin.sqlmesh.cli.SQLMeshCLI
+        beforeCommands:
+          - cd 001_sushi/1_simple
+        commands:
+          - sqlmesh plan --auto-apply
+        outputFiles:
+          - '001_sushi/1_simple/db/sushi-example.db'
+  - id: query
+    type: io.kestra.plugin.jdbc.duckdb.Query
+    inputFiles:
+      data.db: "{{ outputs.sqlmesh.outputFiles['001_sushi/1_simple/db/sushi-example.db']
+        }}"
+    sql: |
+      ATTACH '{{workingDir }}/data.db';
+      SELECT * FROM sushisimple.top_waiters;
+    store: true
diff --git a/blueprints/215.yml b/blueprints/215.yml
@@ -0,0 +1,37 @@
+id: orchestrate_everything
+namespace: blueprints
+
+inputs:
+  - id: use_case
+    description: What do you want to orchestrate?
+    type: ENUM
+    defaults: Data pipelines
+    values:
+      - Data pipelines
+      - Microservices
+      - Business processes
+      - Marketing automation
+tasks:
+  - id: conditional_branching
+    type: io.kestra.core.tasks.flows.Switch
+    value: "{{ inputs.use_case }}"
+    cases:
+      Data pipelines:
+        - id: data_pipelines
+          type: io.kestra.core.tasks.log.Log
+          message: Managing important data products
+
+      Microservices:
+        - id: microservices
+          type: io.kestra.core.tasks.log.Log
+          message: Orchestrating critical applications
+
+      Business processes:
+        - id: business_processes
+          type: io.kestra.core.tasks.log.Log
+          message: Orchestrating critical applications
+
+      Marketing automation:
+        - id: marketing_automation
+          type: io.kestra.core.tasks.log.Log
+          message: Orchestrating critical applications
diff --git a/blueprints/216.yml b/blueprints/216.yml
@@ -0,0 +1,37 @@
+id: http_check
+namespace: dev
+
+inputs:
+  - id: uri
+    type: URI
+    defaults: https://kestra.io
+
+tasks:
+  - id: api
+    type: io.kestra.plugin.fs.http.Request
+    uri: "{{ inputs.uri }}"
+
+  - id: check_status
+    type: io.kestra.core.tasks.flows.If
+    condition: "{{ outputs.api.code != 200 }}"
+    then:
+      - id: unhealthy
+        type: io.kestra.core.tasks.log.Log
+        message: Server unhealthy!!! Response {{ outputs.api.body }}
+      - id: send_slack_alert
+        type: io.kestra.plugin.notifications.slack.SlackIncomingWebhook
+        url: "{{ secret('SLACK_WEBHOOK') }}"
+        payload: |
+          {
+            "channel": "#alerts",
+            "text": "The server {{ inputs.uri }} is down!"
+          }
+    else:
+      - id: healthy
+        type: io.kestra.core.tasks.log.Log
+        message: Everything is fine!
+
+triggers:
+  - id: daily
+    type: io.kestra.core.models.triggers.types.Schedule
+    cron: "0 9 * * *"
diff --git a/blueprints/36.yml b/blueprints/36.yml
@@ -7,20 +7,17 @@ tasks:
     tasks:
       - id: downloadCSV
         type: io.kestra.plugin.fs.http.Download
-        uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv
+        uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv
 
       - id: fetchAuthToken
         type: io.kestra.plugin.gcp.auth.OauthAccessToken
         projectId: YOUR_GCP_PROJECT_NAME
         serviceAccount: "{{ secret('GCP_CREDS') }}"
 
-      - id: local
-        type: io.kestra.core.tasks.storages.LocalFiles
-        inputs:
-          data.csv: "{{outputs.downloadCSV.uri}}"
-
       - id: analyzeSales
         type: io.kestra.plugin.scripts.python.Script
+        inputFiles:
+          data.csv: "{{outputs.downloadCSV.uri}}"
         script: |
           import pandas as pd
           from kestra import Kestra

diff --git a/blueprints/38.yml b/blueprints/38.yml
@@ -8,7 +8,7 @@ tasks:
       INSTALL httpfs;
       LOAD httpfs;
       SELECT sum(total) as total, avg(quantity) as avg_quantity
-      FROM read_csv_auto('https://raw.githubusercontent.com/kestra-io/datasets/main/csv/orders.csv', header=True);
+      FROM read_csv_auto('https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv', header=True);
     fetch: true
 
   - id: slack

diff --git a/blueprints/49.yml b/blueprints/49.yml
@@ -1,26 +1,23 @@
-id: dbtGitDockerBigQuery
+id: dbt_bigquery
 namespace: blueprint
 
 tasks:
-  - id: dbt
+  - id: git
     type: io.kestra.core.tasks.flows.WorkingDirectory
     tasks:
-      - id: cloneRepository
+      - id: clone_repository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
-      - id: serviceAccount
-        type: io.kestra.core.tasks.storages.LocalFiles
-        inputs:
-          sa.json: "{{ secret('GCP_CREDS') }}"
-
-      - id: dbtCore
+      - id: dbt
         type: io.kestra.plugin.dbt.cli.DbtCLI
+        inputFiles:
+          sa.json: "{{ secret('GCP_CREDS') }}"
         docker:
           image: ghcr.io/kestra-io/dbt-bigquery:latest
         profiles: |
-          jaffle_shop:
+          my_dbt_project:
             outputs:
               dev:
                 type: bigquery

diff --git a/blueprints/50.yml b/blueprints/50.yml
@@ -1,4 +1,4 @@
-id: dbtGitDockerDuckDB
+id: dbt_duckdb
 namespace: blueprint
 
 tasks:
@@ -7,7 +7,7 @@ tasks:
     tasks:
       - id: cloneRepository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
       - id: dbt-build
@@ -18,3 +18,13 @@ tasks:
         commands:
           - dbt deps
           - dbt build
+        profiles: |
+          my_dbt_project:
+            outputs:
+              dev:
+                type: duckdb
+                path: ":memory:"
+                fixed_retries: 1
+                threads: 16
+                timeout_seconds: 300
+            target: dev
diff --git a/blueprints/51.yml b/blueprints/51.yml
@@ -1,21 +1,21 @@
-id: dbtGitDockerPostgres
+id: dbt_postgres
 namespace: blueprint
 
 tasks:
-  - id: dbt
+  - id: git
     type: io.kestra.core.tasks.flows.WorkingDirectory
     tasks:
-      - id: cloneRepository
+      - id: cloe_repository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
-      - id: dbtCore
+      - id: dbt
         type: io.kestra.plugin.dbt.cli.DbtCLI
         docker:
           image: ghcr.io/kestra-io/dbt-postgres:latest
         profiles: |
-          jaffle_shop:
+          my_dbt_project:
             outputs:
               dev:
                 type: postgres

diff --git a/blueprints/52.yml b/blueprints/52.yml
@@ -1,21 +1,21 @@
-id: dbtGitDockerSnowflake
+id: dbt_snowflake
 namespace: blueprint
 
 tasks:
-  - id: dbt
+  - id: git
     type: io.kestra.core.tasks.flows.WorkingDirectory
     tasks:
-      - id: cloneRepository
+      - id: clone_repository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
-      - id: dbtCore
+      - id: dbt
         type: io.kestra.plugin.dbt.cli.DbtCLI
         docker:
           image: ghcr.io/kestra-io/dbt-snowflake:latest
         profiles: |
-          jaffle_shop:
+          my_dbt_project:
             outputs:
               dev:
                 type: snowflake

diff --git a/blueprints/53.yml b/blueprints/53.yml
@@ -1,21 +1,21 @@
-id: dbtGitDockerRedshift
+id: dbt_redshift
 namespace: blueprint
 
 tasks:
-  - id: dbt
+  - id: git
     type: io.kestra.core.tasks.flows.WorkingDirectory
     tasks:
-      - id: cloneRepository
+      - id: clone_repository
         type: io.kestra.plugin.git.Clone
-        url: https://github.com/kestra-io/dbt-demo
+        url: https://github.com/kestra-io/dbt-example
         branch: main
 
-      - id: dbtCore
+      - id: dbt
         type: io.kestra.plugin.dbt.cli.DbtCLI
         docker:
           image: ghcr.io/kestra-io/dbt-redshift:latest
         profiles: |
-          jaffle_shop:
+          my_dbt_project:
             outputs:
               dev:
                 type: redshift