From 48b32b28f41f585ace301de5767c625df689e677 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 18 Apr 2024 21:27:15 +0000 Subject: [PATCH] add blueprints --- blueprints/108.yml | 2 +- blueprints/111.yml | 12 ++++++------ blueprints/212.yml | 16 ++++++++++++++++ blueprints/213.yml | 39 +++++++++++++++++++++++++++++++++++++++ blueprints/214.yml | 30 ++++++++++++++++++++++++++++++ blueprints/215.yml | 37 +++++++++++++++++++++++++++++++++++++ blueprints/216.yml | 37 +++++++++++++++++++++++++++++++++++++ blueprints/36.yml | 9 +++------ blueprints/38.yml | 2 +- blueprints/49.yml | 19 ++++++++----------- blueprints/50.yml | 14 ++++++++++++-- blueprints/51.yml | 12 ++++++------ blueprints/52.yml | 12 ++++++------ blueprints/53.yml | 12 ++++++------ blueprints/56.yml | 5 ++--- blueprints/60.yml | 2 +- blueprints/65.yml | 2 +- blueprints/66.yml | 2 +- blueprints/74.yml | 2 +- blueprints/81.yml | 2 +- 20 files changed, 215 insertions(+), 53 deletions(-) create mode 100644 blueprints/212.yml create mode 100644 blueprints/213.yml create mode 100644 blueprints/214.yml create mode 100644 blueprints/215.yml create mode 100644 blueprints/216.yml diff --git a/blueprints/108.yml b/blueprints/108.yml index 9dddd41..435a57f 100644 --- a/blueprints/108.yml +++ b/blueprints/108.yml @@ -4,7 +4,7 @@ namespace: blueprint tasks: - id: extract type: io.kestra.plugin.fs.http.Download - uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - id: transform type: io.kestra.plugin.jdbc.duckdb.Query diff --git a/blueprints/111.yml b/blueprints/111.yml index 2dfe8b8..aa38ab5 100644 --- a/blueprints/111.yml +++ b/blueprints/111.yml @@ -1,13 +1,13 @@ -id: dbtGitDockerDuckDB +id: dbt_motherduck namespace: blueprint tasks: - - id: dbt + - id: git type: io.kestra.core.tasks.flows.WorkingDirectory tasks: - - id: cloneRepository + - id: clone_repository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - id: dbt-build @@ -16,12 +16,12 @@ tasks: docker: image: ghcr.io/kestra-io/dbt-duckdb:latest profiles: | - jaffle_shop: + my_dbt_project: outputs: dev: type: duckdb disable_transactions: true - path: md:jaffle_shop?motherduck_token={{secret('MOTHERDUCK_TOKEN')}} + path: md:my_dbt_project?motherduck_token={{secret('MOTHERDUCK_TOKEN')}} fixed_retries: 1 threads: 16 timeout_seconds: 300 diff --git a/blueprints/212.yml b/blueprints/212.yml new file mode 100644 index 0000000..37d2974 --- /dev/null +++ b/blueprints/212.yml @@ -0,0 +1,16 @@ +id: docker_example +namespace: blueprints + +tasks: + - id: download + type: io.kestra.plugin.fs.http.Download + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv + + - id: docker + type: io.kestra.plugin.scripts.shell.Commands + inputFiles: + data.csv: "{{ outputs.download.uri }}" + docker: + image: ubuntu:latest + commands: + - tail -n +2 data.csv | wc -l diff --git a/blueprints/213.yml b/blueprints/213.yml new file mode 100644 index 0000000..44b6dfc --- /dev/null +++ b/blueprints/213.yml @@ -0,0 +1,39 @@ +id: hubspot_to_bigquery +namespace: blueprints + +tasks: + - id: sync + type: io.kestra.plugin.cloudquery.Sync + inputFiles: + sa.json: "{{ secret('GCP_SERVICE_ACCOUNT') }}" + env: + HUBSPOT_APP_TOKEN: "{{ secret('HUBSPOT_API_TOKEN') }}" + GOOGLE_APPLICATION_CREDENTIALS: sa.json + configs: + - kind: destination + spec: + name: bigquery + path: cloudquery/bigquery + registry: cloudquery + version: "v3.3.16" + write_mode: "append" + spec: + project_id: kestra-prd + dataset_id: hubspot + - kind: source + spec: + name: hubspot + path: cloudquery/hubspot + registry: cloudquery + version: "v3.0.18" + destinations: + - "bigquery" + tables: + - "*" + spec: + max_requests_per_second: 5 + +triggers: + - id: schedule + type: io.kestra.core.models.triggers.types.Schedule + cron: 0 6 * * * diff --git a/blueprints/214.yml b/blueprints/214.yml new file mode 100644 index 0000000..458800a --- /dev/null +++ b/blueprints/214.yml @@ -0,0 +1,30 @@ +id: sqlmesh +namespace: blueprint +description: Clone SQLMesh project and run the project, and query with DuckDB +tasks: + + - id: working_dir + type: io.kestra.core.tasks.flows.WorkingDirectory + tasks: + - id: git_clone + type: io.kestra.plugin.git.Clone + url: https://github.com/TobikoData/sqlmesh-examples.git + branch: main + + - id: sqlmesh + type: io.kestra.plugin.sqlmesh.cli.SQLMeshCLI + beforeCommands: + - cd 001_sushi/1_simple + commands: + - sqlmesh plan --auto-apply + outputFiles: + - '001_sushi/1_simple/db/sushi-example.db' + - id: query + type: io.kestra.plugin.jdbc.duckdb.Query + inputFiles: + data.db: "{{ outputs.sqlmesh.outputFiles['001_sushi/1_simple/db/sushi-example.db'] + }}" + sql: | + ATTACH '{{workingDir }}/data.db'; + SELECT * FROM sushisimple.top_waiters; + store: true diff --git a/blueprints/215.yml b/blueprints/215.yml new file mode 100644 index 0000000..b08cece --- /dev/null +++ b/blueprints/215.yml @@ -0,0 +1,37 @@ +id: orchestrate_everything +namespace: blueprints + +inputs: + - id: use_case + description: What do you want to orchestrate? + type: ENUM + defaults: Data pipelines + values: + - Data pipelines + - Microservices + - Business processes + - Marketing automation +tasks: + - id: conditional_branching + type: io.kestra.core.tasks.flows.Switch + value: "{{ inputs.use_case }}" + cases: + Data pipelines: + - id: data_pipelines + type: io.kestra.core.tasks.log.Log + message: Managing important data products + + Microservices: + - id: microservices + type: io.kestra.core.tasks.log.Log + message: Orchestrating critical applications + + Business processes: + - id: business_processes + type: io.kestra.core.tasks.log.Log + message: Orchestrating critical applications + + Marketing automation: + - id: marketing_automation + type: io.kestra.core.tasks.log.Log + message: Orchestrating critical applications diff --git a/blueprints/216.yml b/blueprints/216.yml new file mode 100644 index 0000000..d4b6535 --- /dev/null +++ b/blueprints/216.yml @@ -0,0 +1,37 @@ +id: http_check +namespace: dev + +inputs: + - id: uri + type: URI + defaults: https://kestra.io + +tasks: + - id: api + type: io.kestra.plugin.fs.http.Request + uri: "{{ inputs.uri }}" + + - id: check_status + type: io.kestra.core.tasks.flows.If + condition: "{{ outputs.api.code != 200 }}" + then: + - id: unhealthy + type: io.kestra.core.tasks.log.Log + message: Server unhealthy!!! Response {{ outputs.api.body }} + - id: send_slack_alert + type: io.kestra.plugin.notifications.slack.SlackIncomingWebhook + url: "{{ secret('SLACK_WEBHOOK') }}" + payload: | + { + "channel": "#alerts", + "text": "The server {{ inputs.uri }} is down!" + } + else: + - id: healthy + type: io.kestra.core.tasks.log.Log + message: Everything is fine! + +triggers: + - id: daily + type: io.kestra.core.models.triggers.types.Schedule + cron: "0 9 * * *" diff --git a/blueprints/36.yml b/blueprints/36.yml index 3cc3bfc..a3092ba 100644 --- a/blueprints/36.yml +++ b/blueprints/36.yml @@ -7,20 +7,17 @@ tasks: tasks: - id: downloadCSV type: io.kestra.plugin.fs.http.Download - uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - id: fetchAuthToken type: io.kestra.plugin.gcp.auth.OauthAccessToken projectId: YOUR_GCP_PROJECT_NAME serviceAccount: "{{ secret('GCP_CREDS') }}" - - id: local - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - data.csv: "{{outputs.downloadCSV.uri}}" - - id: analyzeSales type: io.kestra.plugin.scripts.python.Script + inputFiles: + data.csv: "{{outputs.downloadCSV.uri}}" script: | import pandas as pd from kestra import Kestra diff --git a/blueprints/38.yml b/blueprints/38.yml index 3a91c1a..00c8b96 100644 --- a/blueprints/38.yml +++ b/blueprints/38.yml @@ -8,7 +8,7 @@ tasks: INSTALL httpfs; LOAD httpfs; SELECT sum(total) as total, avg(quantity) as avg_quantity - FROM read_csv_auto('https://raw.githubusercontent.com/kestra-io/datasets/main/csv/orders.csv', header=True); + FROM read_csv_auto('https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv', header=True); fetch: true - id: slack diff --git a/blueprints/49.yml b/blueprints/49.yml index 87e1a42..1e8c167 100644 --- a/blueprints/49.yml +++ b/blueprints/49.yml @@ -1,26 +1,23 @@ -id: dbtGitDockerBigQuery +id: dbt_bigquery namespace: blueprint tasks: - - id: dbt + - id: git type: io.kestra.core.tasks.flows.WorkingDirectory tasks: - - id: cloneRepository + - id: clone_repository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - - id: serviceAccount - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - sa.json: "{{ secret('GCP_CREDS') }}" - - - id: dbtCore + - id: dbt type: io.kestra.plugin.dbt.cli.DbtCLI + inputFiles: + sa.json: "{{ secret('GCP_CREDS') }}" docker: image: ghcr.io/kestra-io/dbt-bigquery:latest profiles: | - jaffle_shop: + my_dbt_project: outputs: dev: type: bigquery diff --git a/blueprints/50.yml b/blueprints/50.yml index 57f4da9..332ef3f 100644 --- a/blueprints/50.yml +++ b/blueprints/50.yml @@ -1,4 +1,4 @@ -id: dbtGitDockerDuckDB +id: dbt_duckdb namespace: blueprint tasks: @@ -7,7 +7,7 @@ tasks: tasks: - id: cloneRepository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - id: dbt-build @@ -18,3 +18,13 @@ tasks: commands: - dbt deps - dbt build + profiles: | + my_dbt_project: + outputs: + dev: + type: duckdb + path: ":memory:" + fixed_retries: 1 + threads: 16 + timeout_seconds: 300 + target: dev diff --git a/blueprints/51.yml b/blueprints/51.yml index 71f4744..fc0abf3 100644 --- a/blueprints/51.yml +++ b/blueprints/51.yml @@ -1,21 +1,21 @@ -id: dbtGitDockerPostgres +id: dbt_postgres namespace: blueprint tasks: - - id: dbt + - id: git type: io.kestra.core.tasks.flows.WorkingDirectory tasks: - - id: cloneRepository + - id: cloe_repository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - - id: dbtCore + - id: dbt type: io.kestra.plugin.dbt.cli.DbtCLI docker: image: ghcr.io/kestra-io/dbt-postgres:latest profiles: | - jaffle_shop: + my_dbt_project: outputs: dev: type: postgres diff --git a/blueprints/52.yml b/blueprints/52.yml index ff4e096..66a3c62 100644 --- a/blueprints/52.yml +++ b/blueprints/52.yml @@ -1,21 +1,21 @@ -id: dbtGitDockerSnowflake +id: dbt_snowflake namespace: blueprint tasks: - - id: dbt + - id: git type: io.kestra.core.tasks.flows.WorkingDirectory tasks: - - id: cloneRepository + - id: clone_repository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - - id: dbtCore + - id: dbt type: io.kestra.plugin.dbt.cli.DbtCLI docker: image: ghcr.io/kestra-io/dbt-snowflake:latest profiles: | - jaffle_shop: + my_dbt_project: outputs: dev: type: snowflake diff --git a/blueprints/53.yml b/blueprints/53.yml index 78c5d37..629029b 100644 --- a/blueprints/53.yml +++ b/blueprints/53.yml @@ -1,21 +1,21 @@ -id: dbtGitDockerRedshift +id: dbt_redshift namespace: blueprint tasks: - - id: dbt + - id: git type: io.kestra.core.tasks.flows.WorkingDirectory tasks: - - id: cloneRepository + - id: clone_repository type: io.kestra.plugin.git.Clone - url: https://github.com/kestra-io/dbt-demo + url: https://github.com/kestra-io/dbt-example branch: main - - id: dbtCore + - id: dbt type: io.kestra.plugin.dbt.cli.DbtCLI docker: image: ghcr.io/kestra-io/dbt-redshift:latest profiles: | - jaffle_shop: + my_dbt_project: outputs: dev: type: redshift diff --git a/blueprints/56.yml b/blueprints/56.yml index fcbc25e..1bdd7aa 100644 --- a/blueprints/56.yml +++ b/blueprints/56.yml @@ -10,11 +10,10 @@ tasks: branch: main url: https://github.com/kestra-io/scripts - - id: runModal - type: io.kestra.plugin.scripts.shell.Commands + - id: modal + type: io.kestra.plugin.modal.cli.ModalCLI commands: - modal run modal/gpu.py - runner: DOCKER docker: image: ghcr.io/kestra-io/modal:latest env: diff --git a/blueprints/60.yml b/blueprints/60.yml index e1db496..03eb776 100644 --- a/blueprints/60.yml +++ b/blueprints/60.yml @@ -5,7 +5,7 @@ tasks: - id: download type: io.kestra.plugin.fs.http.Download - uri: https://raw.githubusercontent.com/kestra-io/datasets/main/csv/orders.csv + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - id: upload type: io.kestra.plugin.googleworkspace.drive.Upload diff --git a/blueprints/65.yml b/blueprints/65.yml index a6598af..f623ab3 100644 --- a/blueprints/65.yml +++ b/blueprints/65.yml @@ -8,7 +8,7 @@ variables: tasks: - id: extract type: io.kestra.plugin.fs.http.Download - uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - id: query type: io.kestra.plugin.jdbc.postgresql.Query diff --git a/blueprints/66.yml b/blueprints/66.yml index 62bfd07..2d77227 100644 --- a/blueprints/66.yml +++ b/blueprints/66.yml @@ -7,7 +7,7 @@ variables: tasks: - id: extract type: io.kestra.plugin.fs.http.Download - uri: https://raw.githubusercontent.com/kestra-io/examples/main/datasets/orders.csv + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/orders.csv - id: enableLocalFiles type: io.kestra.plugin.jdbc.mysql.Query diff --git a/blueprints/74.yml b/blueprints/74.yml index 1b50344..506e105 100644 --- a/blueprints/74.yml +++ b/blueprints/74.yml @@ -2,7 +2,7 @@ id: redis_set_parallel namespace: blueprint inputs: - - name: values + - id: values type: JSON description: Enter your favorite plugins and tasks defaults: | diff --git a/blueprints/81.yml b/blueprints/81.yml index 4c68ef4..4eac100 100644 --- a/blueprints/81.yml +++ b/blueprints/81.yml @@ -21,7 +21,7 @@ tasks: - id: extract type: io.kestra.plugin.fs.http.Download uri: - https://raw.githubusercontent.com/kestra-io/examples/main/datasets/employees00.csv + https://huggingface.co/datasets/kestra/datasets/raw/main/employees/employees00.csv - id: loadToInternalStage type: io.kestra.plugin.jdbc.snowflake.Upload