From 9238453ffc4a5d3234735ad9c9381fecb7b4ed2b Mon Sep 17 00:00:00 2001 From: MasterCaeSaR <80308990+MasterCaeSaR@users.noreply.github.com> Date: Sun, 4 Aug 2024 10:48:34 +0000 Subject: [PATCH] new file: .user.yml modified: Dockerfile modified: dbt_project.yml modified: main.tf modified: models/marts/f_orders_stats.sql new file: package-lock.yml modified: packages.yml modified: profiles.yml modified: variables.tf --- .user.yml | 1 + Dockerfile | 4 ++-- dbt_project.yml | 8 ++++++-- main.tf | 29 ++++++++++++++++------------- models/marts/f_orders_stats.sql | 14 +++++++++++++- package-lock.yml | 4 ++++ packages.yml | 2 +- profiles.yml | 11 +++-------- variables.tf | 8 ++++---- 9 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 .user.yml create mode 100644 package-lock.yml diff --git a/.user.yml b/.user.yml new file mode 100644 index 0000000..36c5722 --- /dev/null +++ b/.user.yml @@ -0,0 +1 @@ +id: 93533b73-9644-414a-a42e-347ce27a2d5a diff --git a/Dockerfile b/Dockerfile index 1cb29e5..d612e50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -ARG DBT_VERSION=1.0.0 -FROM fishtownanalytics/dbt:${DBT_VERSION} +ARG DBT_VERSION=1.8.3 +FROM ghcr.io/dbt-labs/dbt-core:${DBT_VERSION} # Terraform configuration file # COPY terraformrc root/.terraformrc diff --git a/dbt_project.yml b/dbt_project.yml index 9382a61..07c896b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,3 @@ - # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models @@ -9,6 +8,11 @@ config-version: 2 # This setting configures which "profile" dbt uses for this project. profile: 'clickhouse_starschema' +flags: + send_anonymous_usage_stats: False + use_colors: True + partial_parse: True + # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be # found in the "models/" directory. You probably won't need to change these! @@ -40,4 +44,4 @@ models: +tags: ['staging'] marts: +materialized: table - +tags: ['marts'] + +tags: ['marts'] \ No newline at end of file diff --git a/main.tf b/main.tf index d20e1a9..891e6a7 100644 --- a/main.tf +++ b/main.tf @@ -8,17 +8,22 @@ terraform { } provider "yandex" { + token = "t1.9euelZqPjcudic6JlpeZlpKazJTGkO3rnpWalpDMioycm5jIz8jKnZ6Sjs_l9PdBBUNK-e8Dbwev3fT3ATRASvnvA28Hr83n9euelZqdjcjKiszMy5THl4_Kzcadju_8xeuelZqdjcjKiszMy5THl4_Kzcadjg.fCgEd3SWL-ehzJihzWkH_NNZDQQvRfMEenDT-PeHqJfAR1c3pOi3ydQmxLIEeI5cY75ngJ7sPPBqXL8-vCtJCA" + cloud_id = "b1g38qb1ddfldjnj15m9" + folder_id = "b1gcaq5kiuko0f1f02j4" + storage_access_key = "YCAJE8F0PQP6DUrWLKF-DQzkI" + storage_secret_key = "YCM1kZ4lXdVQWWouZ39wJmhYfgTH0JA0Ra2W1O9P" + zone = "ru-central1-d" } - resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" { name = "clickhouse_starschema" - environment = "PRESTABLE" + environment = "PRODUCTION" network_id = yandex_vpc_network.default_network.id sql_database_management = true sql_user_management = true admin_password = var.clickhouse_password - version = "23.3" + version = "24.4" clickhouse { resources { @@ -39,21 +44,21 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" { timezone = "UTC" geobase_uri = "" query_log_retention_size = 1073741824 - query_log_retention_time = 2592000 + query_log_retention_time = 2592000000 query_thread_log_enabled = true query_thread_log_retention_size = 536870912 - query_thread_log_retention_time = 2592000 + query_thread_log_retention_time = 2592000000 part_log_retention_size = 536870912 - part_log_retention_time = 2592000 + part_log_retention_time = 2592000000 metric_log_enabled = true metric_log_retention_size = 536870912 - metric_log_retention_time = 2592000 + metric_log_retention_time = 2592000000 trace_log_enabled = true trace_log_retention_size = 536870912 - trace_log_retention_time = 2592000 + trace_log_retention_time = 2592000000 text_log_enabled = true text_log_retention_size = 536870912 - text_log_retention_time = 2592000 + text_log_retention_time = 2592000000 text_log_level = "TRACE" background_pool_size = 16 background_schedule_pool_size = 16 @@ -137,7 +142,7 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" { host { type = "CLICKHOUSE" - zone = "ru-central1-b" + zone = "ru-central1-d" subnet_id = yandex_vpc_subnet.foo.id assign_public_ip = true } @@ -150,11 +155,9 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" { type = "ANYTIME" } } - resource "yandex_vpc_network" "default_network" {} - resource "yandex_vpc_subnet" "foo" { - zone = "ru-central1-b" + zone = "ru-central1-d" network_id = yandex_vpc_network.default_network.id v4_cidr_blocks = ["10.5.0.0/24"] } \ No newline at end of file diff --git a/models/marts/f_orders_stats.sql b/models/marts/f_orders_stats.sql index f3c422a..900bf70 100644 --- a/models/marts/f_orders_stats.sql +++ b/models/marts/f_orders_stats.sql @@ -5,4 +5,16 @@ ) }} -SELECT 1 +SELECT + toYear(O_ORDERDATE) AS O_ORDERYEAR + , O_ORDERSTATUS + , O_ORDERPRIORITY + , count(DISTINCT O_ORDERKEY) AS num_orders + , count(DISTINCT C_CUSTKEY) AS num_customers + , sum(L_EXTENDEDPRICE * L_DISCOUNT) AS revenue +FROM {{ ref('f_lineorder_flat') }} +WHERE 1=1 +GROUP BY + toYear(O_ORDERDATE) + , O_ORDERSTATUS + , O_ORDERPRIORITY \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..8c9adce --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.2.0 +sha1_hash: 375b38987c2346580b2f54a91a56cbc9dadd6f1f diff --git a/packages.yml b/packages.yml index d4e6640..a288afb 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: dbt-labs/dbt_utils - version: [">=1.0.0", "<1.1.0"] + version: [">=1.0.0", "<1.3.0"] diff --git a/profiles.yml b/profiles.yml index 59a15e8..cdb3751 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,17 +1,12 @@ -config: - send_anonymous_usage_stats: False - use_colors: True - partial_parse: True - clickhouse_starschema: outputs: dev: type: clickhouse schema: default - host: "{{ env_var('DBT_HOST') }}" + host: "rc1d-mb0nbushgtr6bvn8.mdb.yandexcloud.net" port: 8443 - user: "{{ env_var('DBT_USER') }}" - password: "{{ env_var('DBT_PASSWORD') }}" + user: "admin" + password: "jxtymlkbyysqgfhjkm" secure: True verify: False ci: diff --git a/variables.tf b/variables.tf index d32f88b..e14e7f6 100644 --- a/variables.tf +++ b/variables.tf @@ -1,13 +1,13 @@ variable "clickhouse_password" { - description = "Clickhouse admin password" + description = "jxtymlkbyysqgfhjkm" type = string sensitive = true } -# data "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" { -# name = "clickhouse_starschema" +# data "yandex_mdb_clickhouse_cluster" "clickhouse_starschema"" { +# name = "clickhouse_starschema"" # } output "clickhouse_host_fqdn" { value = resource.yandex_mdb_clickhouse_cluster.clickhouse_starschema.host[0].fqdn -} +} \ No newline at end of file