diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b26d167..6494f26 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,53 +1,53 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the -// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile -{ - "name": "Existing Dockerfile", - "build": { - // Sets the run context to one level up instead of the .devcontainer folder. - // "context": "." - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerfile": "../Dockerfile" - }, - - // Features to add to the dev container. More info: https://containers.dev/features. - // "features": {}, - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - "forwardPorts": [ - 8080 - ], - "customizations": { - "vscode": { - "extensions": [ - "innoverio.vscode-dbt-power-user", - "ms-python.python", - "eamodio.gitlens", - "GitHub.vscode-pull-request-github", - "nemesv.copy-file-name" - ] - } - }, - "features": { - "ghcr.io/devcontainers/features/common-utils:2": { - "installZsh": true, - "configureZshAsDefaultShell": true, - "installOhMyZsh": true, - "upgradePackages": true, - "username": "automatic", - "userUid": "automatic", - "userGid": "automatic" - }, - "ghcr.io/stuartleeks/dev-container-features/shell-history:0": {} - }, - - "runArgs": ["--env-file",".env", "--network=mybi-dbt-showcase_default"] - - // Uncomment the next line to run commands after the container is created. - // "postCreateCommand": "cat /etc/os-release", - - // Configure tool-specific properties. - // "customizations": {}, - - // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. - // "remoteUser": "devcontainer" -} +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +{ + "name": "Existing Dockerfile", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + // "context": "." + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "../Dockerfile" + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 8080 + ], + "customizations": { + "vscode": { + "extensions": [ + "innoverio.vscode-dbt-power-user", + "ms-python.python", + "eamodio.gitlens", + "GitHub.vscode-pull-request-github", + "nemesv.copy-file-name" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/common-utils:2": { + "installZsh": true, + "configureZshAsDefaultShell": true, + "installOhMyZsh": true, + "upgradePackages": true, + "username": "automatic", + "userUid": "automatic", + "userGid": "automatic" + }, + "ghcr.io/stuartleeks/dev-container-features/shell-history:0": {} + }, + + "runArgs": ["--env-file",".env", "--network=mybi-dbt-showcase_default"] + + // Uncomment the next line to run commands after the container is created. + // "postCreateCommand": "cat /etc/os-release", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} diff --git a/.env b/.env index dfb28b2..f77e36b 100644 --- a/.env +++ b/.env @@ -1,11 +1,11 @@ -CUBEJS_DEV_MODE=true - -CUBEJS_DB_TYPE=clickhouse -CUBEJS_DB_HOST=clickhouse -CUBEJS_DB_PORT=8123 -CUBEJS_DB_NAME=analytics -CUBEJS_DB_USER=default -CUBEJS_DB_PASS= - -CUBEJS_DB_SSL=false +CUBEJS_DEV_MODE=true + +CUBEJS_DB_TYPE=clickhouse +CUBEJS_DB_HOST=clickhouse +CUBEJS_DB_PORT=8123 +CUBEJS_DB_NAME=analytics +CUBEJS_DB_USER=default +CUBEJS_DB_PASS= + +CUBEJS_DB_SSL=false # NODE_TLS_REJECT_UNAUTHORIZED=0 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 127727f..5294368 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,49 +1,49 @@ -name: Continuous Integration Tests - -on: - pull_request: - branches: [ main ] -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - ci: - name: Continuous Integration Tests - timeout-minutes: 30 - services: - clickhouse: - image: clickhouse/clickhouse-server:22.3 - ports: - - 9000:9000 - - 8123:8123 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: wait for services to start up - run: sleep 10 - - name: dbt version - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt --version - - name: dbt debug - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt debug - - name: dbt deps - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt deps - - name: dbt init_source_data - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt run-operation init_source_data - - name: dbt build - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt build --full-refresh - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # if: ${{ failure() }} - # timeout-minutes: 30 +name: Continuous Integration Tests + +on: + pull_request: + branches: [ main ] +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + ci: + name: Continuous Integration Tests + timeout-minutes: 30 + services: + clickhouse: + image: clickhouse/clickhouse-server:22.3 + ports: + - 9000:9000 + - 8123:8123 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: wait for services to start up + run: sleep 10 + - name: dbt version + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt --version + - name: dbt debug + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt debug + - name: dbt deps + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt deps + - name: dbt init_source_data + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt run-operation init_source_data + - name: dbt build + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt build --full-refresh + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # if: ${{ failure() }} + # timeout-minutes: 30 \ No newline at end of file diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml index c3dbc7a..62f363c 100644 --- a/.github/workflows/publish_docs.yml +++ b/.github/workflows/publish_docs.yml @@ -1,58 +1,58 @@ -name: Publish dbt Documentation on PR Closed - - -# triggers for the workflow -on: - workflow_dispatch: - # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target - pull_request_target: - types: - - closed - branches: - - main - - -jobs: - generate-dbt-docs: - name: Generate dbt docs - runs-on: ubuntu-latest - services: - clickhouse: - image: clickhouse/clickhouse-server:22.3 - ports: - - 9000:9000 - - 8123:8123 - - # Grant GITHUB_TOKEN the permissions required to make a Pages deployment - permissions: - pages: write # to deploy to Pages - id-token: write # to verify the deployment originates from an appropriate source - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - uses: actions/checkout@v3 - - name: dbt debug - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt debug - - name: dbt deps - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt deps - - name: dbt docs - uses: kzzzr/mybi-dbt-action@v4 - with: - command: dbt docs generate - - name: Setup Pages - uses: actions/configure-pages@v2 - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - with: - path: 'target' - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 - +name: Publish dbt Documentation on PR Closed + + +# triggers for the workflow +on: + workflow_dispatch: + # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + pull_request_target: + types: + - closed + branches: + - main + + +jobs: + generate-dbt-docs: + name: Generate dbt docs + runs-on: ubuntu-latest + services: + clickhouse: + image: clickhouse/clickhouse-server:22.3 + ports: + - 9000:9000 + - 8123:8123 + + # Grant GITHUB_TOKEN the permissions required to make a Pages deployment + permissions: + pages: write # to deploy to Pages + id-token: write # to verify the deployment originates from an appropriate source + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - uses: actions/checkout@v3 + - name: dbt debug + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt debug + - name: dbt deps + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt deps + - name: dbt docs + uses: kzzzr/mybi-dbt-action@v4 + with: + command: dbt docs generate + - name: Setup Pages + uses: actions/configure-pages@v2 + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + path: 'target' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 + diff --git a/.gitignore b/.gitignore index 44c3b08..ba7455e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,11 @@ - -target/ -dbt_packages/ -logs/ - -/metabase/plugins/* -!clickhouse.metabase-driver.jar - -.DS_Store - + +target/ +dbt_packages/ +logs/ + +/metabase/plugins/* +!clickhouse.metabase-driver.jar + +.DS_Store + .cubestore/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f612acc..92792a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -ARG DBT_VERSION=1.0.0 -FROM fishtownanalytics/dbt:${DBT_VERSION} - -RUN set -ex \ - && python -m pip install --upgrade pip setuptools \ - && python -m pip install --upgrade dbt-clickhouse numpy - -WORKDIR /usr/app/ -ENV DBT_PROFILES_DIR=. - -ENTRYPOINT ["tail", "-f", "/dev/null"] +ARG DBT_VERSION=1.0.0 +FROM fishtownanalytics/dbt:${DBT_VERSION} + +RUN set -ex \ + && python -m pip install --upgrade pip setuptools \ + && python -m pip install --upgrade dbt-clickhouse numpy + +WORKDIR /usr/app/ +ENV DBT_PROFILES_DIR=. + +ENTRYPOINT ["tail", "-f", "/dev/null"] diff --git a/README.md b/README.md index 8f19f75..ca8e74a 100644 --- a/README.md +++ b/README.md @@ -1,207 +1,207 @@ -# myBI Market + dbt demo project - -![](https://habrastorage.org/webt/l-/1r/pq/l-1rpqoplxi-503grfeyyglux8g.jpeg) - -This repo guides you through building analyics for [myBI Market](https://market.mybi.ru/) with Modern Data Stack: - -- myBI Connect (Extract - Load tool) -- Clickhouse (Database) -- dbt (Transformations) + [mybi-dbt-core](https://github.com/kzzzr/mybi-dbt-core) module -- Metabase (Business Intelligence) -- Github Actions (Continuous Integration + Deployment) - -## Table of Contents - -1. [Configure environment](#configure-environment) -1. [Install and configure mybi_dbt_core package](#install-and-configure-mybi_dbt_core-package) -1. [Configure data sources](#configure-data-sources) -1. [Build staging layer models](#build-staging-layer-models) -1. [Model Data Marts](#model-data-marts) -1. [Set up metrics layer](#set-up-metrics-layer) -1. [Visualize on a dashboard](#visualize-on-a-dashboard) -1. [Publish dbt project docs](#publish-dbt-project-docs) -1. [Introduce Continuous Integration](#introduce-continuous-integration) - -## Configure environment - -1. Install prerequisites: - - IDE (e.g. [VS Code](https://code.visualstudio.com/docs/setup/setup-overview)) - - [Docker](https://docs.docker.com/engine/install/) - -2. Fork & Clone this repository and open in IDE - -3. Spin up Docker containers - - All the services are configured via [Docker containers](./docker-compose.yml). - - - Clickhouse - - Metabase - - Cube - - ```bash - # launch containers: clickhouse, metabase, cube - docker-compose build --no-cache - docker-compose up -d - ``` - -4. Open dev container with dbt installed - - ```bash - devcontainer open . - - # test connections - dbt --version - dbt debug - ``` - - ![Spin up Docker containers](./docs/1_docker_containers.gif) - -## Install and configure [mybi_dbt_core](https://github.com/kzzzr/mybi-dbt-core) package - -1. Install module via [packages.yml](./packages.yml) - - ```bash - dbt clean # clean temp files - dbt deps # install dependencies (modules) - ``` - -2. ✅ Enable only [relevant data models](./dbt_project.yml#L7-L20) (and disable the rest) - - We will use specific data sources: - - * general – [General]() - * direct – [Yandex.Direct](https://docs.mybi.ru/yandeks-direkt-struktura-bazovoy-vygruzki/) - * mytarget – [myTarget](https://docs.mybi.ru/mytarget-struktura-bazovoy-vygruzki/) - * amocrm – [AmoCRM](https://docs.mybi.ru/amocrm-struktura-bazovoy-vygruzki/) - * ga – [Google Analytics](https://docs.mybi.ru/google-analytics-struktura-bazovoy-vygruzki/) - - Confirm with command: `dbt ls --resource-type model -s tag:staging` - - ![Confirm specfic data sources enabled](./docs/2_enable_specific_data_sources.gif) - -3. ✅ Assign variables - - Certain variable [values have to be set](./dbt_project.yml#L28-L46): - - source database connection details - - database and schema name to find mybi source data - - specific `account_id` values to be included - -4. ✅ Turn on custom schema management - - I use [generate_schema_name.sql](./macros/generate_schema_name.sql) macro to set target schema names: - - > Renders a schema name given a custom schema name. In production, this macro - > will render out the overriden schema name for a model. Otherwise, the default - > schema specified in the active target is used. - - Take a look at [custom_schema_management.sql](https://github.com/kzzzr/mybi-dbt-core/blob/main/macros/custom_schema_management.sql) macro to find out more. - -## Configure data sources - -1. [Create](./macros/init_source_data.sql) Clickhouse database of [PostgreSQL Database Engine](https://clickhouse.com/docs/en/engines/database-engines/postgresql/) with source data: - - ```bash - dbt run-operation init_source_data - ``` - - ![Initialized myBI source database](./docs/3_init_source_data.gif) - -## Build staging layer models - -Staging models are alredy configured for you in [mybi_dbt_core](https://github.com/kzzzr/mybi-dbt-core) package: -- [source definitions](https://github.com/kzzzr/mybi-dbt-core/blob/main/models/sources/sources.yml) -- [staging models code](https://github.com/kzzzr/mybi-dbt-core/tree/main/models/staging) -- [tests and docs](https://github.com/kzzzr/mybi-dbt-core/blob/main/models/staging/general/general.yml) - -All you need to do is just build these models in one command: - -```bash -dbt build -s tag:staging -``` - -![Build staging models](./docs/4_build_staging_models.gif) - -## Model Data Marts - -With staging models in place we now can proceed to data modeling. - -1. Intermediate models include: - - - Wide tables for Google Analytics [goals](./models/intermediate/ga/int_ga_goals_facts.sql) and [sessions](./models/intermediate/ga/int_ga_sessions_facts.sql) - - Aggregated table for myTarget [campaigns facts](./models/intermediate/mytarget/int_mytarget_campaigns_facts.sql) - - Aggregated tables for Yandex.Direct [context](./models/intermediate/yd/int_yd_campaigns_facts_context.sql) and [search](./models/intermediate/yd/int_yd_campaigns_facts_search.sql) facts - - [Comprehensive testing](./models/intermediate/intermediate.yml) - - ```bash - dbt build -s tag:intermediate - ``` - -2. Data Marts include: - - - [Costs](./models/marts/f_costs.sql) (uniting Yandex.Direct + myTarget) - - [Google Analytics events](./models/marts/f_ga_events.sql) (uniting sessions and goal completions) - - [Tracker](./models/marts/f_tracker.sql) which combines costs and events in a single table - - ```bash - dbt build --full-refresh -s tag:marts - ``` - - Take a look at the project graph (DAG): - - ![Data marts graph](./docs/5_marts_graph.png) - -## Set up metrics layer - -## dbt Metrics - -- Define dbt Metrics in [metrics.yml](./models/metrics/metrics.yml) -- Access dbt Metrics via materalizing in [f_metrics.sql](./models/metrics/f_metrics.sql) - -## Cube Metrics - -- Define dbt Metrics in [f_tracker.yml](./schema/f_tracker.yml) -- Access dbt Metrics through [Cube playground](http://localhost:4000/#/build?query={%22measures%22:[%22f_tracker.costs%22,%22f_tracker.clicks%22,%22f_tracker.CPC%22],%22order%22:{%22f_tracker.costs%22:%22desc%22},%22dimensions%22:[%22f_tracker.location_country%22],%22timeDimensions%22:[{%22dimension%22:%22f_tracker.date%22,%22granularity%22:%22week%22,%22dateRange%22:%22This%20month%22}]}) or [Metabase](http://localhost:3000/question/13-cubejs-sample-viz) - -![Cube Playground](./docs/9_cube_playground.png) - -## Visualize on a dashboard - -Now we are ready to visualize key metrics on a dashboard. - -I have configured Clickhouse connection and prepared Metabase dashboard which you can access at http://localhost:3000/dashboard/1-mybi-tutorial: -- Email address: `mybi@dbt.tutorial` -- Password: `tutorial101` - -![Explore data from Metabase dashboard](./docs/6_metabase_dashboard.gif) - -You may explore data from Metabase yourself or even build your own dashboard. - - -## Publish dbt project docs - -dbt Docs can be easily served locally on http://localhost:8080: - -```bash -dbt docs generate -dbt docs serve -``` - -Or you may access pre-build version from [Github Pages](https://kzzzr.github.io/mybi-dbt-showcase/#!/overview): - -![Access dbt project docs](./docs/7_dbt_docs.gif) - -## Introduce Continuous Integration - -Let's say you want to introduce some code changes. How do you ensure data quality? - -You protect your `main` branch and require Pull Requests to have: -- ✅ [Continuous Integration checks](./.github/workflows/ci.yml) successfully pass -- ✅ Code review from team member of Code Owner - -![Continuous Integration Checks](./docs/8_ci_checks.gif) - -This way you make sure to deploy high quality and functional code. - -## Contributing - +# myBI Market + dbt demo project + +![](https://habrastorage.org/webt/l-/1r/pq/l-1rpqoplxi-503grfeyyglux8g.jpeg) + +This repo guides you through building analyics for [myBI Market](https://market.mybi.ru/) with Modern Data Stack: + +- myBI Connect (Extract - Load tool) +- Clickhouse (Database) +- dbt (Transformations) + [mybi-dbt-core](https://github.com/kzzzr/mybi-dbt-core) module +- Metabase (Business Intelligence) +- Github Actions (Continuous Integration + Deployment) + +## Table of Contents + +1. [Configure environment](#configure-environment) +1. [Install and configure mybi_dbt_core package](#install-and-configure-mybi_dbt_core-package) +1. [Configure data sources](#configure-data-sources) +1. [Build staging layer models](#build-staging-layer-models) +1. [Model Data Marts](#model-data-marts) +1. [Set up metrics layer](#set-up-metrics-layer) +1. [Visualize on a dashboard](#visualize-on-a-dashboard) +1. [Publish dbt project docs](#publish-dbt-project-docs) +1. [Introduce Continuous Integration](#introduce-continuous-integration) + +## Configure environment + +1. Install prerequisites: + - IDE (e.g. [VS Code](https://code.visualstudio.com/docs/setup/setup-overview)) + - [Docker](https://docs.docker.com/engine/install/) + +2. Fork & Clone this repository and open in IDE + +3. Spin up Docker containers + + All the services are configured via [Docker containers](./docker-compose.yml). + + - Clickhouse + - Metabase + - Cube + + ```bash + # launch containers: clickhouse, metabase, cube + docker-compose build --no-cache + docker-compose up -d + ``` + +4. Open dev container with dbt installed + + ```bash + devcontainer open . + + # test connections + dbt --version + dbt debug + ``` + + ![Spin up Docker containers](./docs/1_docker_containers.gif) + +## Install and configure [mybi_dbt_core](https://github.com/kzzzr/mybi-dbt-core) package + +1. Install module via [packages.yml](./packages.yml) + + ```bash + dbt clean # clean temp files + dbt deps # install dependencies (modules) + ``` + +2. ✅ Enable only [relevant data models](./dbt_project.yml#L7-L20) (and disable the rest) + + We will use specific data sources: + + * general – [General]() + * direct – [Yandex.Direct](https://docs.mybi.ru/yandeks-direkt-struktura-bazovoy-vygruzki/) + * mytarget – [myTarget](https://docs.mybi.ru/mytarget-struktura-bazovoy-vygruzki/) + * amocrm – [AmoCRM](https://docs.mybi.ru/amocrm-struktura-bazovoy-vygruzki/) + * ga – [Google Analytics](https://docs.mybi.ru/google-analytics-struktura-bazovoy-vygruzki/) + + Confirm with command: `dbt ls --resource-type model -s tag:staging` + + ![Confirm specfic data sources enabled](./docs/2_enable_specific_data_sources.gif) + +3. ✅ Assign variables + + Certain variable [values have to be set](./dbt_project.yml#L28-L46): + - source database connection details + - database and schema name to find mybi source data + - specific `account_id` values to be included + +4. ✅ Turn on custom schema management + + I use [generate_schema_name.sql](./macros/generate_schema_name.sql) macro to set target schema names: + + > Renders a schema name given a custom schema name. In production, this macro + > will render out the overriden schema name for a model. Otherwise, the default + > schema specified in the active target is used. + + Take a look at [custom_schema_management.sql](https://github.com/kzzzr/mybi-dbt-core/blob/main/macros/custom_schema_management.sql) macro to find out more. + +## Configure data sources + +1. [Create](./macros/init_source_data.sql) Clickhouse database of [PostgreSQL Database Engine](https://clickhouse.com/docs/en/engines/database-engines/postgresql/) with source data: + + ```bash + dbt run-operation init_source_data + ``` + + ![Initialized myBI source database](./docs/3_init_source_data.gif) + +## Build staging layer models + +Staging models are alredy configured for you in [mybi_dbt_core](https://github.com/kzzzr/mybi-dbt-core) package: +- [source definitions](https://github.com/kzzzr/mybi-dbt-core/blob/main/models/sources/sources.yml) +- [staging models code](https://github.com/kzzzr/mybi-dbt-core/tree/main/models/staging) +- [tests and docs](https://github.com/kzzzr/mybi-dbt-core/blob/main/models/staging/general/general.yml) + +All you need to do is just build these models in one command: + +```bash +dbt build -s tag:staging +``` + +![Build staging models](./docs/4_build_staging_models.gif) + +## Model Data Marts + +With staging models in place we now can proceed to data modeling. + +1. Intermediate models include: + + - Wide tables for Google Analytics [goals](./models/intermediate/ga/int_ga_goals_facts.sql) and [sessions](./models/intermediate/ga/int_ga_sessions_facts.sql) + - Aggregated table for myTarget [campaigns facts](./models/intermediate/mytarget/int_mytarget_campaigns_facts.sql) + - Aggregated tables for Yandex.Direct [context](./models/intermediate/yd/int_yd_campaigns_facts_context.sql) and [search](./models/intermediate/yd/int_yd_campaigns_facts_search.sql) facts + - [Comprehensive testing](./models/intermediate/intermediate.yml) + + ```bash + dbt build -s tag:intermediate + ``` + +2. Data Marts include: + + - [Costs](./models/marts/f_costs.sql) (uniting Yandex.Direct + myTarget) + - [Google Analytics events](./models/marts/f_ga_events.sql) (uniting sessions and goal completions) + - [Tracker](./models/marts/f_tracker.sql) which combines costs and events in a single table + + ```bash + dbt build --full-refresh -s tag:marts + ``` + + Take a look at the project graph (DAG): + + ![Data marts graph](./docs/5_marts_graph.png) + +## Set up metrics layer + +## dbt Metrics + +- Define dbt Metrics in [metrics.yml](./models/metrics/metrics.yml) +- Access dbt Metrics via materalizing in [f_metrics.sql](./models/metrics/f_metrics.sql) + +## Cube Metrics + +- Define dbt Metrics in [f_tracker.yml](./schema/f_tracker.yml) +- Access dbt Metrics through [Cube playground](http://localhost:4000/#/build?query={%22measures%22:[%22f_tracker.costs%22,%22f_tracker.clicks%22,%22f_tracker.CPC%22],%22order%22:{%22f_tracker.costs%22:%22desc%22},%22dimensions%22:[%22f_tracker.location_country%22],%22timeDimensions%22:[{%22dimension%22:%22f_tracker.date%22,%22granularity%22:%22week%22,%22dateRange%22:%22This%20month%22}]}) or [Metabase](http://localhost:3000/question/13-cubejs-sample-viz) + +![Cube Playground](./docs/9_cube_playground.png) + +## Visualize on a dashboard + +Now we are ready to visualize key metrics on a dashboard. + +I have configured Clickhouse connection and prepared Metabase dashboard which you can access at http://localhost:3000/dashboard/1-mybi-tutorial: +- Email address: `mybi@dbt.tutorial` +- Password: `tutorial101` + +![Explore data from Metabase dashboard](./docs/6_metabase_dashboard.gif) + +You may explore data from Metabase yourself or even build your own dashboard. + + +## Publish dbt project docs + +dbt Docs can be easily served locally on http://localhost:8080: + +```bash +dbt docs generate +dbt docs serve +``` + +Or you may access pre-build version from [Github Pages](https://kzzzr.github.io/mybi-dbt-showcase/#!/overview): + +![Access dbt project docs](./docs/7_dbt_docs.gif) + +## Introduce Continuous Integration + +Let's say you want to introduce some code changes. How do you ensure data quality? + +You protect your `main` branch and require Pull Requests to have: +- ✅ [Continuous Integration checks](./.github/workflows/ci.yml) successfully pass +- ✅ Code review from team member of Code Owner + +![Continuous Integration Checks](./docs/8_ci_checks.gif) + +This way you make sure to deploy high quality and functional code. + +## Contributing + If you have any questions or comments please create an issue. \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index c54c00c..2edeb50 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,65 +1,65 @@ -name: 'mybi_dbt_showcase' -version: '1.0.0' -config-version: 2 - -profile: 'mybi_dbt_showcase' - -models: - mybi_dbt_core: - enabled: false - staging: - amocrm: - enabled: true - direct: - enabled: true - ga: - enabled: true - general: - enabled: true - mytarget: - enabled: true - - mybi_dbt_showcase: - +materialized: table - intermediate: - +schema: intermediate - +tags: ["intermediate"] - marts: - +tags: ["marts"] - - metrics: - dbt_metrics_default_calendar: - enabled: false - -vars: - mybi_host: 'rc1a-uifqtk602qck1es3.mdb.yandexcloud.net' - mybi_port: 6432 - mybi_database: 'mybi-ilmwayp' - mybi_user: 'guest-ilmwayp' - mybi_password: 'nAYIVOexUj6s' - mybi_schema: 'public' - - - mybi_dbt_core: - limit_data_days: 5 - - dbt_source_database: mybi - dbt_source_schema: mybi - - account_id_amocrm: 46021 - account_id_direct: 46105 - account_id_ga: 45377 - account_id_mytarget: 46637 - - -model-paths: ["models"] -analysis-paths: ["analyses"] -test-paths: ["tests"] -seed-paths: ["seeds"] -macro-paths: ["macros"] -snapshot-paths: ["snapshots"] - -target-path: "target" # directory which will store compiled SQL files -clean-targets: # directories to be removed by `dbt clean` - - "target" +name: 'mybi_dbt_showcase' +version: '1.0.0' +config-version: 2 + +profile: 'mybi_dbt_showcase' + +models: + mybi_dbt_core: + enabled: false + staging: + amocrm: + enabled: true + direct: + enabled: true + ga: + enabled: true + general: + enabled: true + mytarget: + enabled: true + + mybi_dbt_showcase: + +materialized: table + intermediate: + +schema: intermediate + +tags: ["intermediate"] + marts: + +tags: ["marts"] + + metrics: + dbt_metrics_default_calendar: + enabled: false + +vars: + mybi_host: 'rc1a-uifqtk602qck1es3.mdb.yandexcloud.net' + mybi_port: 6432 + mybi_database: 'mybi-ilmwayp' + mybi_user: 'guest-ilmwayp' + mybi_password: 'nAYIVOexUj6s' + mybi_schema: 'public' + + + mybi_dbt_core: + limit_data_days: 5 + + dbt_source_database: mybi + dbt_source_schema: mybi + + account_id_amocrm: 46021 + account_id_direct: 46105 + account_id_ga: 45377 + account_id_mytarget: 46637 + + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" - "dbt_packages" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4cd4c48..cf141bf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,29 +1,29 @@ -version: '2' -services: - - clickhouse: - image: clickhouse/clickhouse-server:22.3 - ports: - - 9000:9000 - - 8123:8123 - - metabase: - image: metabase/metabase:v0.44.6 - volumes: - - ./metabase/plugins:/plugins - - ./metabase/metabase.db:/metabase.db/ - environment: - - MB_DB_TYPE=h2 - - MB_DB_FILE=/metabase.db - ports: - - 3000:3000 - - cube: - image: cubejs/cube:latest - ports: - - 4000:4000 - - 15432:15432 - env_file: - - .env - volumes: - - .:/cube/conf +version: '2' +services: + + clickhouse: + image: clickhouse/clickhouse-server:22.3 + ports: + - 9000:9000 + - 8123:8123 + + metabase: + image: metabase/metabase:v0.44.6 + volumes: + - ./metabase/plugins:/plugins + - ./metabase/metabase.db:/metabase.db/ + environment: + - MB_DB_TYPE=h2 + - MB_DB_FILE=/metabase.db + ports: + - 3000:3000 + + cube: + image: cubejs/cube:latest + ports: + - 4000:4000 + - 15432:15432 + env_file: + - .env + volumes: + - .:/cube/conf diff --git a/macros/generate_schema_name.sql b/macros/generate_schema_name.sql index 6b67fe9..78d99c6 100644 --- a/macros/generate_schema_name.sql +++ b/macros/generate_schema_name.sql @@ -1,3 +1,3 @@ -{% macro generate_schema_name(custom_schema_name, node) -%} - {{ mybi_dbt_core.generate_schema_name_for_env(custom_schema_name, node) }} +{% macro generate_schema_name(custom_schema_name, node) -%} + {{ mybi_dbt_core.generate_schema_name_for_env(custom_schema_name, node) }} {%- endmacro %} \ No newline at end of file diff --git a/macros/init_source_data.sql b/macros/init_source_data.sql index b1f5be3..447ddb1 100644 --- a/macros/init_source_data.sql +++ b/macros/init_source_data.sql @@ -1,20 +1,20 @@ -{% macro init_source_data() -%} - - {% set sql %} - - CREATE DATABASE IF NOT EXISTS mybi - ENGINE = PostgreSQL( - '{{ var("mybi_host") }}:{{ var("mybi_port") }}' - , '{{ var("mybi_database") }}' - , '{{ var("mybi_user") }}' - , '{{ var("mybi_password") }}' - , '{{ var("mybi_schema") }}') - ; - - {% endset %} - - {% set result = run_query(sql) %} - - {{ print('Initialized myBI source database') }} - -{%- endmacro %} +{% macro init_source_data() -%} + + {% set sql %} + + CREATE DATABASE IF NOT EXISTS mybi + ENGINE = PostgreSQL( + '{{ var("mybi_host") }}:{{ var("mybi_port") }}' + , '{{ var("mybi_database") }}' + , '{{ var("mybi_user") }}' + , '{{ var("mybi_password") }}' + , '{{ var("mybi_schema") }}') + ; + + {% endset %} + + {% set result = run_query(sql) %} + + {{ print('Initialized myBI source database') }} + +{%- endmacro %} diff --git a/metabase/metabase.db/metabase.db.mv.db b/metabase/metabase.db/metabase.db.mv.db index afda70a..55b3d20 100644 Binary files a/metabase/metabase.db/metabase.db.mv.db and b/metabase/metabase.db/metabase.db.mv.db differ diff --git a/metabase/metabase.db/metabase.db.trace.db b/metabase/metabase.db/metabase.db.trace.db index 2264061..9d3b549 100644 --- a/metabase/metabase.db/metabase.db.trace.db +++ b/metabase/metabase.db/metabase.db.trace.db @@ -1,488 +1,488 @@ -2022-11-05 12:14:56 jdbc[6]: exception -org.h2.jdbc.JdbcSQLException: Table "DATABASECHANGELOGLOCK" not found; SQL statement: -SELECT COUNT(*) FROM "PUBLIC"."DATABASECHANGELOGLOCK" [42102-197] -2022-11-05 12:14:59 jdbc[6]: exception -org.h2.jdbc.JdbcSQLException: Column "T.SETTINGS" not found; SQL statement: -select t."SETTINGS" from "PUBLIC"."METABASE_DATABASE" t where 0=1 [42122-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.expression.ExpressionColumn.optimize(ExpressionColumn.java:150) - at org.h2.command.dml.Select.prepare(Select.java:858) - at org.h2.command.Parser.prepareCommand(Parser.java:283) - at org.h2.engine.Session.prepareLocal(Session.java:611) - at org.h2.engine.Session.prepareCommand(Session.java:549) - at org.h2.jdbc.JdbcConnection.prepareCommand(JdbcConnection.java:1247) - at org.h2.jdbc.JdbcStatement.executeQuery(JdbcStatement.java:78) - at com.mchange.v2.c3p0.impl.NewProxyStatement.executeQuery(NewProxyStatement.java:327) - at liquibase.precondition.core.ColumnExistsPrecondition.checkFast(ColumnExistsPrecondition.java:163) - at liquibase.precondition.core.ColumnExistsPrecondition.check(ColumnExistsPrecondition.java:81) - at liquibase.precondition.core.NotPrecondition.check(NotPrecondition.java:35) - at liquibase.precondition.core.AndPrecondition.check(AndPrecondition.java:40) - at liquibase.precondition.core.PreconditionContainer.check(PreconditionContainer.java:213) - at liquibase.changelog.ChangeSet.execute(ChangeSet.java:577) - at liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:56) - at liquibase.changelog.ChangeLogIterator$2.lambda$null$0(ChangeLogIterator.java:113) - at liquibase.Scope.lambda$child$0(Scope.java:180) - at liquibase.Scope.child(Scope.java:189) - at liquibase.Scope.child(Scope.java:179) - at liquibase.Scope.child(Scope.java:158) - at liquibase.changelog.ChangeLogIterator$2.lambda$run$1(ChangeLogIterator.java:112) - at liquibase.Scope.lambda$child$0(Scope.java:180) - at liquibase.Scope.child(Scope.java:189) - at liquibase.Scope.child(Scope.java:179) - at liquibase.Scope.child(Scope.java:158) - at liquibase.Scope.child(Scope.java:243) - at liquibase.changelog.ChangeLogIterator$2.run(ChangeLogIterator.java:93) - at liquibase.Scope.lambda$child$0(Scope.java:180) - at liquibase.Scope.child(Scope.java:189) - at liquibase.Scope.child(Scope.java:179) - at liquibase.Scope.child(Scope.java:158) - at liquibase.Scope.child(Scope.java:243) - at liquibase.Scope.child(Scope.java:247) - at liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:65) - at liquibase.Liquibase.lambda$null$0(Liquibase.java:265) - at liquibase.Scope.lambda$child$0(Scope.java:180) - at liquibase.Scope.child(Scope.java:189) - at liquibase.Scope.child(Scope.java:179) - at liquibase.Scope.child(Scope.java:158) - at liquibase.Scope.child(Scope.java:243) - at liquibase.Liquibase.lambda$update$1(Liquibase.java:264) - at liquibase.Scope.lambda$child$0(Scope.java:180) - at liquibase.Scope.child(Scope.java:189) - at liquibase.Scope.child(Scope.java:179) - at liquibase.Scope.child(Scope.java:158) - at liquibase.Liquibase.runInScope(Liquibase.java:2405) - at liquibase.Liquibase.update(Liquibase.java:211) - at liquibase.Liquibase.update(Liquibase.java:197) - at liquibase.Liquibase.update(Liquibase.java:193) - at metabase.db.liquibase$migrate_up_if_needed_BANG_.invokeStatic(liquibase.clj:142) - at metabase.db.liquibase$migrate_up_if_needed_BANG_.invoke(liquibase.clj:130) - at metabase.db.setup$fn__35437$migrate_BANG___35442$fn__35443$fn__35444.invoke(setup.clj:66) - at metabase.db.liquibase$fn__30951$do_with_liquibase__30956$fn__30957.invoke(liquibase.clj:59) - at metabase.db.liquibase$fn__30951$do_with_liquibase__30956.invoke(liquibase.clj:51) - at metabase.db.setup$fn__35437$migrate_BANG___35442$fn__35443.invoke(setup.clj:61) - at metabase.db.setup$fn__35437$migrate_BANG___35442.invoke(setup.clj:40) - at metabase.db.setup$fn__35496$run_schema_migrations_BANG___35501$fn__35502.invoke(setup.clj:119) - at metabase.db.setup$fn__35496$run_schema_migrations_BANG___35501.invoke(setup.clj:113) - at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554$fn__35557$fn__35558.invoke(setup.clj:145) - at metabase.util$do_with_us_locale.invokeStatic(util.clj:716) - at metabase.util$do_with_us_locale.invoke(util.clj:702) - at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554$fn__35557.invoke(setup.clj:143) - at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554.invoke(setup.clj:142) - at metabase.db.setup$fn__35548$setup_db_BANG___35553.invoke(setup.clj:136) - at metabase.db$setup_db_BANG_$fn__35583.invoke(db.clj:65) - at metabase.db$setup_db_BANG_.invokeStatic(db.clj:60) - at metabase.db$setup_db_BANG_.invoke(db.clj:51) - at metabase.core$init_BANG__STAR_.invokeStatic(core.clj:100) - at metabase.core$init_BANG__STAR_.invoke(core.clj:83) - at metabase.core$init_BANG_.invokeStatic(core.clj:145) - at metabase.core$init_BANG_.invoke(core.clj:140) - at metabase.core$start_normally.invokeStatic(core.clj:157) - at metabase.core$start_normally.invoke(core.clj:151) - at metabase.core$_main.invokeStatic(core.clj:190) - at metabase.core$_main.doInvoke(core.clj:184) - at clojure.lang.RestFn.invoke(RestFn.java:397) - at clojure.lang.AFn.applyToHelper(AFn.java:152) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at metabase.core.main(Unknown Source) -2022-11-07 08:31:27 jdbc[8]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) - at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) - at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) - at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) - at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) -2022-11-07 16:47:31 jdbc[9]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) - at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) - at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) - at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) - at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) -2023-04-25 12:29:31 jdbc[3]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:302) - at com.mchange.v2.c3p0.impl.NewProxyConnection.prepareStatement(NewProxyConnection.java:567) - at clojure.java.jdbc$prepare_statement.invokeStatic(jdbc.clj:679) - at clojure.java.jdbc$prepare_statement.invoke(jdbc.clj:626) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1112) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) -2023-04-25 12:29:31 jdbc[3]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) - at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) - at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) - at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) - at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) -2023-04-25 13:07:46 jdbc[6]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:302) - at com.mchange.v2.c3p0.impl.NewProxyConnection.prepareStatement(NewProxyConnection.java:567) - at clojure.java.jdbc$prepare_statement.invokeStatic(jdbc.clj:679) - at clojure.java.jdbc$prepare_statement.invoke(jdbc.clj:626) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1112) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) -2023-04-25 13:07:46 jdbc[6]: exception -org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] - at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) - at org.h2.message.DbException.get(DbException.java:179) - at org.h2.message.DbException.get(DbException.java:155) - at org.h2.message.DbException.get(DbException.java:144) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) - at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) - at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) - at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) - at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) - at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) - at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) - at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) - at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) - at clojure.java.jdbc$query.invoke(jdbc.clj:1144) - at toucan.db$query.invokeStatic(db.clj:308) - at toucan.db$query.doInvoke(db.clj:304) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at toucan.db$simple_select.invokeStatic(db.clj:414) - at toucan.db$simple_select.invoke(db.clj:403) - at toucan.db$simple_select_one.invokeStatic(db.clj:440) - at toucan.db$simple_select_one.invoke(db.clj:429) - at toucan.db$select_one.invokeStatic(db.clj:670) - at toucan.db$select_one.doInvoke(db.clj:664) - at clojure.lang.RestFn.applyTo(RestFn.java:139) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at toucan.db$select_one_field.invokeStatic(db.clj:682) - at toucan.db$select_one_field.doInvoke(db.clj:675) - at clojure.lang.RestFn.invoke(RestFn.java:442) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) - at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) - at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) - at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) - at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) - at metabase.models.setting$get_raw_value.invoke(setting.clj:432) - at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) - at metabase.models.setting$fn__30570.invoke(setting.clj:489) - at clojure.lang.MultiFn.invoke(MultiFn.java:234) - at clojure.lang.Var.invoke(Var.java:388) - at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) - at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) - at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) - at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) - at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) - at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) - at metabase.util.i18n$site_locale.invoke(i18n.clj:47) - at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) - at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) - at clojure.lang.RestFn.invoke(RestFn.java:410) - at clojure.lang.AFn.applyToHelper(AFn.java:154) - at clojure.lang.RestFn.applyTo(RestFn.java:132) - at clojure.core$apply.invokeStatic(core.clj:669) - at clojure.core$apply.invoke(core.clj:662) - at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) - at clojure.core$str.invokeStatic(core.clj:555) - at clojure.core$str.invoke(core.clj:546) - at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) - at metabase.core$destroy_BANG_.invoke(core.clj:72) - at clojure.lang.AFn.run(AFn.java:22) - at java.base/java.lang.Thread.run(Unknown Source) +2022-11-05 12:14:56 jdbc[6]: exception +org.h2.jdbc.JdbcSQLException: Table "DATABASECHANGELOGLOCK" not found; SQL statement: +SELECT COUNT(*) FROM "PUBLIC"."DATABASECHANGELOGLOCK" [42102-197] +2022-11-05 12:14:59 jdbc[6]: exception +org.h2.jdbc.JdbcSQLException: Column "T.SETTINGS" not found; SQL statement: +select t."SETTINGS" from "PUBLIC"."METABASE_DATABASE" t where 0=1 [42122-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.expression.ExpressionColumn.optimize(ExpressionColumn.java:150) + at org.h2.command.dml.Select.prepare(Select.java:858) + at org.h2.command.Parser.prepareCommand(Parser.java:283) + at org.h2.engine.Session.prepareLocal(Session.java:611) + at org.h2.engine.Session.prepareCommand(Session.java:549) + at org.h2.jdbc.JdbcConnection.prepareCommand(JdbcConnection.java:1247) + at org.h2.jdbc.JdbcStatement.executeQuery(JdbcStatement.java:78) + at com.mchange.v2.c3p0.impl.NewProxyStatement.executeQuery(NewProxyStatement.java:327) + at liquibase.precondition.core.ColumnExistsPrecondition.checkFast(ColumnExistsPrecondition.java:163) + at liquibase.precondition.core.ColumnExistsPrecondition.check(ColumnExistsPrecondition.java:81) + at liquibase.precondition.core.NotPrecondition.check(NotPrecondition.java:35) + at liquibase.precondition.core.AndPrecondition.check(AndPrecondition.java:40) + at liquibase.precondition.core.PreconditionContainer.check(PreconditionContainer.java:213) + at liquibase.changelog.ChangeSet.execute(ChangeSet.java:577) + at liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:56) + at liquibase.changelog.ChangeLogIterator$2.lambda$null$0(ChangeLogIterator.java:113) + at liquibase.Scope.lambda$child$0(Scope.java:180) + at liquibase.Scope.child(Scope.java:189) + at liquibase.Scope.child(Scope.java:179) + at liquibase.Scope.child(Scope.java:158) + at liquibase.changelog.ChangeLogIterator$2.lambda$run$1(ChangeLogIterator.java:112) + at liquibase.Scope.lambda$child$0(Scope.java:180) + at liquibase.Scope.child(Scope.java:189) + at liquibase.Scope.child(Scope.java:179) + at liquibase.Scope.child(Scope.java:158) + at liquibase.Scope.child(Scope.java:243) + at liquibase.changelog.ChangeLogIterator$2.run(ChangeLogIterator.java:93) + at liquibase.Scope.lambda$child$0(Scope.java:180) + at liquibase.Scope.child(Scope.java:189) + at liquibase.Scope.child(Scope.java:179) + at liquibase.Scope.child(Scope.java:158) + at liquibase.Scope.child(Scope.java:243) + at liquibase.Scope.child(Scope.java:247) + at liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:65) + at liquibase.Liquibase.lambda$null$0(Liquibase.java:265) + at liquibase.Scope.lambda$child$0(Scope.java:180) + at liquibase.Scope.child(Scope.java:189) + at liquibase.Scope.child(Scope.java:179) + at liquibase.Scope.child(Scope.java:158) + at liquibase.Scope.child(Scope.java:243) + at liquibase.Liquibase.lambda$update$1(Liquibase.java:264) + at liquibase.Scope.lambda$child$0(Scope.java:180) + at liquibase.Scope.child(Scope.java:189) + at liquibase.Scope.child(Scope.java:179) + at liquibase.Scope.child(Scope.java:158) + at liquibase.Liquibase.runInScope(Liquibase.java:2405) + at liquibase.Liquibase.update(Liquibase.java:211) + at liquibase.Liquibase.update(Liquibase.java:197) + at liquibase.Liquibase.update(Liquibase.java:193) + at metabase.db.liquibase$migrate_up_if_needed_BANG_.invokeStatic(liquibase.clj:142) + at metabase.db.liquibase$migrate_up_if_needed_BANG_.invoke(liquibase.clj:130) + at metabase.db.setup$fn__35437$migrate_BANG___35442$fn__35443$fn__35444.invoke(setup.clj:66) + at metabase.db.liquibase$fn__30951$do_with_liquibase__30956$fn__30957.invoke(liquibase.clj:59) + at metabase.db.liquibase$fn__30951$do_with_liquibase__30956.invoke(liquibase.clj:51) + at metabase.db.setup$fn__35437$migrate_BANG___35442$fn__35443.invoke(setup.clj:61) + at metabase.db.setup$fn__35437$migrate_BANG___35442.invoke(setup.clj:40) + at metabase.db.setup$fn__35496$run_schema_migrations_BANG___35501$fn__35502.invoke(setup.clj:119) + at metabase.db.setup$fn__35496$run_schema_migrations_BANG___35501.invoke(setup.clj:113) + at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554$fn__35557$fn__35558.invoke(setup.clj:145) + at metabase.util$do_with_us_locale.invokeStatic(util.clj:716) + at metabase.util$do_with_us_locale.invoke(util.clj:702) + at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554$fn__35557.invoke(setup.clj:143) + at metabase.db.setup$fn__35548$setup_db_BANG___35553$fn__35554.invoke(setup.clj:142) + at metabase.db.setup$fn__35548$setup_db_BANG___35553.invoke(setup.clj:136) + at metabase.db$setup_db_BANG_$fn__35583.invoke(db.clj:65) + at metabase.db$setup_db_BANG_.invokeStatic(db.clj:60) + at metabase.db$setup_db_BANG_.invoke(db.clj:51) + at metabase.core$init_BANG__STAR_.invokeStatic(core.clj:100) + at metabase.core$init_BANG__STAR_.invoke(core.clj:83) + at metabase.core$init_BANG_.invokeStatic(core.clj:145) + at metabase.core$init_BANG_.invoke(core.clj:140) + at metabase.core$start_normally.invokeStatic(core.clj:157) + at metabase.core$start_normally.invoke(core.clj:151) + at metabase.core$_main.invokeStatic(core.clj:190) + at metabase.core$_main.doInvoke(core.clj:184) + at clojure.lang.RestFn.invoke(RestFn.java:397) + at clojure.lang.AFn.applyToHelper(AFn.java:152) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at metabase.core.main(Unknown Source) +2022-11-07 08:31:27 jdbc[8]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) + at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) + at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) + at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) + at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) +2022-11-07 16:47:31 jdbc[9]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) + at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) + at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) + at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) + at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) +2023-04-25 12:29:31 jdbc[3]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:302) + at com.mchange.v2.c3p0.impl.NewProxyConnection.prepareStatement(NewProxyConnection.java:567) + at clojure.java.jdbc$prepare_statement.invokeStatic(jdbc.clj:679) + at clojure.java.jdbc$prepare_statement.invoke(jdbc.clj:626) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1112) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) +2023-04-25 12:29:31 jdbc[3]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) + at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) + at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) + at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) + at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) +2023-04-25 13:07:46 jdbc[6]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.prepareStatement(JdbcConnection.java:302) + at com.mchange.v2.c3p0.impl.NewProxyConnection.prepareStatement(NewProxyConnection.java:567) + at clojure.java.jdbc$prepare_statement.invokeStatic(jdbc.clj:679) + at clojure.java.jdbc$prepare_statement.invoke(jdbc.clj:626) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1112) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) +2023-04-25 13:07:46 jdbc[6]: exception +org.h2.jdbc.JdbcSQLException: Database is already closed (to disable automatic closing at VM shutdown, add ";DB_CLOSE_ON_EXIT=FALSE" to the db URL) [90121-197] + at org.h2.message.DbException.getJdbcSQLException(DbException.java:357) + at org.h2.message.DbException.get(DbException.java:179) + at org.h2.message.DbException.get(DbException.java:155) + at org.h2.message.DbException.get(DbException.java:144) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1526) + at org.h2.jdbc.JdbcConnection.checkClosed(JdbcConnection.java:1502) + at org.h2.jdbc.JdbcConnection.getAutoCommit(JdbcConnection.java:476) + at com.mchange.v2.c3p0.impl.C3P0ImplUtils.resetTxnState(C3P0ImplUtils.java:245) + at com.mchange.v2.c3p0.impl.NewPooledConnection.reset(NewPooledConnection.java:461) + at com.mchange.v2.c3p0.impl.NewPooledConnection.markClosedProxyConnection(NewPooledConnection.java:417) + at com.mchange.v2.c3p0.impl.NewProxyConnection.close(NewProxyConnection.java:87) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invokeStatic(jdbc.clj:1111) + at clojure.java.jdbc$db_query_with_resultset_STAR_.invoke(jdbc.clj:1093) + at clojure.java.jdbc$query.invokeStatic(jdbc.clj:1182) + at clojure.java.jdbc$query.invoke(jdbc.clj:1144) + at toucan.db$query.invokeStatic(db.clj:308) + at toucan.db$query.doInvoke(db.clj:304) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at toucan.db$simple_select.invokeStatic(db.clj:414) + at toucan.db$simple_select.invoke(db.clj:403) + at toucan.db$simple_select_one.invokeStatic(db.clj:440) + at toucan.db$simple_select_one.invoke(db.clj:429) + at toucan.db$select_one.invokeStatic(db.clj:670) + at toucan.db$select_one.doInvoke(db.clj:664) + at clojure.lang.RestFn.applyTo(RestFn.java:139) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at toucan.db$select_one_field.invokeStatic(db.clj:682) + at toucan.db$select_one_field.doInvoke(db.clj:675) + at clojure.lang.RestFn.invoke(RestFn.java:442) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invokeStatic(cache.clj:111) + at metabase.models.setting.cache$cache_out_of_date_QMARK_.invoke(cache.clj:92) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invokeStatic(cache.clj:161) + at metabase.models.setting.cache$restore_cache_if_needed_BANG_.invoke(cache.clj:141) + at metabase.models.setting$db_or_cache_value.invokeStatic(setting.clj:417) + at metabase.models.setting$db_or_cache_value.invoke(setting.clj:404) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:458) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$get_raw_value.invokeStatic(setting.clj:472) + at metabase.models.setting$get_raw_value.invoke(setting.clj:432) + at metabase.models.setting$fn__30570.invokeStatic(setting.clj:491) + at metabase.models.setting$fn__30570.invoke(setting.clj:489) + at clojure.lang.MultiFn.invoke(MultiFn.java:234) + at clojure.lang.Var.invoke(Var.java:388) + at metabase.util.i18n.impl$fn__1646$f__1647.invoke(impl.clj:188) + at metabase.util.i18n.impl$site_locale_from_setting.invokeStatic(impl.clj:199) + at metabase.util.i18n.impl$site_locale_from_setting.invoke(impl.clj:192) + at metabase.util.i18n$site_locale_string.invokeStatic(i18n.clj:37) + at metabase.util.i18n$site_locale_string.invoke(i18n.clj:32) + at metabase.util.i18n$site_locale.invokeStatic(i18n.clj:50) + at metabase.util.i18n$site_locale.invoke(i18n.clj:47) + at metabase.util.i18n$translate_site_locale.invokeStatic(i18n.clj:69) + at metabase.util.i18n$translate_site_locale.doInvoke(i18n.clj:66) + at clojure.lang.RestFn.invoke(RestFn.java:410) + at clojure.lang.AFn.applyToHelper(AFn.java:154) + at clojure.lang.RestFn.applyTo(RestFn.java:132) + at clojure.core$apply.invokeStatic(core.clj:669) + at clojure.core$apply.invoke(core.clj:662) + at metabase.util.i18n.SiteLocalizedString.toString(i18n.clj:94) + at clojure.core$str.invokeStatic(core.clj:555) + at clojure.core$str.invoke(core.clj:546) + at metabase.core$destroy_BANG_.invokeStatic(core.clj:75) + at metabase.core$destroy_BANG_.invoke(core.clj:72) + at clojure.lang.AFn.run(AFn.java:22) + at java.base/java.lang.Thread.run(Unknown Source) diff --git a/models/intermediate/ga/int_ga_goals_facts.sql b/models/intermediate/ga/int_ga_goals_facts.sql index 16e3d86..dcf9ddc 100644 --- a/models/intermediate/ga/int_ga_goals_facts.sql +++ b/models/intermediate/ga/int_ga_goals_facts.sql @@ -1,81 +1,81 @@ -SELECT - - facts.dt AS dt - , {{ dbt_utils.surrogate_key([ - 'facts.dt' - , 'goals.goal_id' - , 'facts.account_id' - , 'facts.sites_id' - , 'facts.clientids_id' - , 'facts.devices_id' - , 'facts.traffic_id' - , 'facts.locations_id' - ]) }} AS event_id - - -- GOALS - --, facts.goals_id - , goals.goal_id AS goal_id - , goals.name AS goal_name - , goals.active AS goal_active - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , accounts.service AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- SITES - --, facts.sites_id AS sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- CLIENTS - --, facts.clientids_id AS clientids_id - , clients.clientid AS client_id - , clients.userid AS client_userid - , clients.phone AS client_phone - , clients.email AS client_email - - -- DEVICES - --, facts.devices_id AS devices_id - , devices.category AS device_category - , devices.browser AS device_browser - , devices.browser_version AS device_browser_version - , devices.os AS device_os - , devices.os_version AS device_os_version - - -- TRAFFIC - --, facts.traffic_id AS traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword - , traffic.landing_page AS traffic_landing_page - - -- LOCATIONS - --, facts.locations_id AS locations_id - , locations.country_iso AS location_country_iso - , locations.country AS location_country - , locations.region_iso AS location_region_iso - , locations.region AS location_region - , locations.city AS location_city - , locations.latitude AS location_latitude - , locations.longitude AS location_longitude - - -- MEASURES - , facts.completions AS goal_completions - , facts.goal_value AS goal_value - -FROM {{ ref('stg_ga_goals_facts') }} AS facts - LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id - LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id - LEFT JOIN {{ ref('stg_general_clientids') }} AS clients ON facts.clientids_id = clients.id - LEFT JOIN {{ ref('stg_ga_devices') }} AS devices ON devices.id = facts.devices_id - LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id - LEFT JOIN {{ ref('stg_general_locations') }} AS locations ON locations.id = facts.locations_id - LEFT JOIN {{ ref('stg_ga_goals') }} AS goals ON goals.id = facts.goals_id +SELECT + + facts.dt AS dt + , {{ dbt_utils.surrogate_key([ + 'facts.dt' + , 'goals.goal_id' + , 'facts.account_id' + , 'facts.sites_id' + , 'facts.clientids_id' + , 'facts.devices_id' + , 'facts.traffic_id' + , 'facts.locations_id' + ]) }} AS event_id + + -- GOALS + --, facts.goals_id + , goals.goal_id AS goal_id + , goals.name AS goal_name + , goals.active AS goal_active + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , accounts.service AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- SITES + --, facts.sites_id AS sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- CLIENTS + --, facts.clientids_id AS clientids_id + , clients.clientid AS client_id + , clients.userid AS client_userid + , clients.phone AS client_phone + , clients.email AS client_email + + -- DEVICES + --, facts.devices_id AS devices_id + , devices.category AS device_category + , devices.browser AS device_browser + , devices.browser_version AS device_browser_version + , devices.os AS device_os + , devices.os_version AS device_os_version + + -- TRAFFIC + --, facts.traffic_id AS traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword + , traffic.landing_page AS traffic_landing_page + + -- LOCATIONS + --, facts.locations_id AS locations_id + , locations.country_iso AS location_country_iso + , locations.country AS location_country + , locations.region_iso AS location_region_iso + , locations.region AS location_region + , locations.city AS location_city + , locations.latitude AS location_latitude + , locations.longitude AS location_longitude + + -- MEASURES + , facts.completions AS goal_completions + , facts.goal_value AS goal_value + +FROM {{ ref('stg_ga_goals_facts') }} AS facts + LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id + LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id + LEFT JOIN {{ ref('stg_general_clientids') }} AS clients ON facts.clientids_id = clients.id + LEFT JOIN {{ ref('stg_ga_devices') }} AS devices ON devices.id = facts.devices_id + LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id + LEFT JOIN {{ ref('stg_general_locations') }} AS locations ON locations.id = facts.locations_id + LEFT JOIN {{ ref('stg_ga_goals') }} AS goals ON goals.id = facts.goals_id diff --git a/models/intermediate/ga/int_ga_sessions_facts.sql b/models/intermediate/ga/int_ga_sessions_facts.sql index b2f00a9..4b29f38 100644 --- a/models/intermediate/ga/int_ga_sessions_facts.sql +++ b/models/intermediate/ga/int_ga_sessions_facts.sql @@ -1,74 +1,74 @@ -SELECT - - facts.dt AS dt - , {{ dbt_utils.surrogate_key([ - 'facts.dt' - , 'facts.session_id' - , 'facts.traffic_id' - , 'facts.user_type' - ]) }} AS event_id - , facts.session_id AS session_id - , facts.user_type AS user_type - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , accounts.service AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- SITES - --, facts.sites_id AS sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- CLIENTS - --, facts.clientids_id AS clientids_id - , clients.clientid AS client_id - , clients.userid AS client_userid - , clients.phone AS client_phone - , clients.email AS client_email - - -- DEVICES - --, facts.devices_id AS devices_id - , devices.category AS device_category - , devices.browser AS device_browser - , devices.browser_version AS device_browser_version - , devices.os AS device_os - , devices.os_version AS device_os_version - - -- TRAFFIC - --, facts.traffic_id AS traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword - , traffic.landing_page AS traffic_landing_page - - -- LOCATIONS - --, facts.locations_id AS locations_id - , locations.country_iso AS location_country_iso - , locations.country AS location_country - , locations.region_iso AS location_region_iso - , locations.region AS location_region - , locations.city AS location_city - , locations.latitude AS location_latitude - , locations.longitude AS location_longitude - - -- MEASURES - , facts.sessions AS sessions - , facts.bounces AS bounces - , facts.pageviews AS pageviews - , facts.duration AS duration - -FROM {{ ref('stg_ga_sessions_facts') }} AS facts - LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id - LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id - LEFT JOIN {{ ref('stg_general_clientids') }} AS clients ON facts.clientids_id = clients.id - LEFT JOIN {{ ref('stg_ga_devices') }} AS devices ON devices.id = facts.devices_id - LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id - LEFT JOIN {{ ref('stg_general_locations') }} AS locations ON locations.id = facts.locations_id +SELECT + + facts.dt AS dt + , {{ dbt_utils.surrogate_key([ + 'facts.dt' + , 'facts.session_id' + , 'facts.traffic_id' + , 'facts.user_type' + ]) }} AS event_id + , facts.session_id AS session_id + , facts.user_type AS user_type + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , accounts.service AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- SITES + --, facts.sites_id AS sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- CLIENTS + --, facts.clientids_id AS clientids_id + , clients.clientid AS client_id + , clients.userid AS client_userid + , clients.phone AS client_phone + , clients.email AS client_email + + -- DEVICES + --, facts.devices_id AS devices_id + , devices.category AS device_category + , devices.browser AS device_browser + , devices.browser_version AS device_browser_version + , devices.os AS device_os + , devices.os_version AS device_os_version + + -- TRAFFIC + --, facts.traffic_id AS traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword + , traffic.landing_page AS traffic_landing_page + + -- LOCATIONS + --, facts.locations_id AS locations_id + , locations.country_iso AS location_country_iso + , locations.country AS location_country + , locations.region_iso AS location_region_iso + , locations.region AS location_region + , locations.city AS location_city + , locations.latitude AS location_latitude + , locations.longitude AS location_longitude + + -- MEASURES + , facts.sessions AS sessions + , facts.bounces AS bounces + , facts.pageviews AS pageviews + , facts.duration AS duration + +FROM {{ ref('stg_ga_sessions_facts') }} AS facts + LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id + LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id + LEFT JOIN {{ ref('stg_general_clientids') }} AS clients ON facts.clientids_id = clients.id + LEFT JOIN {{ ref('stg_ga_devices') }} AS devices ON devices.id = facts.devices_id + LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id + LEFT JOIN {{ ref('stg_general_locations') }} AS locations ON locations.id = facts.locations_id diff --git a/models/intermediate/intermediate.yml b/models/intermediate/intermediate.yml index d4b0aee..573f199 100644 --- a/models/intermediate/intermediate.yml +++ b/models/intermediate/intermediate.yml @@ -1,69 +1,69 @@ -version: 2 - -models: - - name: int_yd_campaigns_facts_context - description: Yandex.Direct campaign-level facts on context (РСЯ) - columns: - - name: cost_id - description: surrogate key - tests: - - not_null - - unique - - name: dt - description: Date - tests: - - not_null - - - name: int_yd_campaigns_facts_search - description: Yandex.Direct campaign-level facts on search (Поиск) - columns: - - name: cost_id - description: surrogate key - tests: - - not_null - - unique - - name: dt - description: Date - tests: - - not_null - - - name: int_mytarget_campaigns_facts - description: myTarget campaign-level facts - columns: - - name: cost_id - description: surrogate key - tests: - - not_null - - unique - - name: dt - description: Date - tests: - - not_null - - - name: int_ga_sessions_facts - description: Google Analytics sessions facts - columns: - - name: event_id - description: Event unique identifier - tests: - - not_null - - unique: - severity: warn - - name: dt - description: Date - tests: - - not_null - - - name: int_ga_goals_facts - description: Google Analytics goal completions facts - columns: - - name: event_id - description: Event unique identifier - tests: - - not_null - - unique: - severity: warn - - name: dt - description: Date - tests: - - not_null +version: 2 + +models: + - name: int_yd_campaigns_facts_context + description: Yandex.Direct campaign-level facts on context (РСЯ) + columns: + - name: cost_id + description: surrogate key + tests: + - not_null + - unique + - name: dt + description: Date + tests: + - not_null + + - name: int_yd_campaigns_facts_search + description: Yandex.Direct campaign-level facts on search (Поиск) + columns: + - name: cost_id + description: surrogate key + tests: + - not_null + - unique + - name: dt + description: Date + tests: + - not_null + + - name: int_mytarget_campaigns_facts + description: myTarget campaign-level facts + columns: + - name: cost_id + description: surrogate key + tests: + - not_null + - unique + - name: dt + description: Date + tests: + - not_null + + - name: int_ga_sessions_facts + description: Google Analytics sessions facts + columns: + - name: event_id + description: Event unique identifier + tests: + - not_null + - unique: + severity: warn + - name: dt + description: Date + tests: + - not_null + + - name: int_ga_goals_facts + description: Google Analytics goal completions facts + columns: + - name: event_id + description: Event unique identifier + tests: + - not_null + - unique: + severity: warn + - name: dt + description: Date + tests: + - not_null diff --git a/models/intermediate/mytarget/int_mytarget_campaigns_facts.sql b/models/intermediate/mytarget/int_mytarget_campaigns_facts.sql index bfddf6b..97b605b 100644 --- a/models/intermediate/mytarget/int_mytarget_campaigns_facts.sql +++ b/models/intermediate/mytarget/int_mytarget_campaigns_facts.sql @@ -1,102 +1,102 @@ -SELECT - - facts.dt AS dt - , {{ dbt_utils.surrogate_key([ - 'facts.dt' - , 'facts.account_id' - , 'facts.campaigns_id' - , 'facts.sites_id' - , 'facts.traffic_id' - ]) }} AS cost_id - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , accounts.service AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id AS campaign_id - , campaigns.name AS campaign_name - , campaigns.system_status AS campaign_system_status - , campaigns.status AS campaign_status - , campaigns.autobidding AS campaign_autobidding - , campaigns.mixing AS campaign_mixing - , campaigns.age_restrictions AS campaign_age_restrictions - , campaigns.group_members AS campaign_group_members - , campaigns.extended_age AS campaign_extended_age - , campaigns.enable_utm AS campaign_enable_utm - , campaigns.utm AS campaign_utm - - -- SITES - --, facts.sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- TRAFFIC - --, facts.traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword - , traffic.landing_page AS traffic_landing_page - - -- MEASURES - , sum(facts.impressions) AS impressions - , sum(facts.clicks) AS clicks - , sum(facts.cost) AS cost - -FROM {{ ref('stg_mt_banners_facts') }} AS facts - LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id - LEFT JOIN {{ ref('stg_mt_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id - LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id - LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id - -GROUP BY - - facts.dt - , cost_id - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , accounts.service AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id AS campaign_id - , campaigns.name AS campaign_name - , campaigns.system_status AS campaign_system_status - , campaigns.status AS campaign_status - , campaigns.autobidding AS campaign_autobidding - , campaigns.mixing AS campaign_mixing - , campaigns.age_restrictions AS campaign_age_restrictions - , campaigns.group_members AS campaign_group_members - , campaigns.extended_age AS campaign_extended_age - , campaigns.enable_utm AS campaign_enable_utm - , campaigns.utm AS campaign_utm - - -- SITES - --, facts.sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- TRAFFIC - --, facts.traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword +SELECT + + facts.dt AS dt + , {{ dbt_utils.surrogate_key([ + 'facts.dt' + , 'facts.account_id' + , 'facts.campaigns_id' + , 'facts.sites_id' + , 'facts.traffic_id' + ]) }} AS cost_id + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , accounts.service AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id AS campaign_id + , campaigns.name AS campaign_name + , campaigns.system_status AS campaign_system_status + , campaigns.status AS campaign_status + , campaigns.autobidding AS campaign_autobidding + , campaigns.mixing AS campaign_mixing + , campaigns.age_restrictions AS campaign_age_restrictions + , campaigns.group_members AS campaign_group_members + , campaigns.extended_age AS campaign_extended_age + , campaigns.enable_utm AS campaign_enable_utm + , campaigns.utm AS campaign_utm + + -- SITES + --, facts.sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- TRAFFIC + --, facts.traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword + , traffic.landing_page AS traffic_landing_page + + -- MEASURES + , sum(facts.impressions) AS impressions + , sum(facts.clicks) AS clicks + , sum(facts.cost) AS cost + +FROM {{ ref('stg_mt_banners_facts') }} AS facts + LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id + LEFT JOIN {{ ref('stg_mt_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id + LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id + LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id + +GROUP BY + + facts.dt + , cost_id + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , accounts.service AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id AS campaign_id + , campaigns.name AS campaign_name + , campaigns.system_status AS campaign_system_status + , campaigns.status AS campaign_status + , campaigns.autobidding AS campaign_autobidding + , campaigns.mixing AS campaign_mixing + , campaigns.age_restrictions AS campaign_age_restrictions + , campaigns.group_members AS campaign_group_members + , campaigns.extended_age AS campaign_extended_age + , campaigns.enable_utm AS campaign_enable_utm + , campaigns.utm AS campaign_utm + + -- SITES + --, facts.sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- TRAFFIC + --, facts.traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword , traffic.landing_page AS traffic_landing_page \ No newline at end of file diff --git a/models/intermediate/yd/int_yd_campaigns_facts_context.sql b/models/intermediate/yd/int_yd_campaigns_facts_context.sql index a485355..c350cb3 100644 --- a/models/intermediate/yd/int_yd_campaigns_facts_context.sql +++ b/models/intermediate/yd/int_yd_campaigns_facts_context.sql @@ -1,107 +1,107 @@ -SELECT - - facts.dt AS dt - , {{ dbt_utils.surrogate_key([ - 'facts.dt' - , 'facts.account_id' - , 'account_service' - , 'facts.campaigns_id' - , 'facts.sites_id' - , 'facts.traffic_id' - , 'facts.device' - ]) }} AS cost_id - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , 'Яндекс.Директ – Контекст' AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id AS campaign_id - , campaigns.name AS campaign_name - , campaigns.campaign_type AS campaign_type - , campaigns.status AS campaign_status - , campaigns.state AS campaign_state - , campaigns.status_payment AS campaign_status_payment - , campaigns.status_clarification AS campaign_status_clarification - , campaigns.currency AS campaign_currency - , campaigns.daily_budget_amount AS campaign_daily_budget_amount - , campaigns.daily_budget_mode AS campaign_daily_budget_mode - - -- SITES - --, facts.sites_id AS sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- TRAFFIC - --, facts.traffic_id AS traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword - , traffic.landing_page AS traffic_landing_page - - , facts.device AS device - - -- measures - , sum(facts.impressions_context) AS impressions - , sum(facts.clicks_context) AS clicks - , sum(facts.cost_context) AS cost - , sum(facts.bounces) AS bounces - -FROM {{ ref('stg_yd_ads_facts') }} AS facts - LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id - LEFT JOIN {{ ref('stg_yd_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id - LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id - LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id - -GROUP BY - - facts.dt - , cost_id - - -- ACCOUNT - , facts.account_id - , accounts.caption - , accounts.service - , accounts.enabled - , accounts.status - , accounts.interval_start - , accounts.interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id - , campaigns.name - , campaigns.campaign_type - , campaigns.status - , campaigns.state - , campaigns.status_payment - , campaigns.status_clarification - , campaigns.currency - , campaigns.daily_budget_amount - , campaigns.daily_budget_mode - - -- SITES - --, facts.sites_id - , sites.`domain` - , sites.description - - -- TRAFFIC - --, facts.traffic_id - , traffic.`grouping` - , traffic.`source` - , traffic.medium - , traffic.campaign - , traffic.content - , traffic.keyword - , traffic.landing_page - +SELECT + + facts.dt AS dt + , {{ dbt_utils.surrogate_key([ + 'facts.dt' + , 'facts.account_id' + , 'account_service' + , 'facts.campaigns_id' + , 'facts.sites_id' + , 'facts.traffic_id' + , 'facts.device' + ]) }} AS cost_id + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , 'Яндекс.Директ – Контекст' AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id AS campaign_id + , campaigns.name AS campaign_name + , campaigns.campaign_type AS campaign_type + , campaigns.status AS campaign_status + , campaigns.state AS campaign_state + , campaigns.status_payment AS campaign_status_payment + , campaigns.status_clarification AS campaign_status_clarification + , campaigns.currency AS campaign_currency + , campaigns.daily_budget_amount AS campaign_daily_budget_amount + , campaigns.daily_budget_mode AS campaign_daily_budget_mode + + -- SITES + --, facts.sites_id AS sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- TRAFFIC + --, facts.traffic_id AS traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword + , traffic.landing_page AS traffic_landing_page + + , facts.device AS device + + -- measures + , sum(facts.impressions_context) AS impressions + , sum(facts.clicks_context) AS clicks + , sum(facts.cost_context) AS cost + , sum(facts.bounces) AS bounces + +FROM {{ ref('stg_yd_ads_facts') }} AS facts + LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id + LEFT JOIN {{ ref('stg_yd_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id + LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id + LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id + +GROUP BY + + facts.dt + , cost_id + + -- ACCOUNT + , facts.account_id + , accounts.caption + , accounts.service + , accounts.enabled + , accounts.status + , accounts.interval_start + , accounts.interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id + , campaigns.name + , campaigns.campaign_type + , campaigns.status + , campaigns.state + , campaigns.status_payment + , campaigns.status_clarification + , campaigns.currency + , campaigns.daily_budget_amount + , campaigns.daily_budget_mode + + -- SITES + --, facts.sites_id + , sites.`domain` + , sites.description + + -- TRAFFIC + --, facts.traffic_id + , traffic.`grouping` + , traffic.`source` + , traffic.medium + , traffic.campaign + , traffic.content + , traffic.keyword + , traffic.landing_page + , facts.device \ No newline at end of file diff --git a/models/intermediate/yd/int_yd_campaigns_facts_search.sql b/models/intermediate/yd/int_yd_campaigns_facts_search.sql index 3a87dbe..40762eb 100644 --- a/models/intermediate/yd/int_yd_campaigns_facts_search.sql +++ b/models/intermediate/yd/int_yd_campaigns_facts_search.sql @@ -1,107 +1,107 @@ -SELECT - - facts.dt AS dt - , {{ dbt_utils.surrogate_key([ - 'facts.dt' - , 'facts.account_id' - , 'account_service' - , 'facts.campaigns_id' - , 'facts.sites_id' - , 'facts.traffic_id' - , 'facts.device' - ]) }} AS cost_id - - -- ACCOUNTS - , facts.account_id AS account_id - , accounts.caption AS account_caption - , 'Яндекс.Директ – Поиск' AS account_service - , accounts.enabled AS account_enabled - , accounts.status AS account_status - , accounts.interval_start AS account_interval_start - , accounts.interval_end AS account_interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id AS campaign_id - , campaigns.name AS campaign_name - , campaigns.campaign_type AS campaign_type - , campaigns.status AS campaign_status - , campaigns.state AS campaign_state - , campaigns.status_payment AS campaign_status_payment - , campaigns.status_clarification AS campaign_status_clarification - , campaigns.currency AS campaign_currency - , campaigns.daily_budget_amount AS campaign_daily_budget_amount - , campaigns.daily_budget_mode AS campaign_daily_budget_mode - - -- SITES - --, facts.sites_id AS sites_id - , sites.domain AS sites_domain - , sites.description AS sites_description - - -- TRAFFIC - --, facts.traffic_id AS traffic_id - , traffic.grouping AS traffic_grouping - , traffic.source AS traffic_source - , traffic.medium AS traffic_medium - , traffic.campaign AS traffic_campaign - , traffic.content AS traffic_content - , traffic.keyword AS traffic_keyword - , traffic.landing_page AS traffic_landing_page - - , facts.device AS device - - -- measures - , sum(facts.impressions_search) AS impressions - , sum(facts.clicks_search) AS clicks - , sum(facts.cost_search) AS cost - , sum(facts.bounces) AS bounces - -FROM {{ ref('stg_yd_ads_facts') }} AS facts - LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id - LEFT JOIN {{ ref('stg_yd_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id - LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id - LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id - -GROUP BY - - facts.dt - , cost_id - - -- ACCOUNT - , facts.account_id - , accounts.caption - , accounts.service - , accounts.enabled - , accounts.status - , accounts.interval_start - , accounts.interval_end - - -- CAMPAIGNS - --, facts.campaigns_id - , campaigns.campaign_id - , campaigns.name - , campaigns.campaign_type - , campaigns.status - , campaigns.state - , campaigns.status_payment - , campaigns.status_clarification - , campaigns.currency - , campaigns.daily_budget_amount - , campaigns.daily_budget_mode - - -- SITES - --, facts.sites_id - , sites.`domain` - , sites.description - - -- TRAFFIC - --, facts.traffic_id - , traffic.`grouping` - , traffic.`source` - , traffic.medium - , traffic.campaign - , traffic.content - , traffic.keyword - , traffic.landing_page - +SELECT + + facts.dt AS dt + , {{ dbt_utils.surrogate_key([ + 'facts.dt' + , 'facts.account_id' + , 'account_service' + , 'facts.campaigns_id' + , 'facts.sites_id' + , 'facts.traffic_id' + , 'facts.device' + ]) }} AS cost_id + + -- ACCOUNTS + , facts.account_id AS account_id + , accounts.caption AS account_caption + , 'Яндекс.Директ – Поиск' AS account_service + , accounts.enabled AS account_enabled + , accounts.status AS account_status + , accounts.interval_start AS account_interval_start + , accounts.interval_end AS account_interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id AS campaign_id + , campaigns.name AS campaign_name + , campaigns.campaign_type AS campaign_type + , campaigns.status AS campaign_status + , campaigns.state AS campaign_state + , campaigns.status_payment AS campaign_status_payment + , campaigns.status_clarification AS campaign_status_clarification + , campaigns.currency AS campaign_currency + , campaigns.daily_budget_amount AS campaign_daily_budget_amount + , campaigns.daily_budget_mode AS campaign_daily_budget_mode + + -- SITES + --, facts.sites_id AS sites_id + , sites.domain AS sites_domain + , sites.description AS sites_description + + -- TRAFFIC + --, facts.traffic_id AS traffic_id + , traffic.grouping AS traffic_grouping + , traffic.source AS traffic_source + , traffic.medium AS traffic_medium + , traffic.campaign AS traffic_campaign + , traffic.content AS traffic_content + , traffic.keyword AS traffic_keyword + , traffic.landing_page AS traffic_landing_page + + , facts.device AS device + + -- measures + , sum(facts.impressions_search) AS impressions + , sum(facts.clicks_search) AS clicks + , sum(facts.cost_search) AS cost + , sum(facts.bounces) AS bounces + +FROM {{ ref('stg_yd_ads_facts') }} AS facts + LEFT JOIN {{ ref('stg_general_accounts') }} AS accounts ON accounts.account_id = facts.account_id + LEFT JOIN {{ ref('stg_yd_campaigns') }} AS campaigns ON campaigns.id = facts.campaigns_id + LEFT JOIN {{ ref('stg_general_sites') }} AS sites ON sites.id = facts.sites_id + LEFT JOIN {{ ref('stg_general_traffic') }} AS traffic ON traffic.id = facts.traffic_id + +GROUP BY + + facts.dt + , cost_id + + -- ACCOUNT + , facts.account_id + , accounts.caption + , accounts.service + , accounts.enabled + , accounts.status + , accounts.interval_start + , accounts.interval_end + + -- CAMPAIGNS + --, facts.campaigns_id + , campaigns.campaign_id + , campaigns.name + , campaigns.campaign_type + , campaigns.status + , campaigns.state + , campaigns.status_payment + , campaigns.status_clarification + , campaigns.currency + , campaigns.daily_budget_amount + , campaigns.daily_budget_mode + + -- SITES + --, facts.sites_id + , sites.`domain` + , sites.description + + -- TRAFFIC + --, facts.traffic_id + , traffic.`grouping` + , traffic.`source` + , traffic.medium + , traffic.campaign + , traffic.content + , traffic.keyword + , traffic.landing_page + , facts.device \ No newline at end of file diff --git a/models/marts/f_costs.sql b/models/marts/f_costs.sql index 8bde599..f357b2e 100644 --- a/models/marts/f_costs.sql +++ b/models/marts/f_costs.sql @@ -1,120 +1,120 @@ -{{ - config ( - materialized='table', - order_by=["dt", "account_service"] - ) -}} - --- Yandex.Direct CONTEXT -SELECT - - dt - , cost_id - , device - - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - , campaign_id - , campaign_name - , campaign_state - - , sites_domain - , sites_description - - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - , impressions - , clicks - , cost - --, bounces - -FROM {{ ref('int_yd_campaigns_facts_context') }} - -UNION ALL - --- Yandex.Direct SEARCH -SELECT - - dt - , cost_id - , device - - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - , campaign_id - , campaign_name - , campaign_state - - , sites_domain - , sites_description - - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - , impressions - , clicks - , cost - --, bounces - -FROM {{ ref('int_yd_campaigns_facts_search') }} - -UNION ALL - --- myTarget -SELECT - - dt - , cost_id - , 'unknown' AS device - - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - , campaign_id::String - , campaign_name - , campaign_status - - , sites_domain - , sites_description - - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - , impressions - , clicks - , cost - -FROM {{ ref('int_mytarget_campaigns_facts') }} +{{ + config ( + materialized='table', + order_by=["dt", "account_service"] + ) +}} + +-- Yandex.Direct CONTEXT +SELECT + + dt + , cost_id + , device + + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + , campaign_id + , campaign_name + , campaign_state + + , sites_domain + , sites_description + + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + , impressions + , clicks + , cost + --, bounces + +FROM {{ ref('int_yd_campaigns_facts_context') }} + +UNION ALL + +-- Yandex.Direct SEARCH +SELECT + + dt + , cost_id + , device + + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + , campaign_id + , campaign_name + , campaign_state + + , sites_domain + , sites_description + + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + , impressions + , clicks + , cost + --, bounces + +FROM {{ ref('int_yd_campaigns_facts_search') }} + +UNION ALL + +-- myTarget +SELECT + + dt + , cost_id + , 'unknown' AS device + + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + , campaign_id::String + , campaign_name + , campaign_status + + , sites_domain + , sites_description + + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + , impressions + , clicks + , cost + +FROM {{ ref('int_mytarget_campaigns_facts') }} diff --git a/models/marts/f_ga_events.sql b/models/marts/f_ga_events.sql index c278927..fc5a8fd 100644 --- a/models/marts/f_ga_events.sql +++ b/models/marts/f_ga_events.sql @@ -1,132 +1,132 @@ -{{ - config ( - materialized='table', - order_by=["dt", "account_service"] - ) -}} - --- SESSIONS -SELECT - - dt - , event_id - , user_type - - -- ACCOUNTS - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - -- SITES - , sites_domain - , sites_description - - -- CLIENTS - , client_id - , client_phone - , client_email - - -- DEVICES - , UPPER(device_category) AS device_category - , device_browser - , device_browser_version - , device_os - , device_os_version - - -- TRAFFIC - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - -- LOCATIONS - , location_country_iso - , location_country - , location_region_iso - , location_region - , location_city - - -- GOALS - , NULL AS goal_id - , NULL AS goal_name - - -- MEASURES - , sessions - , bounces - , pageviews - , duration - , NULL AS goal_completions - , NULL AS goal_value - -FROM {{ ref('int_ga_sessions_facts') }} - -UNION ALL - --- GOAL COMPLETIONS -SELECT - - dt - , event_id - , NULL AS user_type - - -- ACCOUNTS - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - -- SITES - , sites_domain - , sites_description - - -- CLIENTS - , client_id - , client_phone - , client_email - - -- DEVICES - , UPPER(device_category) AS device_category - , device_browser - , device_browser_version - , device_os - , device_os_version - - -- TRAFFIC - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - -- LOCATIONS - , location_country_iso - , location_country - , location_region_iso - , location_region - , location_city - - -- GOALS - , goal_id - , goal_name - - -- MEASURES - , NULL AS sessions - , NULL AS bounces - , NULL AS pageviews - , NULL AS duration - , goal_completions - , goal_value - -FROM {{ ref('int_ga_goals_facts') }} +{{ + config ( + materialized='table', + order_by=["dt", "account_service"] + ) +}} + +-- SESSIONS +SELECT + + dt + , event_id + , user_type + + -- ACCOUNTS + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + -- SITES + , sites_domain + , sites_description + + -- CLIENTS + , client_id + , client_phone + , client_email + + -- DEVICES + , UPPER(device_category) AS device_category + , device_browser + , device_browser_version + , device_os + , device_os_version + + -- TRAFFIC + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + -- LOCATIONS + , location_country_iso + , location_country + , location_region_iso + , location_region + , location_city + + -- GOALS + , NULL AS goal_id + , NULL AS goal_name + + -- MEASURES + , sessions + , bounces + , pageviews + , duration + , NULL AS goal_completions + , NULL AS goal_value + +FROM {{ ref('int_ga_sessions_facts') }} + +UNION ALL + +-- GOAL COMPLETIONS +SELECT + + dt + , event_id + , NULL AS user_type + + -- ACCOUNTS + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + -- SITES + , sites_domain + , sites_description + + -- CLIENTS + , client_id + , client_phone + , client_email + + -- DEVICES + , UPPER(device_category) AS device_category + , device_browser + , device_browser_version + , device_os + , device_os_version + + -- TRAFFIC + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + -- LOCATIONS + , location_country_iso + , location_country + , location_region_iso + , location_region + , location_city + + -- GOALS + , goal_id + , goal_name + + -- MEASURES + , NULL AS sessions + , NULL AS bounces + , NULL AS pageviews + , NULL AS duration + , goal_completions + , goal_value + +FROM {{ ref('int_ga_goals_facts') }} diff --git a/models/marts/f_tracker.sql b/models/marts/f_tracker.sql index 0aa573b..a85ec13 100644 --- a/models/marts/f_tracker.sql +++ b/models/marts/f_tracker.sql @@ -1,147 +1,147 @@ -{{ - config ( - materialized='table', - order_by=["dt", "account_service"] - ) -}} - --- EVENTS -SELECT - - dt - , event_id - , user_type - - -- ACCOUNTS - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - -- SITES - , sites_domain - , sites_description - - -- TRAFFIC - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - -- DEVICES - , device_category - , device_browser - , device_browser_version - , device_os - , device_os_version - - -- CAMPAIGNS - , NULL AS campaign_id - , NULL AS campaign_name - , NULL AS campaign_state - - -- CLIENTS - , client_id - , client_phone - , client_email - - -- LOCATIONS - , location_country_iso - , location_country - , location_region_iso - , location_region - , location_city - - -- GOALS - , goal_id - , goal_name - - -- MEASURES - , NULL AS impressions - , NULL AS clicks - , NULL AS cost - , sessions - , bounces - , pageviews - , duration - , goal_completions - , goal_value - -FROM {{ ref('f_ga_events') }} - -UNION ALL - -SELECT - - dt - , cost_id - , NULL AS user_type - - -- ACCOUNTS - , account_id - , account_caption - , account_service - , account_enabled - , account_status - , account_interval_start - , account_interval_end - - -- SITES - , sites_domain - , sites_description - - -- TRAFFIC - , traffic_grouping - , traffic_source - , traffic_medium - , traffic_campaign - , traffic_content - , traffic_keyword - , traffic_landing_page - - -- DEVICES - , device AS device_category - , NULL AS device_browser - , NULL AS device_browser_version - , NULL AS device_os - , NULL AS device_os_version - - -- CAMPAIGNS - , campaign_id - , campaign_name - , campaign_state - - -- CLIENTS - , NULL AS client_id - , NULL AS client_phone - , NULL AS client_email - - -- LOCATIONS - , NULL AS location_country_iso - , NULL AS location_country - , NULL AS location_region_iso - , NULL AS location_region - , NULL AS location_city - - -- GOALS - , NULL AS goal_id - , NULL AS goal_name - - -- MEASURES - , impressions - , clicks - , cost - , NULL AS sessions - , NULL AS bounces - , NULL AS pageviews - , NULL AS duration - , NULL AS goal_completions - , NULL AS goal_value - +{{ + config ( + materialized='table', + order_by=["dt", "account_service"] + ) +}} + +-- EVENTS +SELECT + + dt + , event_id + , user_type + + -- ACCOUNTS + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + -- SITES + , sites_domain + , sites_description + + -- TRAFFIC + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + -- DEVICES + , device_category + , device_browser + , device_browser_version + , device_os + , device_os_version + + -- CAMPAIGNS + , NULL AS campaign_id + , NULL AS campaign_name + , NULL AS campaign_state + + -- CLIENTS + , client_id + , client_phone + , client_email + + -- LOCATIONS + , location_country_iso + , location_country + , location_region_iso + , location_region + , location_city + + -- GOALS + , goal_id + , goal_name + + -- MEASURES + , NULL AS impressions + , NULL AS clicks + , NULL AS cost + , sessions + , bounces + , pageviews + , duration + , goal_completions + , goal_value + +FROM {{ ref('f_ga_events') }} + +UNION ALL + +SELECT + + dt + , cost_id + , NULL AS user_type + + -- ACCOUNTS + , account_id + , account_caption + , account_service + , account_enabled + , account_status + , account_interval_start + , account_interval_end + + -- SITES + , sites_domain + , sites_description + + -- TRAFFIC + , traffic_grouping + , traffic_source + , traffic_medium + , traffic_campaign + , traffic_content + , traffic_keyword + , traffic_landing_page + + -- DEVICES + , device AS device_category + , NULL AS device_browser + , NULL AS device_browser_version + , NULL AS device_os + , NULL AS device_os_version + + -- CAMPAIGNS + , campaign_id + , campaign_name + , campaign_state + + -- CLIENTS + , NULL AS client_id + , NULL AS client_phone + , NULL AS client_email + + -- LOCATIONS + , NULL AS location_country_iso + , NULL AS location_country + , NULL AS location_region_iso + , NULL AS location_region + , NULL AS location_city + + -- GOALS + , NULL AS goal_id + , NULL AS goal_name + + -- MEASURES + , impressions + , clicks + , cost + , NULL AS sessions + , NULL AS bounces + , NULL AS pageviews + , NULL AS duration + , NULL AS goal_completions + , NULL AS goal_value + FROM {{ ref('f_costs') }} \ No newline at end of file diff --git a/models/marts/marts.yml b/models/marts/marts.yml index 9af6550..5154ded 100644 --- a/models/marts/marts.yml +++ b/models/marts/marts.yml @@ -1,43 +1,43 @@ -version: 2 - -models: - - name: f_costs - description: Summary of all costs (of all sources) - columns: - - name: cost_id - description: surrogate key - tests: - - not_null - - unique - - name: dt - description: Date - tests: - - not_null - - - name: f_ga_events - description: Summary of all Google Analytics Events - columns: - - name: event_id - description: surrogate key - tests: - - not_null - - unique: - severity: warn - - name: dt - description: Date - tests: - - not_null - - - name: f_tracker - description: Summary of Google Analytics Events and Costs - columns: - - name: event_id - description: surrogate key - tests: - - not_null - - unique: - severity: warn - - name: dt - description: Date - tests: - - not_null +version: 2 + +models: + - name: f_costs + description: Summary of all costs (of all sources) + columns: + - name: cost_id + description: surrogate key + tests: + - not_null + - unique + - name: dt + description: Date + tests: + - not_null + + - name: f_ga_events + description: Summary of all Google Analytics Events + columns: + - name: event_id + description: surrogate key + tests: + - not_null + - unique: + severity: warn + - name: dt + description: Date + tests: + - not_null + + - name: f_tracker + description: Summary of Google Analytics Events and Costs + columns: + - name: event_id + description: surrogate key + tests: + - not_null + - unique: + severity: warn + - name: dt + description: Date + tests: + - not_null diff --git a/models/metrics/dbt_metrics_default_calendar.sql b/models/metrics/dbt_metrics_default_calendar.sql index b3490f9..830b878 100644 --- a/models/metrics/dbt_metrics_default_calendar.sql +++ b/models/metrics/dbt_metrics_default_calendar.sql @@ -1,162 +1,162 @@ -{{ - config ( - materialized='table' - ) -}} - -with days as ( - - - -with rawdata as ( - - - with p as ( - select 0 as generated_number union all select 1 - ) - - , unioned as ( - - select - - - p0.generated_number * power(2, 0) - + - - p1.generated_number * power(2, 1) - + - - p2.generated_number * power(2, 2) - + - - p3.generated_number * power(2, 3) - + - - p4.generated_number * power(2, 4) - + - - p5.generated_number * power(2, 5) - + - - p6.generated_number * power(2, 6) - + - - p7.generated_number * power(2, 7) - + - - p8.generated_number * power(2, 8) - + - - p9.generated_number * power(2, 9) - + - - p10.generated_number * power(2, 10) - + - - p11.generated_number * power(2, 11) - + - - p12.generated_number * power(2, 12) - + - - p13.generated_number * power(2, 13) - - - + 1 - as generated_number - - from - - - p as p0 - cross join - - p as p1 - cross join - - p as p2 - cross join - - p as p3 - cross join - - p as p4 - cross join - - p as p5 - cross join - - p as p6 - cross join - - p as p7 - cross join - - p as p8 - cross join - - p as p9 - cross join - - p as p10 - cross join - - p as p11 - cross join - - p as p12 - cross join - - p as p13 - - - ) - - select * - from unioned - where generated_number <= 14610 - order by generated_number - - - -) - -, all_periods as ( - - select ( - - dateadd( - day, - generated_number, - cast('1990-01-01' as date) - ) ) as date_day - from rawdata - -) - -, - -filtered as ( - - select * - from all_periods - where date_day <= cast('2030-01-01' as date) - -) - -select * from filtered - - -), - -final as ( - select - cast(date_day as date) as date_day, - cast(date_trunc('week', date_day) as date) as date_week, - cast(date_trunc('month', date_day) as date) as date_month, - cast(date_trunc('quarter', date_day) as date) as date_quarter, - cast(date_trunc('year', date_day) as date) as date_year - from days -) - +{{ + config ( + materialized='table' + ) +}} + +with days as ( + + + +with rawdata as ( + + + with p as ( + select 0 as generated_number union all select 1 + ) + + , unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + p1.generated_number * power(2, 1) + + + + p2.generated_number * power(2, 2) + + + + p3.generated_number * power(2, 3) + + + + p4.generated_number * power(2, 4) + + + + p5.generated_number * power(2, 5) + + + + p6.generated_number * power(2, 6) + + + + p7.generated_number * power(2, 7) + + + + p8.generated_number * power(2, 8) + + + + p9.generated_number * power(2, 9) + + + + p10.generated_number * power(2, 10) + + + + p11.generated_number * power(2, 11) + + + + p12.generated_number * power(2, 12) + + + + p13.generated_number * power(2, 13) + + + + 1 + as generated_number + + from + + + p as p0 + cross join + + p as p1 + cross join + + p as p2 + cross join + + p as p3 + cross join + + p as p4 + cross join + + p as p5 + cross join + + p as p6 + cross join + + p as p7 + cross join + + p as p8 + cross join + + p as p9 + cross join + + p as p10 + cross join + + p as p11 + cross join + + p as p12 + cross join + + p as p13 + + + ) + + select * + from unioned + where generated_number <= 14610 + order by generated_number + + + +) + +, all_periods as ( + + select ( + + dateadd( + day, + generated_number, + cast('1990-01-01' as date) + ) ) as date_day + from rawdata + +) + +, + +filtered as ( + + select * + from all_periods + where date_day <= cast('2030-01-01' as date) + +) + +select * from filtered + + +), + +final as ( + select + cast(date_day as date) as date_day, + cast(date_trunc('week', date_day) as date) as date_week, + cast(date_trunc('month', date_day) as date) as date_month, + cast(date_trunc('quarter', date_day) as date) as date_quarter, + cast(date_trunc('year', date_day) as date) as date_year + from days +) + select * from final \ No newline at end of file diff --git a/models/metrics/f_metrics.sql b/models/metrics/f_metrics.sql index 7b3b6f7..adedb8b 100644 --- a/models/metrics/f_metrics.sql +++ b/models/metrics/f_metrics.sql @@ -1,16 +1,16 @@ -{{ - config ( - materialized='table' - ) -}} - - -select - * -from {{ metrics.calculate( - metric_list=[metric('costs')] - , grain='month' - , dimensions=['traffic_source'] - , date_alias='reporting_date' - ) - }} +{{ + config ( + materialized='table' + ) +}} + + +select + * +from {{ metrics.calculate( + metric_list=[metric('costs')] + , grain='month' + , dimensions=['traffic_source'] + , date_alias='reporting_date' + ) + }} diff --git a/models/metrics/metrics.yml b/models/metrics/metrics.yml index 3822599..1ddead3 100644 --- a/models/metrics/metrics.yml +++ b/models/metrics/metrics.yml @@ -1,41 +1,41 @@ -version: 2 - -metrics: - - - name: costs - label: 'Costs spent (RUB)' - model: ref('f_tracker') - description: '' - - calculation_method: sum - expression: cost - - timestamp: dt - time_grains: [day, week, month, quarter, year] - - dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] - - - name: clicks - label: 'Clicks' - model: ref('f_tracker') - description: '' - - calculation_method: sum - expression: clicks - - timestamp: dt - time_grains: [day, week, month, quarter, year] - - dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] - - - name: cpc - label: 'Cost per Click' - description: '' - - calculation_method: derived - expression: "{{ metric('costs') }} / {{ metric('clicks') }}" - - timestamp: dt - time_grains: [day, week, month, quarter, year] - - dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] +version: 2 + +metrics: + + - name: costs + label: 'Costs spent (RUB)' + model: ref('f_tracker') + description: '' + + calculation_method: sum + expression: cost + + timestamp: dt + time_grains: [day, week, month, quarter, year] + + dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] + + - name: clicks + label: 'Clicks' + model: ref('f_tracker') + description: '' + + calculation_method: sum + expression: clicks + + timestamp: dt + time_grains: [day, week, month, quarter, year] + + dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] + + - name: cpc + label: 'Cost per Click' + description: '' + + calculation_method: derived + expression: "{{ metric('costs') }} / {{ metric('clicks') }}" + + timestamp: dt + time_grains: [day, week, month, quarter, year] + + dimensions: [traffic_grouping, traffic_source, traffic_medium, traffic_campaign, device_category, location_country] diff --git a/packages.yml b/packages.yml index 5b00597..0ad0dc7 100644 --- a/packages.yml +++ b/packages.yml @@ -1,5 +1,5 @@ -packages: - - git: "https://github.com/kzzzr/mybi-dbt-core.git" - revision: 'v1.0.0' - - package: dbt-labs/metrics - version: [">=1.0.0", "<2.0.0"] +packages: + - git: "https://github.com/kzzzr/mybi-dbt-core.git" + revision: 'v1.0.0' + - package: dbt-labs/metrics + version: [">=1.0.0", "<2.0.0"] diff --git a/profiles.yml b/profiles.yml index 570faa9..1c7d512 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,18 +1,18 @@ -config: - send_anonymous_usage_stats: False - use_colors: True - partial_parse: True - -mybi_dbt_showcase: - target: prod - - outputs: - - prod: - type: clickhouse - host: clickhouse - port: 8123 - user: default - schema: analytics - threads: 3 - custom_settings: { enable_positional_arguments: 1} +config: + send_anonymous_usage_stats: False + use_colors: True + partial_parse: True + +mybi_dbt_showcase: + target: prod + + outputs: + + prod: + type: clickhouse + host: clickhouse + port: 8123 + user: default + schema: analytics + threads: 3 + custom_settings: { enable_positional_arguments: 1} diff --git a/schema/f_tracker.yml b/schema/f_tracker.yml index fda17d1..d4cd23b 100644 --- a/schema/f_tracker.yml +++ b/schema/f_tracker.yml @@ -1,36 +1,36 @@ -cubes: - - name: f_tracker - sql: SELECT * FROM analytics.f_tracker - measures: - - name: costs - sql: cost - type: sum - - name: clicks - sql: clicks - type: sum - - name: CPC - sql: '{CUBE.costs} / nullif(coalesce({CUBE.clicks}, 1), 0)' - type: number - dimensions: - - name: date - sql: 'dt' - type: time - - name: traffic_grouping - sql: traffic_grouping - type: string - - name: traffic_source - sql: traffic_source - type: string - - name: traffic_medium - sql: traffic_medium - type: string - - name: traffic_campaign - sql: traffic_campaign - type: string - - name: device_category - sql: device_category - type: string - - name: location_country - sql: location_country - type: string - dataSource: default +cubes: + - name: f_tracker + sql: SELECT * FROM analytics.f_tracker + measures: + - name: costs + sql: cost + type: sum + - name: clicks + sql: clicks + type: sum + - name: CPC + sql: '{CUBE.costs} / nullif(coalesce({CUBE.clicks}, 1), 0)' + type: number + dimensions: + - name: date + sql: 'dt' + type: time + - name: traffic_grouping + sql: traffic_grouping + type: string + - name: traffic_source + sql: traffic_source + type: string + - name: traffic_medium + sql: traffic_medium + type: string + - name: traffic_campaign + sql: traffic_campaign + type: string + - name: device_category + sql: device_category + type: string + - name: location_country + sql: location_country + type: string + dataSource: default