GitHub workflow on pr (Velir#93)

* started testing workflow * working local version of unit tests * fixing working directory * adding manual execution * fixing keyfile location
graze · Nov 23, 2022 · f43277b · f43277b
1 parent 6efa9d5
commit f43277b
Show file tree

Hide file tree

Showing 18 changed files with 71 additions and 22 deletions.
diff --git a/.github/workflows/run_unit_tests_on_pr.yml b/.github/workflows/run_unit_tests_on_pr.yml
@@ -0,0 +1,38 @@
+name: Run Unit Tests on Pull Request
+
+on: [pull_request,workflow_dispatch]
+env:
+  BIGQUERY_PROJECT: ${{ secrets.BIGQUERY_PROJECT }}
+  KEYFILE_LOCATION: /dbt-service-account.json
+
+jobs:
+  pytest_run_all:
+    name: Pytest Run All
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./unit_tests
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: Authenticate using service account
+        run: 'echo "$KEYFILE" > ./dbt-service-account.json'
+        shell: bash
+        env:
+          KEYFILE: ${{secrets.GCP_BIGQUERY_USER_KEYFILE}}
+
+      - name: Install dependencies
+        run: |
+          pip install dbt-core
+          pip install dbt-bigquery
+          pip install pytest
+      
+      - name: Run tests
+        run: python -m pytest .
diff --git a/.gitignore b/.gitignore
@@ -2,11 +2,12 @@
 target/
 dbt_packages/
 logs/
-google-cloud-sdk/
-integration_tests/.env
-integration_tests/__pycache__
-integration_tests/.pytest_cache
 
+google-cloud-sdk/
+unit_tests/.env
+unit_tests/__pycache__
+unit_tests/.pytest_cache
+unit_tests/dbt-service-account.json
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# GA4 DBT Package
+# GA4 DBT Package 
 
 This package connects to an exported GA4 dataset and provides useful transformations as well as report-ready dimensional models that can be used to build reports or blend GA4 data with exported GA3 data.
 

diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
diff --git a/unit_tests/.env.example b/unit_tests/.env.example
@@ -0,0 +1 @@
+BIGQUERY_PROJECT=
diff --git a/integration_tests/README.md → unit_tests/README.md b/integration_tests/README.md → unit_tests/README.md
diff --git a/unit_tests/conftest.py b/unit_tests/conftest.py
@@ -0,0 +1,25 @@
+import pytest
+import os
+
+# Import the standard functional fixtures as a plugin
+pytest_plugins = ["dbt.tests.fixtures.project"]
+
+# The profile dictionary, used to write out profiles.yml
+@pytest.fixture(scope="class")
+def dbt_profile_target():
+    # Set project and keyfile for github automated tests
+    if os.environ.get('GITHUB_ACTIONS') is not None:
+        return {
+            'type': 'bigquery',
+            'method': 'service-account',
+            'keyfile': os.getcwd() + os.environ.get("KEYFILE_LOCATION"),
+            'threads': 4,
+            'timeout_seconds': 300,
+            'project':  os.environ.get("BIGQUERY_PROJECT")
+        }
+    return {
+        'type': 'bigquery',
+        'method': 'oauth',
+        'threads': 4,
+        'project':  os.environ.get("BIGQUERY_PROJECT")
+    }
diff --git a/integration_tests/requirements.txt → unit_tests/requirements.txt b/integration_tests/requirements.txt → unit_tests/requirements.txt
diff --git a/...ts/test_macro_default_channel_grouping.py → ...ts/test_macro_default_channel_grouping.py b/...ts/test_macro_default_channel_grouping.py → ...ts/test_macro_default_channel_grouping.py
diff --git a/...ts/test_macro_exclude_query_parameters.py → ...ts/test_macro_exclude_query_parameters.py b/...ts/test_macro_exclude_query_parameters.py → ...ts/test_macro_exclude_query_parameters.py
diff --git a/...st_stg_ga4__derived_session_properties.py → ...st_stg_ga4__derived_session_properties.py b/...st_stg_ga4__derived_session_properties.py → ...st_stg_ga4__derived_session_properties.py
diff --git a/.../test_stg_ga4__derived_user_properties.py → .../test_stg_ga4__derived_user_properties.py b/.../test_stg_ga4__derived_user_properties.py → .../test_stg_ga4__derived_user_properties.py
diff --git a/..._stg_ga4__event_to_query_string_params.py → ..._stg_ga4__event_to_query_string_params.py b/..._stg_ga4__event_to_query_string_params.py → ..._stg_ga4__event_to_query_string_params.py
diff --git a/...ration_tests/test_stg_ga4__events.example → unit_tests/test_stg_ga4__events.example b/...ration_tests/test_stg_ga4__events.example → unit_tests/test_stg_ga4__events.example
diff --git a/integration_tests/test_stg_ga4__events.todo → unit_tests/test_stg_ga4__events.todo b/integration_tests/test_stg_ga4__events.todo → unit_tests/test_stg_ga4__events.todo
diff --git a/...n_tests/test_stg_ga4__page_conversions.py → unit_tests/test_stg_ga4__page_conversions.py b/...n_tests/test_stg_ga4__page_conversions.py → unit_tests/test_stg_ga4__page_conversions.py
diff --git a/...est_stg_ga4__session_conversions_daily.py → ...est_stg_ga4__session_conversions_daily.py b/...est_stg_ga4__session_conversions_daily.py → ...est_stg_ga4__session_conversions_daily.py
@@ -38,5 +38,5 @@ def models(self):
 
     def test_mock_run_and_check(self, project):
         run_dbt(["build", "--vars", "conversion_events: ['my_conversion']"])
-        breakpoint()
+        #breakpoint()
         check_relations_equal(project.adapter, ["actual", "expected"])
diff --git a/.../test_stg_ga4__users_first_last_events.py → .../test_stg_ga4__users_first_last_events.py b/.../test_stg_ga4__users_first_last_events.py → .../test_stg_ga4__users_first_last_events.py