Astro Migration (#212)

The-Academic-Observatory · Mar 17, 2024 · ad505cf · ad505cf
1 parent 3ddef54
commit ad505cf
Show file tree

Hide file tree

Showing 360 changed files with 6,392 additions and 6,689 deletions.
diff --git a/.astro/config.yaml b/.astro/config.yaml
@@ -0,0 +1,2 @@
+project:
+  name: oaebu-workflows
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -0,0 +1,19 @@
+name: Astronomer CI - Deploy code
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+env:
+  ## Set API Token as an environment variable
+  ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }}
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Deploy to Astro
+      uses: astronomer/deploy-action@v0.4
+      with:
+        deployment-id: ${{ secrets.ASTRO_DEPLOYMENT_ID }}
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -1,6 +1,6 @@
 name: Unit Tests
 
-on: [push, merge_group]
+on: [push]
 
 jobs:
   build:
@@ -13,12 +13,12 @@ jobs:
 
     steps:
       - name: Checkout ${{ matrix.python-version }}
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           lfs: true
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -27,14 +27,17 @@ jobs:
           python -m pip install --upgrade pip
 
           cd ..
-          git clone https://github.com/The-Academic-Observatory/observatory-platform.git
+          git clone -b feature/astro-refactor https://github.com/The-Academic-Observatory/observatory-platform.git
           cd observatory-platform
-          pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
-          pip install -e observatory-platform --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -e .[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
           cd ..
 
           cd oaebu-workflows
-          pip install -e .[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -r requirements.txt --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
+
+      - name: Add dag folder to PYTHONPATH
+        run: |
+          echo "PYTHONPATH=$HOME/work/oaebu-workflows/oaebu-workflows/dags" >> "$GITHUB_ENV"
 
       # Required for testing ONIX Telescope
       - name: Set up JDK 11

diff --git a/.gitignore b/.gitignore
@@ -1,26 +1,20 @@
 .idea
 venv
-*.egg-info
 dist
 .vscode
 docs/_build
-config.yaml
 *.pkrvars.hcl
-*.tfvars
-*.tfstate
-*.tfstate.*
-.terraform.tfstate.lock.info
-.terraform/**
 .coverage
 coverage.xml
-config_terraform.yaml
 *.pyc
 AUTHORS
 ChangeLog
-.eggs/
 .DS_Store
-/oaebu_workflows/workflows/oapen_cloud_function.zip
 docs/schemas
 *.Rhistory
 .env
 __pycache__
+/oaebu-workflows/build
+
+# Astronomer
+airflow_settings.yaml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,13 @@
+FROM quay.io/astronomer/astro-runtime:9.10.0
+
+# Root user for installations
+USER root 
+
+# Install git
+RUN apt-get update && apt-get install git -y
+
+USER astro
+
+# Install Observatory Platform
+RUN git clone --branch feature/astro-refactor https://github.com/The-Academic-Observatory/observatory-platform.git
+RUN pip install -e ./observatory-platform/ --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-3.10.txt
diff --git a/oaebu_workflows/__init__.py → dags/.airflowignore b/oaebu_workflows/__init__.py → dags/.airflowignore
diff --git a/oaebu_workflows/dags/__init__.py → dags/__init__.py b/oaebu_workflows/dags/__init__.py → dags/__init__.py
diff --git a/dags/load_dags.py b/dags/load_dags.py
@@ -0,0 +1,3 @@
+from observatory_platform.airflow.workflow import load_dags_from_config
+
+load_dags_from_config()
diff --git a/...s/google_analytics3_telescope/__init__.py → dags/oaebu_workflows/__init__.py b/...s/google_analytics3_telescope/__init__.py → dags/oaebu_workflows/__init__.py
diff --git a/oaebu_workflows/airflow_pools.py → dags/oaebu_workflows/airflow_pools.py b/oaebu_workflows/airflow_pools.py → dags/oaebu_workflows/airflow_pools.py
diff --git a/oaebu_workflows/config.py → dags/oaebu_workflows/config.py b/oaebu_workflows/config.py → dags/oaebu_workflows/config.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Curtin University
+# Copyright 2020-2024 Curtin University
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 import os
 from typing import Optional
 
-from observatory.platform.config import module_file_path
+from observatory_platform.config import module_file_path
 
 
 def test_fixtures_folder(workflow_module: Optional[str] = None) -> str:
@@ -29,9 +29,9 @@ def test_fixtures_folder(workflow_module: Optional[str] = None) -> str:
     """
 
     if workflow_module is not None:
-        return construct_module_path("oaebu_workflows", workflow_module, "tests", "fixtures")
+        return construct_module_path("tests", workflow_module, "fixtures")
 
-    return construct_module_path("oaebu_workflows", "tests", "fixtures")
+    return construct_module_path("tests", "fixtures")
 
 
 def schema_folder(workflow_module: Optional[str] = None) -> str:
@@ -43,9 +43,9 @@ def schema_folder(workflow_module: Optional[str] = None) -> str:
     """
 
     if workflow_module is not None:
-        return construct_module_path("oaebu_workflows", workflow_module, "schema")
+        return construct_module_path("dags", "oaebu_workflows", workflow_module, "schema")
 
-    return construct_module_path("oaebu_workflows", "schema")
+    return construct_module_path("dags", "oaebu_workflows", "schema")
 
 
 def sql_folder(workflow_module: Optional[str] = None) -> str:
@@ -57,9 +57,9 @@ def sql_folder(workflow_module: Optional[str] = None) -> str:
     """
 
     if workflow_module is not None:
-        return construct_module_path("oaebu_workflows", workflow_module, "sql")
+        return construct_module_path("dags", "oaebu_workflows", workflow_module, "sql")
 
-    return construct_module_path("oaebu_workflows", "sql")
+    return construct_module_path("dags", "oaebu_workflows", "sql")
 
 
 def construct_module_path(*parts: str) -> str:
@@ -71,3 +71,9 @@ def construct_module_path(*parts: str) -> str:
         raise FileNotFoundError(f"construct_module_path: directory {file_path} does not exist!")
 
     return file_path
+
+
+def oaebu_user_agent_header() -> dict:
+    return {
+        "User-Agent": "oaebu-workflows v1.0.0 (+https://github.com/The-Academic-Observatory/oaebu-workflows; mailto:agent@observatory.academy) "
+    }
diff --git a/...e_analytics3_telescope/schema/__init__.py → ...s/google_analytics3_telescope/__init__.py b/...e_analytics3_telescope/schema/__init__.py → ...s/google_analytics3_telescope/__init__.py
diff --git a/...ogle_analytics3_telescope/sql/__init__.py → ...e_analytics3_telescope/schema/__init__.py b/...ogle_analytics3_telescope/sql/__init__.py → ...e_analytics3_telescope/schema/__init__.py
diff --git a/...ook_metrics_author_google_analytics3.json → ...ook_metrics_author_google_analytics3.json b/...ook_metrics_author_google_analytics3.json → ...ook_metrics_author_google_analytics3.json
diff --git a/...ok_metrics_country_google_analytics3.json → ...ok_metrics_country_google_analytics3.json b/...ok_metrics_country_google_analytics3.json → ...ok_metrics_country_google_analytics3.json
diff --git a/...chema/book_metrics_google_analytics3.json → ...chema/book_metrics_google_analytics3.json b/...chema/book_metrics_google_analytics3.json → ...chema/book_metrics_google_analytics3.json
diff --git a/...ok_metrics_subject_google_analytics3.json → ...ok_metrics_subject_google_analytics3.json b/...ok_metrics_subject_google_analytics3.json → ...ok_metrics_subject_google_analytics3.json
diff --git a/...ok_product_metrics_google_analytics3.json → ...ok_product_metrics_google_analytics3.json b/...ok_product_metrics_google_analytics3.json → ...ok_product_metrics_google_analytics3.json
diff --git a/...s3_telescope/schema/google_analytics.json → ...s3_telescope/schema/google_analytics.json b/...s3_telescope/schema/google_analytics.json → ...s3_telescope/schema/google_analytics.json
diff --git a/...le_analytics3_telescope/tests/__init__.py → ...ogle_analytics3_telescope/sql/__init__.py b/...le_analytics3_telescope/tests/__init__.py → ...ogle_analytics3_telescope/sql/__init__.py
diff --git a/...country_body_google_analytics3.sql.jinja2 → ...country_body_google_analytics3.sql.jinja2 b/...country_body_google_analytics3.sql.jinja2 → ...country_body_google_analytics3.sql.jinja2
diff --git a/...etrics_country_join_google_analytics3.sql → ...etrics_country_join_google_analytics3.sql b/...etrics_country_join_google_analytics3.sql → ...etrics_country_join_google_analytics3.sql
diff --git a/...etrics_country_null_google_analytics3.sql → ...etrics_country_null_google_analytics3.sql b/...etrics_country_null_google_analytics3.sql → ...etrics_country_null_google_analytics3.sql
diff --git a/...rics_country_struct_google_analytics3.sql → ...rics_country_struct_google_analytics3.sql b/...rics_country_struct_google_analytics3.sql → ...rics_country_struct_google_analytics3.sql
diff --git a/...pe/sql/book_metrics_google_analytics3.sql → ...pe/sql/book_metrics_google_analytics3.sql b/...pe/sql/book_metrics_google_analytics3.sql → ...pe/sql/book_metrics_google_analytics3.sql
diff --git a/...product_body_google_analytics3.sql.jinja2 → ...product_body_google_analytics3.sql.jinja2 b/...product_body_google_analytics3.sql.jinja2 → ...product_body_google_analytics3.sql.jinja2
diff --git a/...k_product_functions_google_analytics3.sql → ...k_product_functions_google_analytics3.sql b/...k_product_functions_google_analytics3.sql → ...k_product_functions_google_analytics3.sql
diff --git a/...l/month_metrics_sum_google_analytics3.sql → ...l/month_metrics_sum_google_analytics3.sql b/...l/month_metrics_sum_google_analytics3.sql → ...l/month_metrics_sum_google_analytics3.sql
diff --git a/...cope/sql/month_null_google_analytics3.sql → ...cope/sql/month_null_google_analytics3.sql b/...cope/sql/month_null_google_analytics3.sql → ...cope/sql/month_null_google_analytics3.sql
diff --git a/...ics3_telescope/tests/fixtures/__init__.py → ...kflows/google_books_telescope/__init__.py b/...ics3_telescope/tests/fixtures/__init__.py → ...kflows/google_books_telescope/__init__.py