The-Academic-Observatory · jdddog · Mar 17, 2024 · Dec 13, 2023 · Dec 14, 2023 · Dec 14, 2023
diff --git a/.astro/config.yaml b/.astro/config.yaml
@@ -0,0 +1,2 @@
+project:
+  name: oaebu-workflows
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -0,0 +1,21 @@
+name: Astronomer CI - Deploy code
+
+on:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - main
+
+env:
+  ## Set API Token as an environment variable
+  ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }}
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Deploy to Astro
+      uses: astronomer/deploy-action@v0.4
+      with:
+        deployment-id: ${{ secrets.ASTRO_DEPLOYMENT_ID }}
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -29,12 +29,12 @@ jobs:
           cd ..
           git clone https://github.com/The-Academic-Observatory/observatory-platform.git
           cd observatory-platform
-          pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
-          pip install -e observatory-platform --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -e observatory-api --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -e observatory-platform --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
           cd ..
 
           cd oaebu-workflows
-          pip install -e .[tests] --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-no-providers-${{ matrix.python-version }}.txt
+          pip install -r requirements.txt --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-${{ matrix.python-version }}.txt
 
       # Required for testing ONIX Telescope
       - name: Set up JDK 11

diff --git a/.gitignore b/.gitignore
@@ -1,26 +1,21 @@
 .idea
 venv
-*.egg-info
 dist
 .vscode
 docs/_build
-config.yaml
 *.pkrvars.hcl
-*.tfvars
-*.tfstate
-*.tfstate.*
-.terraform.tfstate.lock.info
-.terraform/**
 .coverage
 coverage.xml
-config_terraform.yaml
 *.pyc
 AUTHORS
 ChangeLog
-.eggs/
 .DS_Store
-/oaebu_workflows/workflows/oapen_cloud_function.zip
 docs/schemas
 *.Rhistory
 .env
 __pycache__
+/oaebu-workflows/build
+
+# Astronomer
+/tests/
+airflow_settings.yaml
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,29 @@
+FROM quay.io/astronomer/astro-runtime:9.10.0
+
+# The following describes what various dependencies are used for. Some dependencies are specified in packages.txt
+# and some are installed in this Dockerfile if not available in apt by default.
+#
+# KubernetesPodOperator:
+#   * google-cloud-cli and google-cloud-cli-gke-gcloud-auth-plugin.
+#   * apt-transport-https, ca-certificates, gnupg, curl, sudo are installed as dependencies the above packages
+# OpenAlex: google-cloud-cli
+# Crossref Metadata: pigz
+# Open Citations: unzip
+# ORCID: s5cmd
+
+# Install custom dependencies for DAGs that are not available via apt by default
+USER root
+
+# To connect to GKE with KubernetesPodOperator install google-cloud-cli and google-cloud-cli-gke-gcloud-auth-plugin
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | tee /usr/share/keyrings/cloud.google.asc
+RUN apt-get update && apt-get install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
+
+# Install git
+RUN apt-get install git -y
+
+USER astro
+
+# Install Observatory Platform
+RUN git clone --branch feature/astro-refactor https://github.com/The-Academic-Observatory/observatory-platform.git
+RUN pip install -e ./observatory-platform/ --constraint https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-no-providers-3.10.txt
diff --git a/oaebu_workflows/__init__.py → dags/.airflowignore b/oaebu_workflows/__init__.py → dags/.airflowignore
diff --git a/oaebu_workflows/dags/__init__.py → dags/__init__.py b/oaebu_workflows/dags/__init__.py → dags/__init__.py
diff --git a/dags/load_dags.py b/dags/load_dags.py
@@ -0,0 +1,3 @@
+from observatory_platform.airflow.workflow import load_dags_from_config
+
+load_dags_from_config()
diff --git a/...s/google_analytics3_telescope/__init__.py → dags/oaebu_workflows/__init__.py b/...s/google_analytics3_telescope/__init__.py → dags/oaebu_workflows/__init__.py
diff --git a/oaebu_workflows/airflow_pools.py → dags/oaebu_workflows/airflow_pools.py b/oaebu_workflows/airflow_pools.py → dags/oaebu_workflows/airflow_pools.py
diff --git a/oaebu_workflows/config.py → dags/oaebu_workflows/config.py b/oaebu_workflows/config.py → dags/oaebu_workflows/config.py
@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Curtin University
+# Copyright 2020-2024 Curtin University
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 import os
 from typing import Optional
 
-from observatory.platform.config import module_file_path
+from observatory_platform.config import module_file_path
 
 
 def test_fixtures_folder(workflow_module: Optional[str] = None) -> str:
@@ -71,3 +71,9 @@ def construct_module_path(*parts: str) -> str:
         raise FileNotFoundError(f"construct_module_path: directory {file_path} does not exist!")
 
     return file_path
+
+
+def oaebu_user_agent_header() -> dict:
+    return {
+        "User-Agent": "oaebu-workflows v1.0.0 (+https://github.com/The-Academic-Observatory/oaebu-workflows; mailto:agent@observatory.academy) "
+    }
diff --git a/...e_analytics3_telescope/schema/__init__.py → ...s/google_analytics3_telescope/__init__.py b/...e_analytics3_telescope/schema/__init__.py → ...s/google_analytics3_telescope/__init__.py
diff --git a/...ogle_analytics3_telescope/sql/__init__.py → ...e_analytics3_telescope/schema/__init__.py b/...ogle_analytics3_telescope/sql/__init__.py → ...e_analytics3_telescope/schema/__init__.py
diff --git a/...ook_metrics_author_google_analytics3.json → ...ook_metrics_author_google_analytics3.json b/...ook_metrics_author_google_analytics3.json → ...ook_metrics_author_google_analytics3.json
diff --git a/...ok_metrics_country_google_analytics3.json → ...ok_metrics_country_google_analytics3.json b/...ok_metrics_country_google_analytics3.json → ...ok_metrics_country_google_analytics3.json
diff --git a/...chema/book_metrics_google_analytics3.json → ...chema/book_metrics_google_analytics3.json b/...chema/book_metrics_google_analytics3.json → ...chema/book_metrics_google_analytics3.json
diff --git a/...ok_metrics_subject_google_analytics3.json → ...ok_metrics_subject_google_analytics3.json b/...ok_metrics_subject_google_analytics3.json → ...ok_metrics_subject_google_analytics3.json
diff --git a/...ok_product_metrics_google_analytics3.json → ...ok_product_metrics_google_analytics3.json b/...ok_product_metrics_google_analytics3.json → ...ok_product_metrics_google_analytics3.json
diff --git a/...s3_telescope/schema/google_analytics.json → ...s3_telescope/schema/google_analytics.json b/...s3_telescope/schema/google_analytics.json → ...s3_telescope/schema/google_analytics.json
diff --git a/...le_analytics3_telescope/tests/__init__.py → ...ogle_analytics3_telescope/sql/__init__.py b/...le_analytics3_telescope/tests/__init__.py → ...ogle_analytics3_telescope/sql/__init__.py
diff --git a/...country_body_google_analytics3.sql.jinja2 → ...country_body_google_analytics3.sql.jinja2 b/...country_body_google_analytics3.sql.jinja2 → ...country_body_google_analytics3.sql.jinja2
diff --git a/...etrics_country_join_google_analytics3.sql → ...etrics_country_join_google_analytics3.sql b/...etrics_country_join_google_analytics3.sql → ...etrics_country_join_google_analytics3.sql
diff --git a/...etrics_country_null_google_analytics3.sql → ...etrics_country_null_google_analytics3.sql b/...etrics_country_null_google_analytics3.sql → ...etrics_country_null_google_analytics3.sql
diff --git a/...rics_country_struct_google_analytics3.sql → ...rics_country_struct_google_analytics3.sql b/...rics_country_struct_google_analytics3.sql → ...rics_country_struct_google_analytics3.sql
diff --git a/...pe/sql/book_metrics_google_analytics3.sql → ...pe/sql/book_metrics_google_analytics3.sql b/...pe/sql/book_metrics_google_analytics3.sql → ...pe/sql/book_metrics_google_analytics3.sql
diff --git a/...product_body_google_analytics3.sql.jinja2 → ...product_body_google_analytics3.sql.jinja2 b/...product_body_google_analytics3.sql.jinja2 → ...product_body_google_analytics3.sql.jinja2
diff --git a/...k_product_functions_google_analytics3.sql → ...k_product_functions_google_analytics3.sql b/...k_product_functions_google_analytics3.sql → ...k_product_functions_google_analytics3.sql
diff --git a/...l/month_metrics_sum_google_analytics3.sql → ...l/month_metrics_sum_google_analytics3.sql b/...l/month_metrics_sum_google_analytics3.sql → ...l/month_metrics_sum_google_analytics3.sql
diff --git a/...cope/sql/month_null_google_analytics3.sql → ...cope/sql/month_null_google_analytics3.sql b/...cope/sql/month_null_google_analytics3.sql → ...cope/sql/month_null_google_analytics3.sql
diff --git a/...ics3_telescope/tests/fixtures/__init__.py → ...kflows/google_books_telescope/__init__.py b/...ics3_telescope/tests/fixtures/__init__.py → ...kflows/google_books_telescope/__init__.py