From 7e23ec693b5be106a0bd977040dd4d7b74f0ec39 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:10:49 +0200 Subject: [PATCH 01/26] Rename weatherapi to google-sheets --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 4 ++-- .github/workflows/docker_cleanup.yml | 2 +- .github/workflows/test.yaml | 6 ++--- Dockerfile | 2 +- SECURITY.md | 6 ++--- docker-compose.yaml | 8 +++---- pyproject.toml | 28 +++++++++++------------ scripts/deploy.sh | 2 +- scripts/lint-pre-commit.sh | 2 +- scripts/lint.sh | 4 ++-- scripts/static-analysis.sh | 2 +- scripts/static-pre-commit.sh | 2 +- tests/app/test_app.py | 6 ++--- weatherapi/app.py | 2 +- 16 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index f0a6bf4..da9aef1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -13,7 +13,7 @@ Provide a clear and concise description of the bug. Include source code: ```python -from weatherapi.app import app +from google_sheets.app import app ... ``` diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d2284c6..7b726a4 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,4 +5,4 @@ contact_links: about: Please report security vulnerabilities to info@airt.ai - name: Question or Problem about: Ask a question or ask about a problem in GitHub Discussions. - url: https://github.com/airtai/weatherapi/discussions/categories/questions + url: https://github.com/airtai/google-sheets/discussions/categories/questions diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 4be39ef..09b3c8f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -15,10 +15,10 @@ Provide a clear and concise description of the problem you've encountered. For e Clearly and concisely describe the desired outcome or solution. **Feature code example** -To help others understand the proposed feature, illustrate it with a **weatherapi** code example: +To help others understand the proposed feature, illustrate it with a **google-sheets** code example: ```python -from weatherapi.app import app +from google_sheets.app import app ... ``` diff --git a/.github/workflows/docker_cleanup.yml b/.github/workflows/docker_cleanup.yml index 29224b5..68822c3 100644 --- a/.github/workflows/docker_cleanup.yml +++ b/.github/workflows/docker_cleanup.yml @@ -16,7 +16,7 @@ jobs: with: # NOTE: at now only orgs is supported owner: airtai - name: weatherapi + name: google-sheets token: ${{ secrets.GITHUB_TOKEN }} # Keep latest N untagged images diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3d8f8d1..1b18de6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,11 +29,11 @@ jobs: pip install -e ".[docs,lint]" - name: Run mypy shell: bash - run: mypy weatherapi tests + run: mypy google-sheets tests - name: Run bandit shell: bash - run: bandit -c pyproject.toml -r weatherapi + run: bandit -c pyproject.toml -r google-sheets - name: Run Semgrep shell: bash @@ -146,7 +146,7 @@ jobs: - run: ls -la coverage - run: coverage combine coverage - run: coverage report - - run: coverage html --show-contexts --title "weatherapi coverage for ${{ github.sha }}" + - run: coverage html --show-contexts --title "google-sheets coverage for ${{ github.sha }}" - name: Store coverage html uses: actions/upload-artifact@v4 diff --git a/Dockerfile b/Dockerfile index 84fc967..0e4f491 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 RUN python3 -m pip install --upgrade pip -COPY weatherapi ./weatherapi +COPY google-sheets ./google-sheets COPY scripts/* pyproject.toml README.md ./ RUN pip install -e ".[dev]" diff --git a/SECURITY.md b/SECURITY.md index 11cfd95..dbabd5e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,14 +1,14 @@ # Security Policy -Security and stability are paramount for WeatherAPI. +Security and stability are paramount for google-sheets. Learn more below. 👇 ## Versions -The latest version of WeatherAPI is actively supported. +The latest version of google-sheets is actively supported. -We strongly encourage you to write tests for your application and regularly update your WeatherAPI version after confirming that your tests pass. This ensures you benefit from the latest features, bug fixes, and **security updates**. +We strongly encourage you to write tests for your application and regularly update your google-sheets version after confirming that your tests pass. This ensures you benefit from the latest features, bug fixes, and **security updates**. ## Reporting a Vulnerability diff --git a/docker-compose.yaml b/docker-compose.yaml index 5ba32a4..2d5f2cd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,7 +11,7 @@ x-logging: &default-logging version: "3.4" -name: weatherapi +name: google-sheets services: fastapi-app: # nosemgrep image: ghcr.io/${GITHUB_REPOSITORY}:${TAG} @@ -22,8 +22,8 @@ services: - DOMAIN=${DOMAIN} logging: *default-logging networks: - - weatherapi + - google-sheets networks: - weatherapi: - name: weatherapi + google-sheets: + name: google-sheets diff --git a/pyproject.toml b/pyproject.toml index 6b9411c..fb4212d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "weatherapi" -description = "WeatherAPI: A simple weather API made for usage with FastAgency" +name = "google-sheets" +description = "google-sheets: A simple weather API made for usage with FastAgency" readme = "README.md" authors = [ { name = "airt", email = "info@airt.ai" }, @@ -78,26 +78,26 @@ test-core = [ ] testing = [ - "weatherapi[test-core]", - "weatherapi[server]", # Uvicorn is needed for testing + "google-sheets[test-core]", + "google-sheets[server]", # Uvicorn is needed for testing ] dev = [ - "weatherapi[server,lint,testing]", + "google-sheets[server,lint,testing]", "pre-commit==3.7.1", "detect-secrets==1.5.0", ] [project.urls] -Tracker = "https://github.com/airtai/weatherapi/issues" -Source = "https://github.com/airtai/weatherapi" +Tracker = "https://github.com/airtai/google-sheets/issues" +Source = "https://github.com/airtai/google-sheets" Discord = "https://discord.gg/qFm6aSqq59" [project.scripts] -# weatherapi = "weatherapi.__main__:cli" +# google-sheets = "google-sheets.__main__:cli" [tool.hatch.version] -path = "weatherapi/__about__.py" +path = "google-sheets/__about__.py" [tool.hatch.build] skip-excluded-dirs = true @@ -110,15 +110,15 @@ exclude = [ allow-direct-references = true [tool.hatch.build.targets.wheel] -only-include = ["weatherapi"] +only-include = ["google-sheets"] [tool.hatch.build.targets.wheel.sources] "src" = "" -# "scripts" = "weatherapi/templates" +# "scripts" = "google-sheets/templates" [tool.mypy] -files = ["weatherapi", "tests"] +files = ["google-sheets", "tests"] strict = true python_version = "3.9" @@ -145,7 +145,7 @@ disallow_any_unimported = false fix = true line-length = 88 # target-version = 'py39' -include = ["weatherapi/**/*.py", "weatherapi/**/*.pyi", "tests/**/*.py", "pyproject.toml"] +include = ["google-sheets/**/*.py", "google-sheets/**/*.pyi", "tests/**/*.py", "pyproject.toml"] exclude = ["docs/docs_src"] [tool.ruff.lint] @@ -206,7 +206,7 @@ concurrency = [ "thread" ] source = [ - "weatherapi", + "google-sheets", # "tests", ] context = '${CONTEXT}' diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 8fd8bc7..cd45fe8 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -22,7 +22,7 @@ fi ssh_command="ssh -o StrictHostKeyChecking=no -i key.pem azureuser@$DOMAIN" -container_name="weatherapi" +container_name="google-sheets" log_file="${container_name}.log" echo "INFO: Capturing docker container logs" diff --git a/scripts/lint-pre-commit.sh b/scripts/lint-pre-commit.sh index e8d2f45..70492b6 100755 --- a/scripts/lint-pre-commit.sh +++ b/scripts/lint-pre-commit.sh @@ -28,5 +28,5 @@ pip install --editable ".[dev]" \ # and specify the package to run on explicitly. # Note that we do not use --ignore-missing-imports, # as this can give us false confidence in our results. -# mypy weatherapi +# mypy google-sheets ./scripts/lint.sh diff --git a/scripts/lint.sh b/scripts/lint.sh index 114d2e1..f01198d 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash echo "Running pyup_dirs..." -pyup_dirs --py38-plus --recursive weatherapi tests +pyup_dirs --py38-plus --recursive google-sheets tests echo "Running ruff linter (isort, flake, pyupgrade, etc. replacement)..." ruff check @@ -10,4 +10,4 @@ echo "Running ruff formater (black replacement)..." ruff format # echo "Running black..." -# black weatherapi examples tests docs +# black google-sheets examples tests docs diff --git a/scripts/static-analysis.sh b/scripts/static-analysis.sh index 8fd829b..156fba1 100755 --- a/scripts/static-analysis.sh +++ b/scripts/static-analysis.sh @@ -6,7 +6,7 @@ echo "Running mypy..." mypy echo "Running bandit..." -bandit -c pyproject.toml -r weatherapi +bandit -c pyproject.toml -r google-sheets echo "Running semgrep..." semgrep scan --config auto --error diff --git a/scripts/static-pre-commit.sh b/scripts/static-pre-commit.sh index 06ae1e7..7f2a29b 100755 --- a/scripts/static-pre-commit.sh +++ b/scripts/static-pre-commit.sh @@ -28,5 +28,5 @@ pip install --editable ".[dev]" \ # and specify the package to run on explicitly. # Note that we do not use --ignore-missing-imports, # as this can give us false confidence in our results. -# mypy weatherapi +# mypy google-sheets ./scripts/static-analysis.sh diff --git a/tests/app/test_app.py b/tests/app/test_app.py index b5a8bd8..3a27068 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -2,8 +2,8 @@ from fastapi.testclient import TestClient -from weatherapi import __version__ as version -from weatherapi.app import app +from google_sheets import __version__ as version +from google_sheets.app import app client = TestClient(app) @@ -37,7 +37,7 @@ def test_weather_route(self) -> None: def test_openapi(self) -> None: expected = { "openapi": "3.1.0", - "info": {"title": "WeatherAPI", "version": version}, + "info": {"title": "google-sheets", "version": version}, "servers": [ {"url": "http://localhost:8000", "description": "Weather app server"} ], diff --git a/weatherapi/app.py b/weatherapi/app.py index 7ed9c8d..0482918 100644 --- a/weatherapi/app.py +++ b/weatherapi/app.py @@ -23,7 +23,7 @@ app = FastAPI( servers=[{"url": base_url, "description": "Weather app server"}], version=__version__, - title="WeatherAPI", + title="google-sheets", ) From af5731442e67ecc639d6628b6b4d5a1422eaa740 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:12:19 +0200 Subject: [PATCH 02/26] wip --- {weatherapi => google-sheets}/__about__.py | 0 {weatherapi => google-sheets}/__init__.py | 0 {weatherapi => google-sheets}/app.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {weatherapi => google-sheets}/__about__.py (100%) rename {weatherapi => google-sheets}/__init__.py (100%) rename {weatherapi => google-sheets}/app.py (100%) diff --git a/weatherapi/__about__.py b/google-sheets/__about__.py similarity index 100% rename from weatherapi/__about__.py rename to google-sheets/__about__.py diff --git a/weatherapi/__init__.py b/google-sheets/__init__.py similarity index 100% rename from weatherapi/__init__.py rename to google-sheets/__init__.py diff --git a/weatherapi/app.py b/google-sheets/app.py similarity index 100% rename from weatherapi/app.py rename to google-sheets/app.py From a594971f64d064213c95f1eabcf9ab68729838e5 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:12:59 +0200 Subject: [PATCH 03/26] wip --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca242ec..1c1303e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,13 +36,13 @@ After activating the virtual environment as described above, run: pip install -e ".[dev]" ``` -This will install all the dependencies and your local **WeatherAPI** in your virtual environment. +This will install all the dependencies and your local **GoogleSheetsAPI** in your virtual environment. -### Using Your local **WeatherAPI** +### Using Your local **GoogleSheetsAPI** -If you create a Python file that imports and uses **WeatherAPI**, and run it with the Python from your local environment, it will use your local **WeatherAPI** source code. +If you create a Python file that imports and uses **GoogleSheetsAPI**, and run it with the Python from your local environment, it will use your local **GoogleSheetsAPI** source code. -Whenever you update your local **WeatherAPI** source code, it will automatically use the latest version when you run your Python file again. This is because it is installed with `-e`. +Whenever you update your local **GoogleSheetsAPI** source code, it will automatically use the latest version when you run your Python file again. This is because it is installed with `-e`. This way, you don't have to "install" your local version to be able to test every change. @@ -50,7 +50,7 @@ This way, you don't have to "install" your local version to be able to test ever ### Pytest -To run tests with your current **WeatherAPI** application and Python environment, use: +To run tests with your current **GoogleSheetsAPI** application and Python environment, use: ```bash pytest tests From ab48c7f07ab1cbbe9f02e1b864078ab9cead1ab2 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:25:44 +0200 Subject: [PATCH 04/26] wip --- {google-sheets => google_sheets}/__about__.py | 0 {google-sheets => google_sheets}/__init__.py | 0 {google-sheets => google_sheets}/app.py | 0 pyproject.toml | 10 +++++----- 4 files changed, 5 insertions(+), 5 deletions(-) rename {google-sheets => google_sheets}/__about__.py (100%) rename {google-sheets => google_sheets}/__init__.py (100%) rename {google-sheets => google_sheets}/app.py (100%) diff --git a/google-sheets/__about__.py b/google_sheets/__about__.py similarity index 100% rename from google-sheets/__about__.py rename to google_sheets/__about__.py diff --git a/google-sheets/__init__.py b/google_sheets/__init__.py similarity index 100% rename from google-sheets/__init__.py rename to google_sheets/__init__.py diff --git a/google-sheets/app.py b/google_sheets/app.py similarity index 100% rename from google-sheets/app.py rename to google_sheets/app.py diff --git a/pyproject.toml b/pyproject.toml index fb4212d..34cafd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,10 +94,10 @@ Source = "https://github.com/airtai/google-sheets" Discord = "https://discord.gg/qFm6aSqq59" [project.scripts] -# google-sheets = "google-sheets.__main__:cli" +# google_sheets = "google_sheets.__main__:cli" [tool.hatch.version] -path = "google-sheets/__about__.py" +path = "google_sheets/__about__.py" [tool.hatch.build] skip-excluded-dirs = true @@ -118,7 +118,7 @@ only-include = ["google-sheets"] [tool.mypy] -files = ["google-sheets", "tests"] +files = ["google_sheets", "tests"] strict = true python_version = "3.9" @@ -145,7 +145,7 @@ disallow_any_unimported = false fix = true line-length = 88 # target-version = 'py39' -include = ["google-sheets/**/*.py", "google-sheets/**/*.pyi", "tests/**/*.py", "pyproject.toml"] +include = ["google_sheets/**/*.py", "google_sheets/**/*.pyi", "tests/**/*.py", "pyproject.toml"] exclude = ["docs/docs_src"] [tool.ruff.lint] @@ -206,7 +206,7 @@ concurrency = [ "thread" ] source = [ - "google-sheets", + "google_sheets", # "tests", ] context = '${CONTEXT}' From 2b9374eed07bf4591fdaa1ba52f2ba317193312b Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:37:01 +0200 Subject: [PATCH 05/26] wip --- pyproject.toml | 6 +++--- scripts/lint.sh | 4 ++-- scripts/static-analysis.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 34cafd2..f9fa5e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "google-sheets" -description = "google-sheets: A simple weather API made for usage with FastAgency" +description = "google-sheets: A simple Google Sheets API made for usage with FastAgency" readme = "README.md" authors = [ { name = "airt", email = "info@airt.ai" }, @@ -110,11 +110,11 @@ exclude = [ allow-direct-references = true [tool.hatch.build.targets.wheel] -only-include = ["google-sheets"] +only-include = ["google_sheets"] [tool.hatch.build.targets.wheel.sources] "src" = "" -# "scripts" = "google-sheets/templates" +# "scripts" = "google_sheets/templates" [tool.mypy] diff --git a/scripts/lint.sh b/scripts/lint.sh index f01198d..37ab623 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash echo "Running pyup_dirs..." -pyup_dirs --py38-plus --recursive google-sheets tests +pyup_dirs --py38-plus --recursive google_sheets tests echo "Running ruff linter (isort, flake, pyupgrade, etc. replacement)..." ruff check @@ -10,4 +10,4 @@ echo "Running ruff formater (black replacement)..." ruff format # echo "Running black..." -# black google-sheets examples tests docs +# black google_sheets examples tests docs diff --git a/scripts/static-analysis.sh b/scripts/static-analysis.sh index 156fba1..c8e907d 100755 --- a/scripts/static-analysis.sh +++ b/scripts/static-analysis.sh @@ -6,7 +6,7 @@ echo "Running mypy..." mypy echo "Running bandit..." -bandit -c pyproject.toml -r google-sheets +bandit -c pyproject.toml -r google_sheets echo "Running semgrep..." semgrep scan --config auto --error From 51569cc9810fd7d49b20d5708cb43f8964b045f5 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:41:23 +0200 Subject: [PATCH 06/26] Renaming done --- README.md | 4 ++-- google_sheets/__about__.py | 2 +- google_sheets/__init__.py | 2 +- google_sheets/app.py | 2 +- scripts/run_server.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e2af006..dc639e4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# Weather API -A simple weather API made for usage with FastAgency +# Google Sheets API +A simple Google Sheets API made for usage with FastAgency diff --git a/google_sheets/__about__.py b/google_sheets/__about__.py index a9fae82..787b2d1 100644 --- a/google_sheets/__about__.py +++ b/google_sheets/__about__.py @@ -1,3 +1,3 @@ -"""A simple weather API made for usage with FastAgency.""" +"""A simple Google Sheets API made for usage with FastAgency.""" __version__ = "0.1.0" diff --git a/google_sheets/__init__.py b/google_sheets/__init__.py index 43eb614..1f72f24 100644 --- a/google_sheets/__init__.py +++ b/google_sheets/__init__.py @@ -1,4 +1,4 @@ -"""A simple weather API made for usage with FastAgency.""" +"""A simple Google Sheets API made for usage with FastAgency.""" from .__about__ import __version__ diff --git a/google_sheets/app.py b/google_sheets/app.py index 0482918..e879ead 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -21,7 +21,7 @@ ) app = FastAPI( - servers=[{"url": base_url, "description": "Weather app server"}], + servers=[{"url": base_url, "description": "Google Sheets app server"}], version=__version__, title="google-sheets", ) diff --git a/scripts/run_server.sh b/scripts/run_server.sh index d0ac6d2..d8e6c98 100755 --- a/scripts/run_server.sh +++ b/scripts/run_server.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -uvicorn weatherapi.app:app --workers 2 --host 0.0.0.0 --proxy-headers +uvicorn google_sheets.app:app --workers 2 --host 0.0.0.0 --proxy-headers From f904f3b67c37d369fd20270733bf1760c53cd045 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:47:19 +0200 Subject: [PATCH 07/26] Fix ci --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1b18de6..c15db06 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,11 +29,11 @@ jobs: pip install -e ".[docs,lint]" - name: Run mypy shell: bash - run: mypy google-sheets tests + run: mypy google_sheets tests - name: Run bandit shell: bash - run: bandit -c pyproject.toml -r google-sheets + run: bandit -c pyproject.toml -r google_sheets - name: Run Semgrep shell: bash From ebecf53878ac861804b40e3f62594dbcd14346a7 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:50:02 +0200 Subject: [PATCH 08/26] Fix tests --- google_sheets/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index e879ead..0482918 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -21,7 +21,7 @@ ) app = FastAPI( - servers=[{"url": base_url, "description": "Google Sheets app server"}], + servers=[{"url": base_url, "description": "Weather app server"}], version=__version__, title="google-sheets", ) From fba1a5d9d73d9e6115d8b764bd8f1b8612ac8b3c Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 13:54:37 +0200 Subject: [PATCH 09/26] Fix Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0e4f491..81db697 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 RUN python3 -m pip install --upgrade pip -COPY google-sheets ./google-sheets +COPY google_sheets ./google_sheets COPY scripts/* pyproject.toml README.md ./ RUN pip install -e ".[dev]" From 57d2dbadf692c8a3f81658e2b80d83e9f257e8b7 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 14:48:33 +0200 Subject: [PATCH 10/26] Implement first endpoint --- .gitignore | 2 + google_sheets/app.py | 83 ++++++++++++++++++++++++++++++++++++- google_sheets/db_helpers.py | 34 +++++++++++++++ pyproject.toml | 2 + schema.prisma | 20 +++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 google_sheets/db_helpers.py create mode 100644 schema.prisma diff --git a/.gitignore b/.gitignore index 9e5161e..9a7d8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ venv* htmlcov token .DS_Store + +client_secret.json diff --git a/google_sheets/app.py b/google_sheets/app.py index 0482918..e45c386 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -1,13 +1,18 @@ import datetime +import json import logging from os import environ -from typing import Annotated, List +from pathlib import Path +from typing import Annotated, Any, List, Union import python_weather -from fastapi import FastAPI, Query +from fastapi import FastAPI, HTTPException, Query +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build from pydantic import BaseModel from . import __version__ +from .db_helpers import get_db_connection, get_wasp_db_url __all__ = ["app"] @@ -26,6 +31,19 @@ title="google-sheets", ) +# Load client secret data from the JSON file +with Path("client_secret.json").open() as secret_file: + client_secret_data = json.load(secret_file) + +# OAuth2 configuration +oauth2_settings = { + "auth_uri": client_secret_data["web"]["auth_uri"], + "tokenUrl": client_secret_data["web"]["token_uri"], + "clientId": client_secret_data["web"]["client_id"], + "clientSecret": client_secret_data["web"]["client_secret"], + "redirectUri": client_secret_data["web"]["redirect_uris"][0], +} + class HourlyForecast(BaseModel): forecast_time: datetime.time @@ -79,3 +97,64 @@ async def get_weather( hourly_forecasts=hourly_forecasts, ) return weather_response + + +async def get_user(user_id: Union[int, str]) -> Any: + wasp_db_url = await get_wasp_db_url() + async with get_db_connection(db_url=wasp_db_url) as db: + user = await db.query_first( + f'SELECT * from "User" where id={user_id}' # nosec: [B608] + ) + if not user: + raise HTTPException(status_code=404, detail=f"user_id {user_id} not found") + return user + + +async def load_user_credentials(user_id: Union[int, str]) -> Any: + await get_user(user_id=user_id) + async with get_db_connection() as db: + data = await db.gauth.find_unique_or_raise(where={"user_id": user_id}) + + return data.creds + + +def _get_sheet(user_credentials: Any, spreadshit_id: str, range: str) -> Any: + sheets_credentials = { + "refresh_token": user_credentials["refresh_token"], + "client_id": oauth2_settings["clientId"], + "client_secret": oauth2_settings["clientSecret"], + } + + creds = Credentials.from_authorized_user_info( + info=sheets_credentials, scopes=["https://www.googleapis.com/auth/spreadsheets"] + ) + service = build("sheets", "v4", credentials=creds) + + # Call the Sheets API + sheet = service.spreadsheets() + result = sheet.values().get(spreadsheetId=spreadshit_id, range=range).execute() + values = result.get("values", []) + + return values + + +@app.get("/sheet", description="Get data from a Google Sheet") +async def get_sheet( + user_id: Annotated[ + int, Query(description="The user ID for which the data is requested") + ], + spreadshit_id: Annotated[ + str, Query(description="ID of the Google Sheet to fetch data from") + ], + range: Annotated[ + str, + Query(description="The range of cells to fetch data from. E.g. 'Sheet1!A1:B2'"), + ], +) -> Union[str, List[List[str]]]: + user_credentials = await load_user_credentials(user_id) + values = _get_sheet(user_credentials, spreadshit_id, range) + + if not values: + return "No data found." + + return values # type: ignore[no-any-return] diff --git a/google_sheets/db_helpers.py b/google_sheets/db_helpers.py new file mode 100644 index 0000000..928e4b7 --- /dev/null +++ b/google_sheets/db_helpers.py @@ -0,0 +1,34 @@ +from contextlib import asynccontextmanager +from os import environ +from typing import AsyncGenerator, Optional + +from prisma import Prisma # type: ignore[attr-defined] + + +@asynccontextmanager +async def get_db_connection( + db_url: Optional[str] = None, +) -> AsyncGenerator[Prisma, None]: + if not db_url: + db_url = environ.get("DATABASE_URL", None) + if not db_url: + raise ValueError( + "No database URL provided nor set as environment variable 'DATABASE_URL'" + ) # pragma: no cover + if "connect_timeout" not in db_url: + db_url += "?connect_timeout=60" + db = Prisma(datasource={"url": db_url}) + await db.connect() + try: + yield db + finally: + await db.disconnect() + + +async def get_wasp_db_url() -> str: + curr_db_url = environ.get("DATABASE_URL") + wasp_db_name = environ.get("WASP_DB_NAME", "waspdb") + wasp_db_url = curr_db_url.replace(curr_db_url.split("/")[-1], wasp_db_name) # type: ignore[union-attr] + if "connect_timeout" not in wasp_db_url: + wasp_db_url += "?connect_timeout=60" + return wasp_db_url diff --git a/pyproject.toml b/pyproject.toml index f9fa5e1..4f9943c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,8 @@ dependencies = [ "pydantic>=2.3,<3", "fastapi>=0.110.2", "python-weather==2.0.3", + "prisma==0.13.1", + "google-api-python-client==2.133.0", ] [project.optional-dependencies] diff --git a/schema.prisma b/schema.prisma new file mode 100644 index 0000000..24477d1 --- /dev/null +++ b/schema.prisma @@ -0,0 +1,20 @@ +datasource db { + // could be postgresql or mysql + provider = "postgresql" + url = env("DATABASE_URL") +} + +generator db { + provider = "prisma-client-py" + interface = "asyncio" + recursive_type_depth = 5 +} + +model GAuth { + id String @id @default(cuid()) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + user_id Int @unique + creds Json + info Json +} From 94e6328a56c2ca9a78c5651f584cfafb12c45f06 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Tue, 18 Jun 2024 15:53:12 +0200 Subject: [PATCH 11/26] wip --- google_sheets/app.py | 6 ++++-- pyproject.toml | 1 + tests/app/test_app.py | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index e45c386..3a7a520 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -6,6 +6,7 @@ from typing import Annotated, Any, List, Union import python_weather +from asyncify import asyncify from fastapi import FastAPI, HTTPException, Query from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -118,6 +119,7 @@ async def load_user_credentials(user_id: Union[int, str]) -> Any: return data.creds +@asyncify # type: ignore[misc] def _get_sheet(user_credentials: Any, spreadshit_id: str, range: str) -> Any: sheets_credentials = { "refresh_token": user_credentials["refresh_token"], @@ -143,7 +145,7 @@ async def get_sheet( user_id: Annotated[ int, Query(description="The user ID for which the data is requested") ], - spreadshit_id: Annotated[ + spreadsheet_id: Annotated[ str, Query(description="ID of the Google Sheet to fetch data from") ], range: Annotated[ @@ -152,7 +154,7 @@ async def get_sheet( ], ) -> Union[str, List[List[str]]]: user_credentials = await load_user_credentials(user_id) - values = _get_sheet(user_credentials, spreadshit_id, range) + values = await _get_sheet(user_credentials, spreadsheet_id, range) if not values: return "No data found." diff --git a/pyproject.toml b/pyproject.toml index 4f9943c..5536ba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "python-weather==2.0.3", "prisma==0.13.1", "google-api-python-client==2.133.0", + "asyncify==0.10.0", ] [project.optional-dependencies] diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 3a27068..4e67a79 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -1,4 +1,5 @@ import datetime +from unittest.mock import patch from fastapi.testclient import TestClient @@ -9,6 +10,28 @@ class TestRoutes: + def test_get_sheet(self) -> None: + with patch( + "google_sheets.app.load_user_credentials", + return_value={"refresh_token": "abcdf"}, + ) as mock_load_user_credentials: + excepted = [ + ["Campaign", "Ad Group", "Keyword"], + ["Campaign A", "Ad group A", "Keyword A"], + ["Campaign A", "Ad group A", "Keyword B"], + ["Campaign A", "Ad group A", "Keyword C"], + ] + with patch( + "google_sheets.app._get_sheet", return_value=excepted + ) as mock_get_sheet: + response = client.get( + "/sheet?user_id=123&spreadsheet_id=abc&range=Sheet1" + ) + mock_load_user_credentials.assert_called_once() + mock_get_sheet.assert_called_once() + assert response.status_code == 200 + assert response.json() == excepted + def test_weather_route(self) -> None: response = client.get("/?city=Chennai") assert response.status_code == 200 From 529a3b917328b93e33ea5f654c23a73fd4800976 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 09:22:01 +0200 Subject: [PATCH 12/26] Fix ci --- .github/workflows/test.yaml | 13 +- google_sheets/app.py | 8 +- tests/app/test_app.py | 284 ++++++++++++++++++------------------ 3 files changed, 158 insertions(+), 147 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c15db06..c19c0fc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -70,7 +70,10 @@ jobs: if: matrix.pydantic-version == 'pydantic-v2' run: pip install --pre "pydantic>=2,<3" - run: mkdir coverage - + - name: Create client secrets file + run: echo '{"web":{"client_id":"dummy.apps.googleusercontent.com","project_id":"dummy-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"dummy-secret","redirect_uris":["http://localhost:9000/login/callback"]}}' > client_secret.json + - name: Prisma generate + run: prisma generate - name: Test run: bash scripts/test.sh env: @@ -98,6 +101,10 @@ jobs: - name: Install Dependencies if: steps.cache.outputs.cache-hit != 'true' run: pip install .[docs,testing] + - name: Create client secrets file + run: echo '{"web":{"client_id":"dummy.apps.googleusercontent.com","project_id":"dummy-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"dummy-secret","redirect_uris":["http://localhost:9000/login/callback"]}}' > client_secret.json + - name: Prisma generate + run: prisma generate - name: Test run: bash scripts/test.sh @@ -116,6 +123,10 @@ jobs: - name: Install Dependencies if: steps.cache.outputs.cache-hit != 'true' run: pip install .[docs,testing] + - name: Create client secrets file + run: echo '{"web":{"client_id":"dummy.apps.googleusercontent.com","project_id":"dummy-id","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_secret":"dummy-secret","redirect_uris":["http://localhost:9000/login/callback"]}}' > client_secret.json + - name: Prisma generate + run: prisma generate - name: Test run: bash scripts/test.sh diff --git a/google_sheets/app.py b/google_sheets/app.py index 3a7a520..5ea0033 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -3,7 +3,7 @@ import logging from os import environ from pathlib import Path -from typing import Annotated, Any, List, Union +from typing import Annotated, Any, Dict, List, Union import python_weather from asyncify import asyncify @@ -114,20 +114,20 @@ async def get_user(user_id: Union[int, str]) -> Any: async def load_user_credentials(user_id: Union[int, str]) -> Any: await get_user(user_id=user_id) async with get_db_connection() as db: - data = await db.gauth.find_unique_or_raise(where={"user_id": user_id}) + data = await db.gauth.find_unique_or_raise(where={"user_id": user_id}) # type: ignore[typeddict-item] return data.creds @asyncify # type: ignore[misc] def _get_sheet(user_credentials: Any, spreadshit_id: str, range: str) -> Any: - sheets_credentials = { + sheets_credentials: Dict[str, str] = { "refresh_token": user_credentials["refresh_token"], "client_id": oauth2_settings["clientId"], "client_secret": oauth2_settings["clientSecret"], } - creds = Credentials.from_authorized_user_info( + creds = Credentials.from_authorized_user_info( # type: ignore[no-untyped-call] info=sheets_credentials, scopes=["https://www.googleapis.com/auth/spreadsheets"] ) service = build("sheets", "v4", credentials=creds) diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 4e67a79..dd8130a 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -3,7 +3,7 @@ from fastapi.testclient import TestClient -from google_sheets import __version__ as version +# from google_sheets import __version__ as version from google_sheets.app import app client = TestClient(app) @@ -57,145 +57,145 @@ def test_weather_route(self) -> None: assert first_hourly_forecast.get("temperature") > 0 # type: ignore assert first_hourly_forecast.get("description") is not None - def test_openapi(self) -> None: - expected = { - "openapi": "3.1.0", - "info": {"title": "google-sheets", "version": version}, - "servers": [ - {"url": "http://localhost:8000", "description": "Weather app server"} - ], - "paths": { - "/": { - "get": { - "summary": "Get Weather", - "operationId": "get_weather__get", - "description": "Get weather forecast for a given city", - "parameters": [ - { - "name": "city", - "in": "query", - "description": "city for which forecast is requested", - "required": True, - "schema": { - "type": "string", - "title": "City", - "description": "city for which forecast is requested", - }, - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Weather" - } - } - }, - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - }, - }, - } - } - }, - "components": { - "schemas": { - "DailyForecast": { - "properties": { - "forecast_date": { - "type": "string", - "format": "date", - "title": "Forecast Date", - }, - "temperature": {"type": "integer", "title": "Temperature"}, - "hourly_forecasts": { - "items": { - "$ref": "#/components/schemas/HourlyForecast" - }, - "type": "array", - "title": "Hourly Forecasts", - }, - }, - "type": "object", - "required": [ - "forecast_date", - "temperature", - "hourly_forecasts", - ], - "title": "DailyForecast", - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "type": "array", - "title": "Detail", - } - }, - "type": "object", - "title": "HTTPValidationError", - }, - "HourlyForecast": { - "properties": { - "forecast_time": { - "type": "string", - "format": "time", - "title": "Forecast Time", - }, - "temperature": {"type": "integer", "title": "Temperature"}, - "description": {"type": "string", "title": "Description"}, - }, - "type": "object", - "required": ["forecast_time", "temperature", "description"], - "title": "HourlyForecast", - }, - "ValidationError": { - "properties": { - "loc": { - "items": { - "anyOf": [{"type": "string"}, {"type": "integer"}] - }, - "type": "array", - "title": "Location", - }, - "msg": {"type": "string", "title": "Message"}, - "type": {"type": "string", "title": "Error Type"}, - }, - "type": "object", - "required": ["loc", "msg", "type"], - "title": "ValidationError", - }, - "Weather": { - "properties": { - "city": {"type": "string", "title": "City"}, - "temperature": {"type": "integer", "title": "Temperature"}, - "daily_forecasts": { - "items": {"$ref": "#/components/schemas/DailyForecast"}, - "type": "array", - "title": "Daily Forecasts", - }, - }, - "type": "object", - "required": ["city", "temperature", "daily_forecasts"], - "title": "Weather", - }, - } - }, - } - response = client.get("/openapi.json") - assert response.status_code == 200 - resp_json = response.json() + # def test_openapi(self) -> None: + # expected = { + # "openapi": "3.1.0", + # "info": {"title": "google-sheets", "version": version}, + # "servers": [ + # {"url": "http://localhost:8000", "description": "Weather app server"} + # ], + # "paths": { + # "/": { + # "get": { + # "summary": "Get Weather", + # "operationId": "get_weather__get", + # "description": "Get weather forecast for a given city", + # "parameters": [ + # { + # "name": "city", + # "in": "query", + # "description": "city for which forecast is requested", + # "required": True, + # "schema": { + # "type": "string", + # "title": "City", + # "description": "city for which forecast is requested", + # }, + # } + # ], + # "responses": { + # "200": { + # "description": "Successful Response", + # "content": { + # "application/json": { + # "schema": { + # "$ref": "#/components/schemas/Weather" + # } + # } + # }, + # }, + # "422": { + # "description": "Validation Error", + # "content": { + # "application/json": { + # "schema": { + # "$ref": "#/components/schemas/HTTPValidationError" + # } + # } + # }, + # }, + # }, + # } + # } + # }, + # "components": { + # "schemas": { + # "DailyForecast": { + # "properties": { + # "forecast_date": { + # "type": "string", + # "format": "date", + # "title": "Forecast Date", + # }, + # "temperature": {"type": "integer", "title": "Temperature"}, + # "hourly_forecasts": { + # "items": { + # "$ref": "#/components/schemas/HourlyForecast" + # }, + # "type": "array", + # "title": "Hourly Forecasts", + # }, + # }, + # "type": "object", + # "required": [ + # "forecast_date", + # "temperature", + # "hourly_forecasts", + # ], + # "title": "DailyForecast", + # }, + # "HTTPValidationError": { + # "properties": { + # "detail": { + # "items": { + # "$ref": "#/components/schemas/ValidationError" + # }, + # "type": "array", + # "title": "Detail", + # } + # }, + # "type": "object", + # "title": "HTTPValidationError", + # }, + # "HourlyForecast": { + # "properties": { + # "forecast_time": { + # "type": "string", + # "format": "time", + # "title": "Forecast Time", + # }, + # "temperature": {"type": "integer", "title": "Temperature"}, + # "description": {"type": "string", "title": "Description"}, + # }, + # "type": "object", + # "required": ["forecast_time", "temperature", "description"], + # "title": "HourlyForecast", + # }, + # "ValidationError": { + # "properties": { + # "loc": { + # "items": { + # "anyOf": [{"type": "string"}, {"type": "integer"}] + # }, + # "type": "array", + # "title": "Location", + # }, + # "msg": {"type": "string", "title": "Message"}, + # "type": {"type": "string", "title": "Error Type"}, + # }, + # "type": "object", + # "required": ["loc", "msg", "type"], + # "title": "ValidationError", + # }, + # "Weather": { + # "properties": { + # "city": {"type": "string", "title": "City"}, + # "temperature": {"type": "integer", "title": "Temperature"}, + # "daily_forecasts": { + # "items": {"$ref": "#/components/schemas/DailyForecast"}, + # "type": "array", + # "title": "Daily Forecasts", + # }, + # }, + # "type": "object", + # "required": ["city", "temperature", "daily_forecasts"], + # "title": "Weather", + # }, + # } + # }, + # } + # response = client.get("/openapi.json") + # assert response.status_code == 200 + # resp_json = response.json() - assert resp_json == expected + # assert resp_json == expected From a871e615798a1d4f762836245892fa26671215b5 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 10:24:19 +0200 Subject: [PATCH 13/26] Cleanup --- google_sheets/app.py | 59 +-------- pyproject.toml | 1 - tests/app/test_app.py | 289 ++++++++++++++++++------------------------ 3 files changed, 124 insertions(+), 225 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 5ea0033..9db90a8 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -1,16 +1,13 @@ -import datetime import json import logging from os import environ from pathlib import Path from typing import Annotated, Any, Dict, List, Union -import python_weather from asyncify import asyncify from fastapi import FastAPI, HTTPException, Query from google.oauth2.credentials import Credentials from googleapiclient.discovery import build -from pydantic import BaseModel from . import __version__ from .db_helpers import get_db_connection, get_wasp_db_url @@ -27,7 +24,7 @@ ) app = FastAPI( - servers=[{"url": base_url, "description": "Weather app server"}], + servers=[{"url": base_url, "description": "Google Sheets app server"}], version=__version__, title="google-sheets", ) @@ -46,60 +43,6 @@ } -class HourlyForecast(BaseModel): - forecast_time: datetime.time - temperature: int - description: str - - -class DailyForecast(BaseModel): - forecast_date: datetime.date - temperature: int - hourly_forecasts: List[HourlyForecast] - - -class Weather(BaseModel): - city: str - temperature: int - daily_forecasts: List[DailyForecast] - - -@app.get("/", description="Get weather forecast for a given city") -async def get_weather( - city: Annotated[str, Query(description="city for which forecast is requested")], -) -> Weather: - async with python_weather.Client(unit=python_weather.METRIC) as client: - # fetch a weather forecast from a city - weather = await client.get(city) - - daily_forecasts = [] - # get the weather forecast for a few days - for daily in weather.daily_forecasts: - hourly_forecasts = [ - HourlyForecast( - forecast_time=hourly.time, - temperature=hourly.temperature, - description=hourly.description, - ) - for hourly in daily.hourly_forecasts - ] - daily_forecasts.append( - DailyForecast( - forecast_date=daily.date, - temperature=daily.temperature, - hourly_forecasts=hourly_forecasts, - ) - ) - - weather_response = Weather( - city=city, - temperature=weather.temperature, - daily_forecasts=daily_forecasts, - hourly_forecasts=hourly_forecasts, - ) - return weather_response - - async def get_user(user_id: Union[int, str]) -> Any: wasp_db_url = await get_wasp_db_url() async with get_db_connection(db_url=wasp_db_url) as db: diff --git a/pyproject.toml b/pyproject.toml index 5536ba4..4212252 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,6 @@ dynamic = ["version"] dependencies = [ "pydantic>=2.3,<3", "fastapi>=0.110.2", - "python-weather==2.0.3", "prisma==0.13.1", "google-api-python-client==2.133.0", "asyncify==0.10.0", diff --git a/tests/app/test_app.py b/tests/app/test_app.py index dd8130a..4f541c7 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -1,9 +1,8 @@ -import datetime from unittest.mock import patch from fastapi.testclient import TestClient -# from google_sheets import __version__ as version +from google_sheets import __version__ as version from google_sheets.app import app client = TestClient(app) @@ -32,170 +31,128 @@ def test_get_sheet(self) -> None: assert response.status_code == 200 assert response.json() == excepted - def test_weather_route(self) -> None: - response = client.get("/?city=Chennai") + def test_openapi(self) -> None: + expected = { + "openapi": "3.1.0", + "info": {"title": "google-sheets", "version": version}, + "servers": [ + { + "url": "http://localhost:8000", + "description": "Google Sheets app server", + } + ], + "paths": { + "/sheet": { + "get": { + "summary": "Get Sheet", + "description": "Get data from a Google Sheet", + "operationId": "get_sheet_sheet_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": True, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id", + }, + "description": "The user ID for which the data is requested", + }, + { + "name": "spreadsheet_id", + "in": "query", + "required": True, + "schema": { + "type": "string", + "description": "ID of the Google Sheet to fetch data from", + "title": "Spreadsheet Id", + }, + "description": "ID of the Google Sheet to fetch data from", + }, + { + "name": "range", + "in": "query", + "required": True, + "schema": { + "type": "string", + "description": "The range of cells to fetch data from. E.g. 'Sheet1!A1:B2'", + "title": "Range", + }, + "description": "The range of cells to fetch data from. E.g. 'Sheet1!A1:B2'", + }, + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "type": "array", + "items": {"type": "string"}, + }, + }, + ], + "title": "Response Get Sheet Sheet Get", + } + } + }, + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + }, + }, + } + } + }, + "components": { + "schemas": { + "HTTPValidationError": { + "properties": { + "detail": { + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "type": "array", + "title": "Detail", + } + }, + "type": "object", + "title": "HTTPValidationError", + }, + "ValidationError": { + "properties": { + "loc": { + "items": { + "anyOf": [{"type": "string"}, {"type": "integer"}] + }, + "type": "array", + "title": "Location", + }, + "msg": {"type": "string", "title": "Message"}, + "type": {"type": "string", "title": "Error Type"}, + }, + "type": "object", + "required": ["loc", "msg", "type"], + "title": "ValidationError", + }, + } + }, + } + response = client.get("/openapi.json") assert response.status_code == 200 resp_json = response.json() - assert resp_json.get("city") == "Chennai" - assert resp_json.get("temperature") > 0 - assert len(resp_json.get("daily_forecasts")) > 0 - daily_forecasts = resp_json.get("daily_forecasts") - assert isinstance(daily_forecasts, list) - - first_daily_forecast = daily_forecasts[0] - assert ( - first_daily_forecast.get("forecast_date") - == datetime.date.today().isoformat() - ) - assert first_daily_forecast.get("temperature") > 0 - assert len(first_daily_forecast.get("hourly_forecasts")) > 0 - - first_hourly_forecast = first_daily_forecast.get("hourly_forecasts")[0] - assert isinstance(first_hourly_forecast, dict) - assert first_hourly_forecast.get("forecast_time") is not None - assert first_hourly_forecast.get("temperature") > 0 # type: ignore - assert first_hourly_forecast.get("description") is not None - - # def test_openapi(self) -> None: - # expected = { - # "openapi": "3.1.0", - # "info": {"title": "google-sheets", "version": version}, - # "servers": [ - # {"url": "http://localhost:8000", "description": "Weather app server"} - # ], - # "paths": { - # "/": { - # "get": { - # "summary": "Get Weather", - # "operationId": "get_weather__get", - # "description": "Get weather forecast for a given city", - # "parameters": [ - # { - # "name": "city", - # "in": "query", - # "description": "city for which forecast is requested", - # "required": True, - # "schema": { - # "type": "string", - # "title": "City", - # "description": "city for which forecast is requested", - # }, - # } - # ], - # "responses": { - # "200": { - # "description": "Successful Response", - # "content": { - # "application/json": { - # "schema": { - # "$ref": "#/components/schemas/Weather" - # } - # } - # }, - # }, - # "422": { - # "description": "Validation Error", - # "content": { - # "application/json": { - # "schema": { - # "$ref": "#/components/schemas/HTTPValidationError" - # } - # } - # }, - # }, - # }, - # } - # } - # }, - # "components": { - # "schemas": { - # "DailyForecast": { - # "properties": { - # "forecast_date": { - # "type": "string", - # "format": "date", - # "title": "Forecast Date", - # }, - # "temperature": {"type": "integer", "title": "Temperature"}, - # "hourly_forecasts": { - # "items": { - # "$ref": "#/components/schemas/HourlyForecast" - # }, - # "type": "array", - # "title": "Hourly Forecasts", - # }, - # }, - # "type": "object", - # "required": [ - # "forecast_date", - # "temperature", - # "hourly_forecasts", - # ], - # "title": "DailyForecast", - # }, - # "HTTPValidationError": { - # "properties": { - # "detail": { - # "items": { - # "$ref": "#/components/schemas/ValidationError" - # }, - # "type": "array", - # "title": "Detail", - # } - # }, - # "type": "object", - # "title": "HTTPValidationError", - # }, - # "HourlyForecast": { - # "properties": { - # "forecast_time": { - # "type": "string", - # "format": "time", - # "title": "Forecast Time", - # }, - # "temperature": {"type": "integer", "title": "Temperature"}, - # "description": {"type": "string", "title": "Description"}, - # }, - # "type": "object", - # "required": ["forecast_time", "temperature", "description"], - # "title": "HourlyForecast", - # }, - # "ValidationError": { - # "properties": { - # "loc": { - # "items": { - # "anyOf": [{"type": "string"}, {"type": "integer"}] - # }, - # "type": "array", - # "title": "Location", - # }, - # "msg": {"type": "string", "title": "Message"}, - # "type": {"type": "string", "title": "Error Type"}, - # }, - # "type": "object", - # "required": ["loc", "msg", "type"], - # "title": "ValidationError", - # }, - # "Weather": { - # "properties": { - # "city": {"type": "string", "title": "City"}, - # "temperature": {"type": "integer", "title": "Temperature"}, - # "daily_forecasts": { - # "items": {"$ref": "#/components/schemas/DailyForecast"}, - # "type": "array", - # "title": "Daily Forecasts", - # }, - # }, - # "type": "object", - # "required": ["city", "temperature", "daily_forecasts"], - # "title": "Weather", - # }, - # } - # }, - # } - # response = client.get("/openapi.json") - # assert response.status_code == 200 - # resp_json = response.json() - - # assert resp_json == expected + assert resp_json == expected From 50aeb37c4db089e07079ffbdbb2f656bd95fee16 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 10:43:43 +0200 Subject: [PATCH 14/26] Refactoring --- google_sheets/app.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 9db90a8..a10228c 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -62,8 +62,8 @@ async def load_user_credentials(user_id: Union[int, str]) -> Any: return data.creds -@asyncify # type: ignore[misc] -def _get_sheet(user_credentials: Any, spreadshit_id: str, range: str) -> Any: +async def _build_service(user_id: int) -> Any: + user_credentials = await load_user_credentials(user_id) sheets_credentials: Dict[str, str] = { "refresh_token": user_credentials["refresh_token"], "client_id": oauth2_settings["clientId"], @@ -74,10 +74,14 @@ def _get_sheet(user_credentials: Any, spreadshit_id: str, range: str) -> Any: info=sheets_credentials, scopes=["https://www.googleapis.com/auth/spreadsheets"] ) service = build("sheets", "v4", credentials=creds) + return service + +@asyncify # type: ignore[misc] +def _get_sheet(service: Any, spreadsheet_id: str, range: str) -> Any: # Call the Sheets API sheet = service.spreadsheets() - result = sheet.values().get(spreadsheetId=spreadshit_id, range=range).execute() + result = sheet.values().get(spreadsheetId=spreadsheet_id, range=range).execute() values = result.get("values", []) return values @@ -96,8 +100,10 @@ async def get_sheet( Query(description="The range of cells to fetch data from. E.g. 'Sheet1!A1:B2'"), ], ) -> Union[str, List[List[str]]]: - user_credentials = await load_user_credentials(user_id) - values = await _get_sheet(user_credentials, spreadsheet_id, range) + service = await _build_service(user_id) + values = await _get_sheet( + service=service, spreadsheet_id=spreadsheet_id, range=range + ) if not values: return "No data found." From f33e07c4f688dbf4bd822d82e9fdcf8d88b40468 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 12:38:10 +0200 Subject: [PATCH 15/26] wip --- .gitignore | 2 +- google_sheets/app.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9a7d8c6..9372fbb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ htmlcov token .DS_Store -client_secret.json +client_secret*.json diff --git a/google_sheets/app.py b/google_sheets/app.py index a10228c..c1e5c49 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -87,6 +87,33 @@ def _get_sheet(service: Any, spreadsheet_id: str, range: str) -> Any: return values +async def list_sheets(user_id: int) -> List[Any]: + user_credentials = await load_user_credentials(user_id) + sheets_credentials: Dict[str, str] = { + "refresh_token": user_credentials["refresh_token"], + "client_id": oauth2_settings["clientId"], + "client_secret": oauth2_settings["clientSecret"], + } + + creds = Credentials.from_authorized_user_info( # type: ignore[no-untyped-call] + info=sheets_credentials, + scopes=[ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/drive.metadata.readonly", + ], + ) + service = build("drive", "v3", credentials=creds) + + # Call the Drive v3 API + results = ( + service.files() + .list(pageSize=10, fields="nextPageToken, files(id, name)") + .execute() + ) + items = results.get("files", []) + return items # type: ignore[no-any-return] + + @app.get("/sheet", description="Get data from a Google Sheet") async def get_sheet( user_id: Annotated[ @@ -109,3 +136,13 @@ async def get_sheet( return "No data found." return values # type: ignore[no-any-return] + + +@app.get("/all", description="Get all sheets associated with the user") +async def get_all_file_names( + user_id: Annotated[ + int, Query(description="The user ID for which the data is requested") + ], +) -> List[Any]: + files = await list_sheets(user_id=user_id) + return files From 7b59089ea5b9efe0dc7ab5b4c7c90101760e8b2f Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 12:55:02 +0200 Subject: [PATCH 16/26] wip --- google_sheets/app.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index c1e5c49..19e5ac3 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -87,7 +87,7 @@ def _get_sheet(service: Any, spreadsheet_id: str, range: str) -> Any: return values -async def list_sheets(user_id: int) -> List[Any]: +async def list_sheets(user_id: int) -> List[Dict[str, str]]: user_credentials = await load_user_credentials(user_id) sheets_credentials: Dict[str, str] = { "refresh_token": user_credentials["refresh_token"], @@ -107,7 +107,11 @@ async def list_sheets(user_id: int) -> List[Any]: # Call the Drive v3 API results = ( service.files() - .list(pageSize=10, fields="nextPageToken, files(id, name)") + .list( + q="mimeType='application/vnd.google-apps.spreadsheet'", + pageSize=10, + fields="nextPageToken, files(id, name)", + ) .execute() ) items = results.get("files", []) @@ -143,6 +147,8 @@ async def get_all_file_names( user_id: Annotated[ int, Query(description="The user ID for which the data is requested") ], -) -> List[Any]: - files = await list_sheets(user_id=user_id) - return files +) -> Dict[str, str]: + files: List[Dict[str, str]] = await list_sheets(user_id=user_id) + # create dict where key is id and value is name + files_dict = {file["id"]: file["name"] for file in files} + return files_dict From 9414b1eae97a395e658226eea6960e3c13f5dc48 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 13:28:15 +0200 Subject: [PATCH 17/26] Return error message ig the user hasn't logged in yet --- google_sheets/app.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 19e5ac3..516fdc9 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -8,6 +8,7 @@ from fastapi import FastAPI, HTTPException, Query from google.oauth2.credentials import Credentials from googleapiclient.discovery import build +from prisma.errors import RecordNotFoundError from . import __version__ from .db_helpers import get_db_connection, get_wasp_db_url @@ -57,12 +58,17 @@ async def get_user(user_id: Union[int, str]) -> Any: async def load_user_credentials(user_id: Union[int, str]) -> Any: await get_user(user_id=user_id) async with get_db_connection() as db: - data = await db.gauth.find_unique_or_raise(where={"user_id": user_id}) # type: ignore[typeddict-item] + try: + data = await db.gauth.find_unique_or_raise(where={"user_id": user_id}) # type: ignore[typeddict-item] + except RecordNotFoundError as e: + raise HTTPException( + status_code=404, detail="User hasn't grant access yet!" + ) from e return data.creds -async def _build_service(user_id: int) -> Any: +async def _build_service(user_id: int, service_name: str, version: str) -> Any: user_credentials = await load_user_credentials(user_id) sheets_credentials: Dict[str, str] = { "refresh_token": user_credentials["refresh_token"], @@ -71,9 +77,13 @@ async def _build_service(user_id: int) -> Any: } creds = Credentials.from_authorized_user_info( # type: ignore[no-untyped-call] - info=sheets_credentials, scopes=["https://www.googleapis.com/auth/spreadsheets"] + info=sheets_credentials, + scopes=[ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/drive.metadata.readonly", + ], ) - service = build("sheets", "v4", credentials=creds) + service = build(serviceName=service_name, version=version, credentials=creds) return service @@ -87,23 +97,8 @@ def _get_sheet(service: Any, spreadsheet_id: str, range: str) -> Any: return values -async def list_sheets(user_id: int) -> List[Dict[str, str]]: - user_credentials = await load_user_credentials(user_id) - sheets_credentials: Dict[str, str] = { - "refresh_token": user_credentials["refresh_token"], - "client_id": oauth2_settings["clientId"], - "client_secret": oauth2_settings["clientSecret"], - } - - creds = Credentials.from_authorized_user_info( # type: ignore[no-untyped-call] - info=sheets_credentials, - scopes=[ - "https://www.googleapis.com/auth/spreadsheets", - "https://www.googleapis.com/auth/drive.metadata.readonly", - ], - ) - service = build("drive", "v3", credentials=creds) - +@asyncify # type: ignore[misc] +def list_sheets(service: Any) -> List[Dict[str, str]]: # Call the Drive v3 API results = ( service.files() @@ -131,7 +126,7 @@ async def get_sheet( Query(description="The range of cells to fetch data from. E.g. 'Sheet1!A1:B2'"), ], ) -> Union[str, List[List[str]]]: - service = await _build_service(user_id) + service = await _build_service(user_id=user_id, service_name="sheets", version="v4") values = await _get_sheet( service=service, spreadsheet_id=spreadsheet_id, range=range ) @@ -148,7 +143,8 @@ async def get_all_file_names( int, Query(description="The user ID for which the data is requested") ], ) -> Dict[str, str]: - files: List[Dict[str, str]] = await list_sheets(user_id=user_id) + service = await _build_service(user_id=user_id, service_name="drive", version="v3") + files: List[Dict[str, str]] = await list_sheets(service=service) # create dict where key is id and value is name files_dict = {file["id"]: file["name"] for file in files} return files_dict From b14d692ef12ba3e39108b31303e40ac0a8414ea1 Mon Sep 17 00:00:00 2001 From: Robert Jambrecic Date: Wed, 19 Jun 2024 14:40:56 +0200 Subject: [PATCH 18/26] Refactoring and tests added --- google_sheets/app.py | 36 +++++++++++------------ tests/app/test_app.py | 67 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/google_sheets/app.py b/google_sheets/app.py index 516fdc9..3bcd57c 100644 --- a/google_sheets/app.py +++ b/google_sheets/app.py @@ -97,22 +97,6 @@ def _get_sheet(service: Any, spreadsheet_id: str, range: str) -> Any: return values -@asyncify # type: ignore[misc] -def list_sheets(service: Any) -> List[Dict[str, str]]: - # Call the Drive v3 API - results = ( - service.files() - .list( - q="mimeType='application/vnd.google-apps.spreadsheet'", - pageSize=10, - fields="nextPageToken, files(id, name)", - ) - .execute() - ) - items = results.get("files", []) - return items # type: ignore[no-any-return] - - @app.get("/sheet", description="Get data from a Google Sheet") async def get_sheet( user_id: Annotated[ @@ -137,14 +121,30 @@ async def get_sheet( return values # type: ignore[no-any-return] -@app.get("/all", description="Get all sheets associated with the user") +@asyncify # type: ignore[misc] +def _get_files(service: Any) -> List[Dict[str, str]]: + # Call the Drive v3 API + results = ( + service.files() + .list( + q="mimeType='application/vnd.google-apps.spreadsheet'", + pageSize=100, # The default value is 100 + fields="nextPageToken, files(id, name)", + ) + .execute() + ) + items = results.get("files", []) + return items # type: ignore[no-any-return] + + +@app.get("/get-all-file-names", description="Get all sheets associated with the user") async def get_all_file_names( user_id: Annotated[ int, Query(description="The user ID for which the data is requested") ], ) -> Dict[str, str]: service = await _build_service(user_id=user_id, service_name="drive", version="v3") - files: List[Dict[str, str]] = await list_sheets(service=service) + files: List[Dict[str, str]] = await _get_files(service=service) # create dict where key is id and value is name files_dict = {file["id"]: file["name"] for file in files} return files_dict diff --git a/tests/app/test_app.py b/tests/app/test_app.py index 4f541c7..af8e8ba 100644 --- a/tests/app/test_app.py +++ b/tests/app/test_app.py @@ -31,6 +31,27 @@ def test_get_sheet(self) -> None: assert response.status_code == 200 assert response.json() == excepted + def test_get_all_file_names(self) -> None: + with ( + patch( + "google_sheets.app.load_user_credentials", + return_value={"refresh_token": "abcdf"}, + ) as mock_load_user_credentials, + patch( + "google_sheets.app._get_files", + return_value=[ + {"id": "abc", "name": "file1"}, + {"id": "def", "name": "file2"}, + ], + ) as mock_get_files, + ): + expected = {"abc": "file1", "def": "file2"} + response = client.get("/get-all-file-names?user_id=123") + mock_load_user_credentials.assert_called_once() + mock_get_files.assert_called_once() + assert response.status_code == 200 + assert response.json() == expected + def test_openapi(self) -> None: expected = { "openapi": "3.1.0", @@ -115,7 +136,51 @@ def test_openapi(self) -> None: }, }, } - } + }, + "/get-all-file-names": { + "get": { + "summary": "Get All File Names", + "description": "Get all sheets associated with the user", + "operationId": "get_all_file_names_get_all_file_names_get", + "parameters": [ + { + "name": "user_id", + "in": "query", + "required": True, + "schema": { + "type": "integer", + "description": "The user ID for which the data is requested", + "title": "User Id", + }, + "description": "The user ID for which the data is requested", + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": {"type": "string"}, + "title": "Response Get All File Names Get All File Names Get", + } + } + }, + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + }, + }, + } + }, }, "components": { "schemas": { From eb68004024cb93f7c1177ec65c704156e32f1ab5 Mon Sep 17 00:00:00 2001 From: rjambrecic <32619626+rjambrecic@users.noreply.github.com> Date: Thu, 20 Jun 2024 10:32:59 +0200 Subject: [PATCH 19/26] Update deploy pipeline (#12) * Update deploy pipeline * Remove loki logging driver --------- Co-authored-by: Kumaran Rajendhiran --- .github/workflows/test.yaml | 4 +++- docker-compose.yaml | 12 ------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c19c0fc..1618ddd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -235,9 +235,11 @@ jobs: needs: [check] if: github.ref_name == 'main' || github.ref_name == 'dev' env: + CLIENT_SECRET: ${{ github.ref_name == 'main' && secrets.PROD_CLIENT_SECRET || secrets.STAGING_CLIENT_SECRET }} + DATABASE_URL: ${{ github.ref_name == 'main' && secrets.PROD_DATABASE_URL || secrets.STAGING_DATABASE_URL }} GITHUB_USERNAME: ${{ github.actor }} GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} - DEVELOPER_TOKEN: ${{ secrets.DEVELOPER_TOKEN }} + # DEVELOPER_TOKEN: ${{ secrets.DEVELOPER_TOKEN }} DOMAIN: ${{ github.ref_name == 'main' && vars.PROD_DOMAIN || vars.STAGING_DOMAIN }} SSH_KEY: ${{ github.ref_name == 'main' && secrets.PROD_SSH_KEY || secrets.STAGING_SSH_KEY }} steps: diff --git a/docker-compose.yaml b/docker-compose.yaml index 2d5f2cd..7a466b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,3 @@ -x-logging: &default-logging - driver: loki - options: - loki-url: 'http://localhost:3100/api/prom/push' - loki-pipeline-stages: | - - multiline: - firstline: '^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2}' - max_wait_time: 3s - - regex: - expression: '^(?P