Merge pull request #23 from GSA/feature/prepare-for-pypi

init pypi github actions config

.github/workflows/deploy.yml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+on:
+  # TODO: configure repo to and uncomment below
+  # pull_request:
+  #   branches: [main]
+  #   types: [closed]
+  workflow_dispatch:
+    inputs:
+      version_no:
+        description: 'Release Version:'
+        required: true
+
+jobs:
+  deploy:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Update setup.py if manual release
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          # TODO update for pyproject.toml
+          sed -i "s/version='[0-9]\{1,2\}.[0-9]\{1,4\}.[0-9]\{1,4\}',/version='${{github.event.inputs.version_no}}',/g" setup.py
+      - name: Create packages
+        run: |
+          python setup.py sdist
+          python setup.py bdist_wheel
+      - name: pypi-publish
+        uses: pypa/gh-action-pypi-publish@v1.8.11
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/test.yml

@@ -1,10 +1,10 @@
 ---
 name: Tests on Commit
-on: [push, pull_request]
+on: [push]
 
 env:
-  PY_VERSION: "3.9"
-  POETRY_VERSION: "1.4.2"
+  PY_VERSION: "3.10"
+  POETRY_VERSION: "1.7.1"
 
 jobs:
   lint:

.gitignore

@@ -7,6 +7,7 @@ pytest-coverage.txt
 __pycache__/
 venv/
 credentials.py
+dist/
 
 # any previous versions of schemas
 **/dataset_**.json
@@ -16,4 +17,4 @@ credentials.py
 tmp/
 
 # vscode debugger
-.vscode/
+.vscode/

LICENSE.md

@@ -0,0 +1,21 @@
+## Public Domain
+
+This project constitutes a work of the United States Government and is not subject to domestic copyright protection under 17 USC § 105. Additionally, we waive copyright and related rights in the work worldwide through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
+
+All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. See [CONTRIBUTING](CONTRIBUTING.md) for more information. 
+
+## GNU General Public License
+
+This project utilizes code [licensed under the terms of the GNU General Public License](https://wordpress.org/about/gpl/) and therefore is licensed under GPL v2 or later.
+
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+Visit http://www.gnu.org/licenses/ to learn more about the GNU General Public License.
+
+## Other Information
+
+In no way are the patent or trademark rights of any person affected by CC0, nor are the rights that other persons may have in the work or in how the work is used, such as publicity or privacy rights.
+
+Unless expressly stated otherwise, the person who associated a work with this deed makes no warranties about the work, and disclaims liability for all uses of the work, to the fullest extent permitted by applicable law. When using or citing the work, you should not imply endorsement by the author or the affirmer.

Makefile

@@ -0,0 +1,26 @@
+pypi-upload: build-dist  ## Uploads new package to PyPi after clean, build
+	poetry publish
+
+# pypi-upload-test: build-dist  ## Uploads new package to TEST PyPi after clean, build
+# 	twine upload -r testpypi dist/*	
+
+build-dist: clean-dist  ## Builds new package dist
+	poetry build --verbose
+
+clean-dist:  ## Cleans dist dir
+	rm -rf dist/*
+
+test: up ## Runs poetry tests, ignores ckan load
+	poetry run pytest --ignore=./tests/load/ckan
+
+up: ## Sets up local docker environment
+	docker compose up -d
+
+lint:  ## Lints wtih ruff
+	ruff .
+
+# Output documentation for top-level targets
+# Thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+.PHONY: help
+help: ## This help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-10s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

harvester/__init__.py

@@ -1,3 +1,16 @@
+# TODO: maybe turn off this ruff ignore?
+# ruff: noqa: F405, F403
+
+__all__ = ["compare", "extract", "load", "transform", "validate", "utils"]
+
+# TODO these imports will need to be updated to ensure a consistent api
+from .compare import compare
+from .extract import extract
+from .load import load
+from .transform import transform
+from .utils import *
+from .validate import *
+
 # configuration settings
 bucket_name = "test-bucket"
 content_types = {

harvester/compare.py

@@ -0,0 +1,11 @@
+import logging
+
+logger = logging.getLogger("harvester")
+
+
+# stub, TODO complete
+def compare(compare_obj):
+    """Compares records"""
+    logger.info("Hello from harvester.compare()")
+
+    return compare_obj

harvester/extract/dcatus.py → harvester/extract.py

@@ -1,5 +1,9 @@
+import logging
+
 import requests
-from requests.exceptions import RequestException, JSONDecodeError
+from requests.exceptions import JSONDecodeError, RequestException
+
+logger = logging.getLogger("harvester")
 
 
 def download_dcatus_catalog(url):
@@ -20,3 +24,14 @@ def download_dcatus_catalog(url):
         return resp.json()
     except JSONDecodeError as e:
         return Exception(e)
+
+
+def extract(harvest_source) -> list:
+    """Extracts all records from a harvest_source"""
+    logger.info("Hello from harvester.extract()")
+
+    datasets = []
+
+    # stub
+
+    return datasets

harvester/load/ckan.py → harvester/load.py

@@ -1,6 +1,19 @@
-import ckanapi
+import logging
 import re
 
+import ckanapi
+
+logger = logging.getLogger("harvester")
+
+
+def load():
+    """ """
+    logger.info("Hello from harvester.load()")
+
+    # stub
+
+    pass
+
 
 def create_ckan_extra_base(*args):
     keys = ["publisher_hierarchy", "resource-type", "publisher"]

harvester/transform.py

@@ -0,0 +1,11 @@
+import logging
+
+logger = logging.getLogger("harvester")
+
+
+# stub, TODO complete
+def transform(transform_obj):
+    """Transforms records"""
+    logger.info("Hello from harvester.transform()")
+
+    return transform_obj

harvester/utils/__init__.py

@@ -0,0 +1,3 @@
+from . import json
+
+__all__ = ["json"]

harvester/validate/__init__.py

@@ -0,0 +1,3 @@
+from . import dcat_us
+
+__all__ = ["dcat_us"]