release python package

.devcontainer/requirements.txt

@@ -7,4 +7,5 @@ pytest==7.1.3
 pyarrow==9.0.0
 nanoid==0.1
 requests==2.28.1
-python-dotenv==0.21.0
+python-dotenv==0.21.0
+toml==0.10.2

.github/workflows/python-release.yaml

@@ -0,0 +1,62 @@
+name: Dataplane Python Package Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  worker:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+
+      - name: Release check out ${{ steps.get_version.outputs.VERSION }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ steps.get_version.outputs.VERSION }}
+
+      - name: Update pyproject.toml version ${{ steps.get_version.outputs.VERSION }}
+        run: python3 update_toml_release.py ${{ steps.get_version.outputs.VERSION }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python3 -m pip install --upgrade twine
+          if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
+
+      - name: Build python package
+        run: python3 -m build
+
+      - name: Distribute python package
+        run: python3 -m twine upload dist/* -u "__token__" -p ${{ secrets.PYPI_API_TOKEN }}
+
+      - name: Commit report
+        continue-on-error: true
+        run: |
+          git config --global user.name 'Dataplane Actions'
+          git config --global user.email 'saul-data@users.noreply.github.com'
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
+          git checkout "${GITHUB_REF:11}"
+          git add pyproject.toml
+          git commit -am "Automated TOML release update"
+          git push
+          # git diff --exit-code || git commit -am "Automated TOML release update"
+          # git diff --exit-code || git push

.github/workflows/python-tests.yaml

@@ -32,6 +32,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           if [ -f tests/requirements.txt ]; then pip install -r tests/requirements.txt; fi
+          
       - name: Test with pytest
         run: |
           pytest -s

notes-distribute.md

@@ -1,3 +1,11 @@
+### Github Actions Release
+1. Update release with latest version
+2. Github Actions will release to PyPI with that version
+3. Automatically update toml and git commit to main
+```shell
+python update_toml_release.py 0.0.15
+```
+
 ### Steps to distribute
 
 1. Update toml with latest version

pyproject.toml

@@ -1,26 +1,23 @@
 [build-system]
-requires = ["setuptools>=61.0","redis>=1.0.0","pyarrow>1.0.0", "pandas>1.0.0", "requests>1.0.0"]
+requires = [ "setuptools>=61.0", "redis>=1.0.0", "pyarrow>1.0.0", "pandas>1.0.0", "requests>1.0.0",]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataplane"
 version = "0.0.14"
-authors = [
-  { name="Saul Frank" }, {name ="Dataplane, Inc."}
-]
 description = "The data engineering library to build robust, reliable and on time data pipelines in Python."
 readme = "README.md"
 requires-python = ">=3.7"
-classifiers = [
-    "Programming Language :: Python :: 3",
-    "License :: OSI Approved :: BSD License",
-    "Operating System :: OS Independent",
-]
+classifiers = [ "Programming Language :: Python :: 3", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent",]
+keywords = [ "workflow", "data-science", "data", "airflow", "etl", "dataplane", "data-engineering", "data-pipelines", "datawarehouse", "automation", "data-analysis", "scheduler",]
+[[project.authors]]
+name = "Saul Frank"
 
-keywords = ["workflow", "data-science", "data", "airflow", "etl", "dataplane", "data-engineering", "data-pipelines", "datawarehouse", "automation", "data-analysis", "scheduler"]
+[[project.authors]]
+name = "Dataplane, Inc."
 
 [project.urls]
-"Homepage" = "https://dataplane.app"
-"Github" = "https://github.com/dataplane-app/dataplane-python-package"
-"Issues" = "https://github.com/dataplane-app/dataplane/issues"
-"Feedback" = "https://github.com/dataplane-app/dataplane/discussions"
+Homepage = "https://dataplane.app"
+Github = "https://github.com/dataplane-app/dataplane-python-package"
+Issues = "https://github.com/dataplane-app/dataplane/issues"
+Feedback = "https://github.com/dataplane-app/dataplane/discussions"

tests/requirements.txt

@@ -7,4 +7,5 @@ pytest==7.1.3
 pyarrow==9.0.0
 nanoid==0.1
 requests==2.28.1
-python-dotenv==0.21.0
+python-dotenv==0.21.0
+toml==0.10.2

update_toml_release.py

@@ -0,0 +1,16 @@
+import toml
+import sys
+
+data = toml.load("pyproject.toml") 
+
+# Modify field
+version = sys.argv[1]
+version = version.replace("v", "")
+data['project']['version']=version
+
+# To use the dump function, you need to open the file in 'write' mode
+# It did not work if I just specify file location like in load
+f = open("pyproject.toml",'w')
+toml.dump(data, f)
+f.close()
+print(f"Dataplane python project updated to {version}")