Test

.github/.github/pull_request_template.md

@@ -1,7 +1,7 @@
 <!-- We welcome contributions. All patches must include a sign-off. Please see CONTRIBUTING.md for details -->
 
 
-## What type of PR is this? 
+## What type of PR is this?
 <!-- Check all that apply, delete what doesn't apply. -->
 
 - [ ] Refactor
@@ -13,8 +13,8 @@
 
 ## How is this tested?
 
-- [ ] Unit tests 
-- [ ] E2E Tests 
+- [ ] Unit tests
+- [ ] E2E Tests
 - [ ] Manually
 - [ ] N/A
 

.github/CODEOWNERS

@@ -2,4 +2,4 @@
 # the repo. Unless a later match takes precedence, these
 # users will be requested for review when someone opens a
 # pull request.
-*       @susodapop @arikfr @yunbodeng-db @andrefurlan-db
+*       @rcypher-databricks @yunbodeng-db @andrefurlan-db @jackyhu-db @benc-db @kravets-levko

.github/workflows/code-quality-checks.yml

@@ -1,5 +1,5 @@
 name: Code Quality Checks
-on: 
+on:
   push:
     branches:
       - main
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: [3.9, "3.10", "3.11", "3.12"]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -58,11 +58,62 @@ jobs:
       #----------------------------------------------
       - name: Run tests
         run: poetry run python -m pytest tests/unit
+  run-unit-tests-with-arrow:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9, "3.10", "3.11", "3.12"]
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      -   name: Check out repository
+          uses: actions/checkout@v2
+      -   name: Set up python ${{ matrix.python-version }}
+          id: setup-python
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python-version }}
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      -   name: Install Poetry
+          uses: snok/install-poetry@v1
+          with:
+            virtualenvs-create: true
+            virtualenvs-in-project: true
+            installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      -   name: Load cached venv
+          id: cached-poetry-dependencies
+          uses: actions/cache@v2
+          with:
+            path: .venv-pyarrow
+            key: venv-pyarrow-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      -   name: Install dependencies
+          if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+          run: poetry install --no-interaction --no-root
+      #----------------------------------------------
+      # install your root project, if required
+      #----------------------------------------------
+      -   name: Install library
+          run: poetry install --no-interaction --all-extras
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      -   name: Run tests
+          run: poetry run python -m pytest tests/unit
   check-linting:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.9, "3.10", "3.11", "3.12"]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -114,7 +165,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.9, "3.10", "3.11", "3.12"]
     steps:
       #----------------------------------------------
       #       check-out repo and set-up python
@@ -157,7 +208,9 @@ jobs:
       - name: Install library
         run: poetry install --no-interaction
       #----------------------------------------------
-      #              black the code
+      #              mypy the code
       #----------------------------------------------
       - name: Mypy
-        run: poetry run mypy --install-types --non-interactive src
+        run: |
+          mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153
+          poetry run mypy --install-types --non-interactive src

.github/workflows/integration.yml

@@ -0,0 +1,57 @@
+name: Integration Tests
+on:
+  push:
+    paths-ignore:
+      - "**.MD"
+      - "**.md"
+
+jobs:
+  run-e2e-tests:
+    runs-on: ubuntu-latest
+    environment: azure-prod
+    env:
+      DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
+      DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }}
+      DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
+      DATABRICKS_CATALOG: peco
+      DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
+    steps:
+      #----------------------------------------------
+      #       check-out repo and set-up python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      #----------------------------------------------
+      #  -----  install & configure poetry  -----
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      #----------------------------------------------
+      #       load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # install dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install dependencies
+        run: poetry install --no-interaction --all-extras
+      #----------------------------------------------
+      #              run test suite
+      #----------------------------------------------
+      - name: Run e2e tests
+        run: poetry run python -m pytest tests/e2e

.github/workflows/publish-manual.yml

@@ -0,0 +1,78 @@
+name: Publish to PyPI Manual [Production]
+
+# Allow manual triggering of the workflow
+on:
+  workflow_dispatch: {}
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+
+    steps:
+      #----------------------------------------------
+      # Step 1: Check out the repository code
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v2  # Check out the repository to access the code
+
+      #----------------------------------------------
+      # Step 2: Set up Python environment
+      #----------------------------------------------
+      - name: Set up python
+        id: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9  # Specify the Python version to be used
+
+      #----------------------------------------------
+      # Step 3: Install and configure Poetry
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1  # Install Poetry, the Python package manager
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+#      #----------------------------------------------
+#      # Step 4: Load cached virtual environment (if available)
+#      #----------------------------------------------
+#      - name: Load cached venv
+#        id: cached-poetry-dependencies
+#        uses: actions/cache@v2
+#        with:
+#          path: .venv  # Path to the virtual environment
+#          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+#          # Cache key is generated based on OS, Python version, repo name, and the `poetry.lock` file hash
+
+#      #----------------------------------------------
+#      # Step 5: Install dependencies if the cache is not found
+#      #----------------------------------------------
+#      - name: Install dependencies
+#        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'  # Only run if the cache was not hit
+#        run: poetry install --no-interaction --no-root  # Install dependencies without interaction
+
+#      #----------------------------------------------
+#      # Step 6: Update the version to the manually provided version
+#      #----------------------------------------------
+#      - name: Update pyproject.toml with the specified version
+#        run: poetry version ${{ github.event.inputs.version }}  # Use the version provided by the user input
+
+      #----------------------------------------------
+      # Step 7: Build and publish the first package to PyPI
+      #----------------------------------------------
+      - name: Build and publish databricks sql connector to PyPI
+        working-directory: ./databricks_sql_connector
+        run: |
+          poetry build
+          poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }}  # Publish with PyPI token
+      #----------------------------------------------
+      # Step 7: Build and publish the second package to PyPI
+      #----------------------------------------------
+
+      - name: Build and publish databricks sql connector core to PyPI
+        working-directory: ./databricks_sql_connector_core
+        run: |
+          poetry build
+          poetry publish -u __token__ -p ${{ secrets.PROD_PYPI_TOKEN }}  # Publish with PyPI token

.github/workflows/publish.yml

@@ -61,4 +61,4 @@ jobs:
       - name: Build and publish to pypi
         uses: JRubics/poetry-publish@v1.10
         with:
-          pypi_token: ${{ secrets.PROD_PYPI_TOKEN }}
+          pypi_token: ${{ secrets.PROD_PYPI_TOKEN }}

.gitignore

@@ -195,7 +195,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # End of https://www.toptal.com/developers/gitignore/api/python,macos
 
@@ -204,4 +204,7 @@ dist/
 build/
 
 # vs code stuff
-.vscode
+.vscode
+
+# don't commit authentication info to source control
+test.env