Merge branch 'main' into rm-cryptodome-oscrypto

snowflakedb · Sep 26, 2023 · 8f15398 · 8f15398
2 parents 6910dc5 + ad3e295
commit 8f15398
Show file tree

Hide file tree

Showing 83 changed files with 2,511 additions and 731 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
@@ -80,11 +80,7 @@ jobs:
             id: macosx_x86_64
           - image: macos-latest
             id: macosx_arm64
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
-        exclude:
-          - os:
-              id: macosx_arm64
-            python-version: 3.7
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }}
     runs-on: ${{ matrix.os.image }}
     steps:
@@ -128,7 +124,7 @@ jobs:
            download_name: macosx_x86_64
          - image_name: windows-2019
            download_name: win_amd64
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         cloud-provider: [aws, azure, gcp]
     steps:
       - uses: actions/checkout@v3
@@ -185,7 +181,7 @@ jobs:
         os:
          - image_name: ubuntu-latest
            download_name: linux
-        python-version: [3.7]
+        python-version: [3.8]
         cloud-provider: [aws]
     steps:
       - uses: actions/checkout@v3
@@ -215,7 +211,7 @@ jobs:
         shell: bash
 
   test-fips:
-    name: Test FIPS linux-3.7-${{ matrix.cloud-provider }}
+    name: Test FIPS linux-3.8-${{ matrix.cloud-provider }}
     needs: build
     runs-on: ubuntu-latest
     strategy:
@@ -234,30 +230,76 @@ jobs:
       - name: Download wheel(s)
         uses: actions/download-artifact@v3
         with:
-          name: manylinux_x86_64_py3.7
+          name: manylinux_x86_64_py3.8
           path: dist
       - name: Show wheels downloaded
         run: ls -lh dist
         shell: bash
       - name: Run tests
         run: ./ci/test_fips_docker.sh
         env:
-          PYTHON_VERSION: 3.7
+          PYTHON_VERSION: 3.8
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
       - uses: actions/upload-artifact@v3
         with:
-          name: coverage_linux-fips-3.7-${{ matrix.cloud-provider }}
+          name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }}
           path: |
             .coverage
             coverage.xml
 
+  test-lambda:
+    name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        cloud-provider: [aws]
+    steps:
+      - name: Set shortver
+        run: echo "shortver=${longver//./}" >> $GITHUB_ENV
+        env:
+          longver: ${{ matrix.python-version }}
+        shell: bash
+      - uses: actions/checkout@v3
+      - name: Setup parameters file
+        shell: bash
+        env:
+          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
+        run: |
+          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
+          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
+      - name: Download wheel(s)
+        uses: actions/download-artifact@v3
+        with:
+          name: manylinux_x86_64_py${{ matrix.python-version }}
+          path: dist
+      - name: Show wheels downloaded
+        run: ls -lh dist
+        shell: bash
+      - name: Run tests
+        run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION}
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
+      - uses: actions/upload-artifact@v3
+        with:
+          name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          path: |
+            .coverage.py${{ env.shortver }}-lambda-ci
+            junit.py${{ env.shortver }}-lambda-ci-dev.xml
+
   combine-coverage:
     if: ${{ success() || failure() }}
     name: Combine coverage
-    needs: [lint, test, test-fips]
+    needs: [lint, test, test-fips, test-lambda]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -267,7 +309,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
       - name: Upgrade setuptools and pip

diff --git a/.github/workflows/cla_bot.yml b/.github/workflows/cla_bot.yml
@@ -22,7 +22,7 @@ jobs:
           PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
         with:
           path-to-signatures: 'signatures/version1.json'
-          path-to-document: 'https://github.com/Snowflake-Labs/CLA/blob/main/README.md'
+          path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
           branch: 'main'
           allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
           remote-organization-name: 'snowflakedb'

diff --git a/.github/workflows/create_req_files.yml b/.github/workflows/create_req_files.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
     -   id: debug-statements
     -   id: check-ast
 -   repo: https://github.com/Lucas-C/pre-commit-hooks.git
-    rev: v1.4.2
+    rev: v1.5.1
     hooks:
     -   id: insert-license
         name: insert-py-license
@@ -41,7 +41,7 @@ repos:
             - --license-filepath
             - license_header.txt
 -   repo: https://github.com/asottile/yesqa
-    rev: v1.4.0
+    rev: v1.5.0
     hooks:
     -   id: yesqa
 -   repo: https://github.com/mgedmin/check-manifest
@@ -60,18 +60,18 @@ repos:
               - --append-only
           files: ^src/snowflake/connector/.*\.py$
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.3.1
+    rev: v3.7.0
     hooks:
         - id: pyupgrade
-          args: [--py37-plus]
+          args: [--py38-plus]
 -   repo: https://github.com/PyCQA/flake8
     rev: 6.0.0
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-bugbear
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.991'
+    rev: 'v1.4.0'
     hooks:
     -   id: mypy
         files: |
@@ -104,7 +104,7 @@ repos:
             - types-pyOpenSSL
             - types-setuptools
 -   repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.3.0
     hooks:
         - id: black
           args:

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -18,6 +18,10 @@ Please [open a new issue](https://github.com/snowflakedb/snowflake-connector-pyt
 We encourage everyone to first open an issue to discuss any feature work or bug fixes with one of the maintainers.
 This should help guide contributors through potential pitfalls.
 
+## Contributor License Agreement ("CLA")
+
+We require our contributors to sign a CLA, available at https://github.com/snowflakedb/CLA/blob/main/README.md. A Github Actions bot will assist you when you open a pull request.
+
 ### Setup a development environment
 
 What is a development environment? It's a [virtualenv](https://virtualenv.pypa.io) that has all of necessary

diff --git a/DESCRIPTION.md b/DESCRIPTION.md
@@ -8,13 +8,65 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne
 
 # Release Notes
 
-- v3.0.5(TBD)
-  - Added the ability to read Snowflake's central configuration file.
+- v3.2.1(TBD)
+
+  - Fixed a bug where url port and path were ignore in private link oscp retry.
+  - Added thread safety in telemetry when instantiating multiple connections concurrently.
+  - Bumped platformdirs dependency from >=2.6.0,<3.9.0 to >=2.6.0,<4.0.0.0 and made necessary changes to allow this.
+  - Removed the deprecation warning from the vendored urllib3 about urllib3.contrib.pyopenssl deprecation.
+  - Remove dependencies on Cryptodome and oscrypto and remove the `use_openssl_only` parameter. All connections now go through OpenSSL via the cryptography library, which was already a dependency.
+
+- v3.2.0(September 06,2023)
+
+  - Made the ``parser`` -> ``manager`` renaming more consistent in ``snowflake.connector.config_manager`` module.
+  - Added support for default values for ConfigOptions
+  - Added default_connection_name to config.toml file
+
+- v3.1.1(August 28,2023)
+
+  - Fixed a bug in retry logic for okta authentication to refresh token.
+  - Support `RSAPublicKey` when constructing `AuthByKeyPair` in addition to raw bytes.
+  - Fixed a bug when connecting through SOCKS5 proxy, the attribute `proxy_header` is missing on `SOCKSProxyManager`.
+  - Cherry-picked https://github.com/urllib3/urllib3/commit/fd2759aa16b12b33298900c77d29b3813c6582de onto vendored urllib3 (v1.26.15) to enable enforce_content_length by default.
+  - Fixed a bug in tag generation of OOB telemetry event.
+
+- v3.1.0(July 31,2023)
+
+  - Added a feature that lets you add connection definitions to the `connections.toml` configuration file. A connection definition refers to a collection of connection parameters, for example, if you wanted to define a connection named `prod``:
+
+    ```toml
+    [prod]
+    account = "my_account"
+    user = "my_user"
+    password = "my_password"
+    ```
+    By default, we look for the `connections.toml` file in the location specified in the `SNOWFLAKE_HOME` environment variable (default: `~/.snowflake`). If this folder does not exist, the Python connector looks for the file in the [platformdirs](https://github.com/platformdirs/platformdirs/blob/main/README.rst) location, as follows:
+
+    - On Linux: `~/.config/snowflake/`,  but follows XDG settings
+    - On Mac: `~/Library/Application Support/snowflake/`
+    - On Windows: `%USERPROFILE%\AppData\Local\snowflake\`
+
+    You can determine which file is used by running the following command:
+
+    ```
+    python -c "from snowflake.connector.constants import CONNECTIONS_FILE; print(str(CONNECTIONS_FILE))"
+    ```
   - Bumped cryptography dependency from <41.0.0,>=3.1.0 to >=3.1.0,<42.0.0.
   - Improved OCSP response caching to remove tmp cache files on Windows.
+  - Improved OCSP response caching to reduce the times of disk writing.
   - Added a parameter `server_session_keep_alive` in `SnowflakeConnection` that skips session deletion when client connection closes.
   - Tightened our pinning of platformdirs, to prevent their new releases breaking us.
-  - Remove dependencies on Cryptodome and oscrypto and remove the `use_openssl_only` parameter. All connections now go through OpenSSL via the cryptography library, which was already a dependency.
+  - Fixed a bug where SFPlatformDirs would incorrectly append application_name/version to its path.
+  - Added retry reason for queries that are retried by the client.
+  - Fixed a bug where `write_pandas` fails when user does not have the privilege to create stage or file format in the target schema, but has the right privilege for the current schema.
+  - Remove Python 3.7 support.
+  - Worked around a segfault which sometimes occurred during cache serialization in multi-threaded scenarios.
+  - Improved error handling of connection reset error.
+  - Fixed a bug about deleting the temporary files happened when running PUT command.
+  - Allowed to pass `type_mapper` to `fetch_pandas_batches()` and `fetch_pandas_all()`.
+  - Fixed a bug where pickle.dump segfaults during cache serialization in multi-threaded scenarios.
+  - Improved retry logic for okta authentication to refresh token if authentication gets throttled.
+  - Note that this release does not include the changes introduced in the previous 3.1.0a1 release. Those will be released at a later time.
 
 - v3.0.4(May 23,2023)
   - Fixed a bug in which `cursor.execute()` could modify the argument statement_params dictionary object when executing a multistatement query.

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -47,12 +47,10 @@ timestamps {
           println("Exception computing commit hash from: ${response}")
           }
         parallel (
-          'Test Python 37': { build job: 'RT-PyConnector37-PC',parameters: params},
           'Test Python 38': { build job: 'RT-PyConnector38-PC',parameters: params},
           'Test Python 39': { build job: 'RT-PyConnector39-PC',parameters: params},
           'Test Python 310': { build job: 'RT-PyConnector310-PC',parameters: params},
           'Test Python 311': { build job: 'RT-PyConnector311-PC',parameters: params},
-          'Test Python Lambda 37': { build job: 'RT-PyConnector37-PC-Lambda',parameters: params}
           )
         }
       }

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ using the Snowflake JDBC or ODBC drivers.
 
 The connector has **no** dependencies on JDBC or ODBC.
 It can be installed using ``pip`` on Linux, Mac OSX, and Windows platforms
-where Python 3.7.0 (or higher) is installed.
+where Python 3.8.0 (or higher) is installed.
 
 Snowflake Documentation is available at:
 https://docs.snowflake.com/
@@ -27,7 +27,7 @@ https://community.snowflake.com/s/article/How-To-Submit-a-Support-Case-in-Snowfl
 
 ### Locally
 
-Install Python 3.7.0 or higher. Clone the Snowflake Connector for Python repository, then run the following commands
+Install Python 3.8.0 or higher. Clone the Snowflake Connector for Python repository, then run the following commands
 to create a wheel package using PEP-517 build:
 
 ```shell
@@ -42,7 +42,7 @@ Find the `snowflake_connector_python*.whl` package in the `./dist` directory.
 ### In Docker
 Or use our Dockerized build script `ci/build_docker.sh` and find the built wheel files in `dist/repaired_wheels`.
 
-Note: `ci/build_docker.sh` can be used to compile only certain versions, like this: `ci/build_docker.sh "3.7 3.8"`
+Note: `ci/build_docker.sh` can be used to compile only certain versions, like this: `ci/build_docker.sh "3.8 3.9"`
 
 ## Code hygiene and other utilities
 These tools are integrated into `tox` to allow us to easily set them up universally on any computer.

diff --git a/ci/build_darwin.sh b/ci/build_darwin.sh
@@ -2,12 +2,12 @@
 #
 # Build Snowflake Python Connector on Mac
 # NOTES:
-#   - To compile only a specific version(s) pass in versions like: `./build_darwin.sh "3.7 3.8"`
+#   - To compile only a specific version(s) pass in versions like: `./build_darwin.sh "3.8 3.9"`
 arch=$(uname -m)
 if [[ "$arch" == "arm64" ]]; then
   PYTHON_VERSIONS="${1:-3.8 3.9 3.10 3.11}"
 else
-  PYTHON_VERSIONS="${1:-3.7 3.8 3.9 3.10 3.11}"
+  PYTHON_VERSIONS="${1:-3.8 3.9 3.10 3.11}"
 fi
 
 THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

diff --git a/ci/build_docker.sh b/ci/build_docker.sh
@@ -2,7 +2,7 @@
 #
 # Build Snowflake Python Connector in Docker
 # NOTES:
-#   - To compile only a specific version(s) pass in versions like: `./build_docker.sh "3.7 3.8"`
+#   - To compile only a specific version(s) pass in versions like: `./build_docker.sh "3.8 3.9"`
 set -o pipefail
 
 THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

diff --git a/ci/build_linux.sh b/ci/build_linux.sh
@@ -3,11 +3,11 @@
 # Build Snowflake Python Connector on Linux
 # NOTES:
 #   - This is designed to ONLY be called in our build docker image
-#   - To compile only a specific version(s) pass in versions like: `./build_linux.sh "3.7 3.8"`
+#   - To compile only a specific version(s) pass in versions like: `./build_linux.sh "3.8 3.9"`
 set -o pipefail
 
 U_WIDTH=16
-PYTHON_VERSIONS="${1:-3.7 3.8 3.9 3.10 3.11}"
+PYTHON_VERSIONS="${1:-3.8 3.9 3.10 3.11}"
 THIS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 CONNECTOR_DIR="$(dirname "${THIS_DIR}")"
 DIST_DIR="${CONNECTOR_DIR}/dist"

diff --git a/ci/build_windows.bat b/ci/build_windows.bat
@@ -6,14 +6,14 @@
 SET SCRIPT_DIR=%~dp0
 SET CONNECTOR_DIR=%~dp0\..\
 
-set python_versions= 3.7 3.8 3.9 3.10 3.11
+set python_versions= 3.8 3.9 3.10 3.11
 
 cd %CONNECTOR_DIR%
 
 set venv_dir=%WORKSPACE%\venv-flake8
 if %errorlevel% neq 0 goto :error
 
-py -3.7 -m venv %venv_dir%
+py -3.8 -m venv %venv_dir%
 if %errorlevel% neq 0 goto :error
 
 call %venv_dir%\scripts\activate

diff --git a/ci/docker/connector_test_lambda/Dockerfile310 b/ci/docker/connector_test_lambda/Dockerfile310
@@ -0,0 +1,12 @@
+FROM public.ecr.aws/lambda/python:3.10-x86_64
+
+RUN yum install -y git
+
+WORKDIR /home/user/snowflake-connector-python
+RUN chmod 777 /home/user/snowflake-connector-python
+ENV PATH="${PATH}:/opt/python/cp310-cp310/bin/"
+ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"
+
+RUN pip3 install -U pip setuptools wheel tox tox-external_wheels
+
+CMD [ "app.handler" ]