From cec2897c2997b05bbae2f4bb0830e971571657d0 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 16 Sep 2024 10:00:40 -0700 Subject: [PATCH 1/4] use github as src --- api/poetry.lock | 17 ++++++++--------- api/pyproject.toml | 6 ++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 3c6955c30..57d471236 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -543,16 +543,15 @@ name = "cffdrs" version = "0.1.2" description = "" optional = false -python-versions = ">=3.10,<4.0" -files = [ - {file = "cffdrs-0.1.2-py3-none-any.whl", hash = "sha256:a124e09bb963670bc22cfa25de5e8d4b63f582b24bdeffd4f413afe8da7a8507"}, - {file = "cffdrs-0.1.2.tar.gz", hash = "sha256:d655ad3b2648ea2200d093c17d2098a1f49fc26617b50a33157baa38dcf8079b"}, -] +python-versions = ">=3.7" +files = [] +develop = false [package.source] -type = "legacy" -url = "https://artifacts.developer.gov.bc.ca/artifactory/api/pypi/pe1e-gen-python-local/simple" -reference = "psu" +type = "git" +url = "https://github.com/cffdrs/cffdrs_py.git" +reference = "HEAD" +resolved_reference = "c7603073264906242a2c8d46ceecb01d1f539975" [[package]] name = "cffi" @@ -5917,4 +5916,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10.4,<3.11" -content-hash = "754870411c09e78230ba1750bb3749588f32cb5a8fbc89857095d817c4581286" +content-hash = "c2e7265b2a3e10e7a196ef394c3f62cdd65c722a776896ffb8297e189e054355" diff --git a/api/pyproject.toml b/api/pyproject.toml index 0d8ca3796..5c97b3847 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -42,9 +42,9 @@ scikit-learn = "^1.1.3" httpx = "^0.27.0" orjson = "^3.9.0" greenlet = "^3.0.0" -sentry-sdk = {extras = ["fastapi"], version = "^2.0.1"} +sentry-sdk = { extras = ["fastapi"], version = "^2.0.1" } numba = "^0.59.1" -cffdrs = {version = "^0.1.1", source = "psu"} +cffdrs = { git = "https://github.com/cffdrs/cffdrs_py.git" } geopandas = "^1.0.1" shapely = "^2.0.5" @@ -68,7 +68,6 @@ ruff = "^0.4.0" [[tool.poetry.source]] name = "psu" -url = "https://artifacts.developer.gov.bc.ca/artifactory/api/pypi/pe1e-gen-python-local/simple" priority = "supplemental" [build-system] @@ -105,4 +104,3 @@ per-file-ignores = { "alembic/versions/00df3c7b5cba_rethink_classification.py" = ] } line-length = 185 ignore = ["E712", "F401"] - From 1bd1a8aef6a0f503a920dab2cefab70b1b82fe04 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 16 Sep 2024 10:19:30 -0700 Subject: [PATCH 2/4] remove from pipeline --- .github/workflows/deployment.yml | 4 ++-- .github/workflows/integration.yml | 2 -- .github/workflows/post_merge_integration.yml | 9 --------- Dockerfile | 8 +------- openshift/scripts/oc_build.sh | 3 --- openshift/templates/build.bc.yaml | 13 ------------- openshift/templates/build.web.bc.yaml | 5 ----- 7 files changed, 3 insertions(+), 41 deletions(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 8fa0d2afa..44cad61c7 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -43,7 +43,7 @@ jobs: shell: bash run: | oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_TOOL_TOKEN }}" - GIT_BRANCH=${GITHUB_HEAD_REF} MODULE_NAME=web DOCKER_FILE=Dockerfile.web PATH_BC=openshift/templates/build.web.bc.yaml SENTRY_AUTH_TOKEN="${{ secrets.SENTRY_AUTH_TOKEN }}" ARTIFACTORY_SVCACCT_TOKEN="${{ secrets.ARTIFACTORY_SVCACCT_TOKEN}}" bash openshift/scripts/oc_build.sh ${SUFFIX} apply + GIT_BRANCH=${GITHUB_HEAD_REF} MODULE_NAME=web DOCKER_FILE=Dockerfile.web PATH_BC=openshift/templates/build.web.bc.yaml SENTRY_AUTH_TOKEN="${{ secrets.SENTRY_AUTH_TOKEN }}" bash openshift/scripts/oc_build.sh ${SUFFIX} apply build-api-image: name: Build API Image @@ -61,7 +61,7 @@ jobs: shell: bash run: | oc login "${{ secrets.OPENSHIFT_CLUSTER }}" --token="${{ secrets.OC4_TOOL_TOKEN }}" - GIT_BRANCH=${GITHUB_HEAD_REF} MODULE_NAME=api ARTIFACTORY_PYPI_USERNAME=${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ARTIFACTORY_PYPI_PASSWORD=${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} bash openshift/scripts/oc_build.sh ${SUFFIX} apply + GIT_BRANCH=${GITHUB_HEAD_REF} MODULE_NAME=api bash openshift/scripts/oc_build.sh ${SUFFIX} apply # TODO: Delete once pmtiles has run for some time # build-tileserv-image: # name: Build tileserv Image diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index bcd65ab96..2f93c7212 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -63,7 +63,6 @@ jobs: if: steps.cache-venv.outputs.cache-hit != 'true' working-directory: ./api run: | - poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} poetry run python -m pip install --upgrade pip poetry install poetry run python -m pip install gdal==$(gdal-config --version) @@ -139,7 +138,6 @@ jobs: if: steps.cache-venv.outputs.cache-hit != 'true' working-directory: ./api run: | - poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} poetry run python -m pip install --upgrade pip poetry install poetry run python -m pip install gdal==$(gdal-config --version) diff --git a/.github/workflows/post_merge_integration.yml b/.github/workflows/post_merge_integration.yml index 0773101ea..7c097466b 100644 --- a/.github/workflows/post_merge_integration.yml +++ b/.github/workflows/post_merge_integration.yml @@ -60,9 +60,6 @@ jobs: with: path: ~/work/wps/wps/api/.venv key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }} - - name: Configure artifactory creds for poetry - working-directory: ./api - run: poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} - name: Install python dependencies using poetry (api) if: steps.cache-venv.outputs.cache-hit != 'true' working-directory: ./api @@ -138,9 +135,6 @@ jobs: with: path: ~/work/wps/wps/api/.venv key: ${{ runner.os }}-venv-poetry-1.6.1-${{ hashFiles('**/poetry.lock') }} - - name: Configure artifactory creds for poetry - working-directory: ./api - run: poetry config http-basic.psu ${{ secrets.ARTIFACTORY_PYPI_USERNAME }} ${{ secrets.ARTIFACTORY_PYPI_PASSWORD }} - name: Install python dependencies using poetry (api) if: steps.cache-venv.outputs.cache-hit != 'true' working-directory: ./api @@ -206,9 +200,6 @@ jobs: run: corepack enable - name: use new yarn run: yarn set version berry - - name: Setup yarn for scoped artifactory packages - working-directory: ./web - run: yarn config set npmScopes.psu.npmRegistryServer https://artifacts.developer.gov.bc.ca/artifactory/api/npm/pe1e-psu-npm-local/ && yarn config set npmScopes.psu.npmAlwaysAuth true && yarn config set npmScopes.psu.npmAuthToken ${{ secrets.ARTIFACTORY_SVCACCT_TOKEN }} - name: Install node dependencies (web) working-directory: ./web if: steps.yarn-cache.outputs.cache-hit != 'true' diff --git a/Dockerfile b/Dockerfile index 30ead8b4b..14cc9b194 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,7 @@ ARG DOCKER_IMAGE=image-registry.openshift-image-registry.svc:5000/e1e498-tools/w # To build locally, point to a local base image you've already built (see openshift/wps-api-base) # e.g. : docker build --build-arg DOCKER_IMAGE=wps-api-base:my-tag . -# Stage 1: Install Python packages, including internally published cffdrs. Installation from artifactory -# requires a username/password which we don't want in our final image, so we use a multi-stage build. +# Stage 1: Install Python packages FROM ${DOCKER_IMAGE} AS builder # We don't want to run our app as root, so we define a worker user. @@ -29,11 +28,6 @@ RUN python -m pip install --upgrade pip # Copy poetry files. COPY --chown=$USERNAME:$USERNAME ./api/pyproject.toml ./api/poetry.lock /app/ -ARG ARTIFACTORY_PYPI_USERNAME -ARG ARTIFACTORY_PYPI_PASSWORD - -RUN poetry config http-basic.psu "$ARTIFACTORY_PYPI_USERNAME" "$ARTIFACTORY_PYPI_PASSWORD" - # Install dependencies. RUN poetry install --without dev # Get a python binding for gdal that matches the version of gdal we have installed. diff --git a/openshift/scripts/oc_build.sh b/openshift/scripts/oc_build.sh index 7ae735ba3..627502448 100755 --- a/openshift/scripts/oc_build.sh +++ b/openshift/scripts/oc_build.sh @@ -30,9 +30,6 @@ OC_PROCESS="oc -n ${PROJ_TOOLS} process -f ${PATH_BC} \ -p SUFFIX=${SUFFIX} \ -p GIT_BRANCH=${GIT_BRANCH} \ ${SENTRY_AUTH_TOKEN:+ "-p SENTRY_AUTH_TOKEN=${SENTRY_AUTH_TOKEN}"} \ - ${ARTIFACTORY_SVCACCT_TOKEN:+ "-p ARTIFACTORY_SVCACCT_TOKEN=${ARTIFACTORY_SVCACCT_TOKEN}"} \ - ${ARTIFACTORY_PYPI_USERNAME:+ "-p ARTIFACTORY_PYPI_USERNAME=${ARTIFACTORY_PYPI_USERNAME}"} \ - ${ARTIFACTORY_PYPI_PASSWORD:+ "-p ARTIFACTORY_PYPI_PASSWORD=${ARTIFACTORY_PYPI_PASSWORD}"} \ ${DOCKER_IMAGE:+ "-p DOCKER_IMAGE=${DOCKER_IMAGE}"} \ ${DOCKER_FILE:+ "-p DOCKER_FILE=${DOCKER_FILE}"}" diff --git a/openshift/templates/build.bc.yaml b/openshift/templates/build.bc.yaml index 10f54d9e0..a7a347a35 100644 --- a/openshift/templates/build.bc.yaml +++ b/openshift/templates/build.bc.yaml @@ -37,12 +37,6 @@ parameters: description: Dockerfile to use required: true value: Dockerfile - - name: ARTIFACTORY_PYPI_USERNAME - description: Username for internal pypi artifactory instance - required: true - - name: ARTIFACTORY_PYPI_PASSWORD - description: Password for internal pypi artifactory instance - required: true objects: - apiVersion: v1 kind: ImageStream @@ -86,12 +80,5 @@ objects: contextDir: ./ strategy: dockerStrategy: - buildArgs: - - name: "DOCKER_IMAGE" - value: "${{DOCKER_IMAGE}}" - - name: "ARTIFACTORY_PYPI_USERNAME" - value: "${{ARTIFACTORY_PYPI_USERNAME}}" - - name: "ARTIFACTORY_PYPI_PASSWORD" - value: "${{ARTIFACTORY_PYPI_PASSWORD}}" dockerfilePath: ${DOCKER_FILE} triggers: [] diff --git a/openshift/templates/build.web.bc.yaml b/openshift/templates/build.web.bc.yaml index 7bbe19dc9..f8a91308b 100644 --- a/openshift/templates/build.web.bc.yaml +++ b/openshift/templates/build.web.bc.yaml @@ -36,9 +36,6 @@ parameters: - name: SENTRY_AUTH_TOKEN description: Sentry auth token for uploading source maps required: true - - name: ARTIFACTORY_SVCACCT_TOKEN - description: Arctifactory service account token - required: true objects: - apiVersion: v1 kind: ImageStream @@ -85,7 +82,5 @@ objects: env: - name: "SENTRY_AUTH_TOKEN" value: ${SENTRY_AUTH_TOKEN} - - name: "ARTIFACTORY_SVCACCT_TOKEN" - value: ${ARTIFACTORY_SVCACCT_TOKEN} dockerfilePath: ${DOCKER_FILE} triggers: [] From 126c6fb8123fe52f7dc8e844228bfada26d060ee Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 16 Sep 2024 10:29:59 -0700 Subject: [PATCH 3/4] remove source --- api/pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/pyproject.toml b/api/pyproject.toml index 5c97b3847..4412b9049 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -65,11 +65,6 @@ pytest-watch = "^4.2.0" pytest-testmon = "^2.0.0" ruff = "^0.4.0" - -[[tool.poetry.source]] -name = "psu" -priority = "supplemental" - [build-system] requires = ["poetry>=1.1.11"] build-backend = "poetry.masonry.api" From da2e72a51957614f1be4c755e52e307ff701cfa5 Mon Sep 17 00:00:00 2001 From: Brett Edwards Date: Mon, 16 Sep 2024 10:37:00 -0700 Subject: [PATCH 4/4] use revision --- api/poetry.lock | 4 ++-- api/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/poetry.lock b/api/poetry.lock index 57d471236..364097cda 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -550,7 +550,7 @@ develop = false [package.source] type = "git" url = "https://github.com/cffdrs/cffdrs_py.git" -reference = "HEAD" +reference = "c760307" resolved_reference = "c7603073264906242a2c8d46ceecb01d1f539975" [[package]] @@ -5916,4 +5916,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.10.4,<3.11" -content-hash = "c2e7265b2a3e10e7a196ef394c3f62cdd65c722a776896ffb8297e189e054355" +content-hash = "660a4ee280e9d284b527f1d2a6e63dbe8de1aeaa0d81417e90e672b773d08ee2" diff --git a/api/pyproject.toml b/api/pyproject.toml index 4412b9049..6ac2f89be 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -44,7 +44,7 @@ orjson = "^3.9.0" greenlet = "^3.0.0" sentry-sdk = { extras = ["fastapi"], version = "^2.0.1" } numba = "^0.59.1" -cffdrs = { git = "https://github.com/cffdrs/cffdrs_py.git" } +cffdrs = {git = "https://github.com/cffdrs/cffdrs_py.git", rev = "c760307"} geopandas = "^1.0.1" shapely = "^2.0.5"