From 667ab12c65230e2436ad549f8242a0f926b94da9 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Fri, 15 Jan 2021 19:05:09 -0800 Subject: [PATCH 01/12] revise helm chart * realized each app will need its own image pull secret * Added image pull secret to this chart * modified actions to grant service account access to image pull secret --- .github/workflows/deploy_dev.yaml | 35 +++++++-- .github/workflows/deploy_prod.yaml | 15 ++++ docs/CDCI-pipeline.md | 77 +++++-------------- docs/build-deploy-notes.md | 9 ++- helm-charts/templates/_helpers.tpl | 8 ++ .../templates/smk-app-imagepullsecret.yaml | 11 +++ helm-charts/values.yaml | 7 +- 7 files changed, 94 insertions(+), 68 deletions(-) create mode 100644 helm-charts/templates/smk-app-imagepullsecret.yaml diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index a36524c..6057e08 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -30,6 +30,7 @@ jobs: # could also reference the github url? - uses: actions/checkout@v2 id: checkout + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' with: fetch-depth: 0 @@ -118,6 +119,7 @@ jobs: DOCKER_VERSION_TAG=${{ steps.retrieveimagetag.outputs.DOCKER_VERSION_TAG }} IMAGE_REGISTRY=${{ steps.retrieveimagetag.outputs.IMAGE_REGISTRY }} REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} + IMAGE_PULL_SECRET_NAME=${REPONAME}-image-pull-secret # -- debug-- verify the variables have been retrieved and have the values expected echo EMAIL $EMAIL @@ -125,16 +127,29 @@ jobs: echo DOCKER_VERSION_TAG $DOCKER_VERSION_TAG echo IMAGE_REGISTRY $IMAGE_REGISTRY + # potential issue: + # - helm chart creates the image pull secret + # - tries to use it with the service account + # - service account hasn't been granted access + helm upgrade $REPONAME ./helm-charts --install \ --set app_name=$REPONAME \ --set env=dev \ - --set app_image_pull_secret_params.name=github-imagepull-secrets \ + --set app_image_pull_secret_params.email=$EMAIL \ + --set app_image_pull_secret_params.password=$GHCR_TOKEN \ + --set app_image_pull_secret_params.name=$IMAGE_PULL_SECRET_NAME \ --set app_image_pull_secret_params.imagetag=$DOCKER_VERSION_TAG \ --set app_image_pull_secret_params.registry=$IMAGE_REGISTRY + # finally need to grant the service account access to the the + # image_pull_secret that was just created + serviceAccountName=$(oc whoami | cut -d':' -f4) + oc patch serviceaccount $serviceAccountName -p '{"imagePullSecrets": [{"name": "'$IMAGE_PULL_SECRET_NAME'"}]}' + # # ADD THE ROUTE TO THE ISSUE - name: Update issue id: updateissue + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' && env.GHCR_TOKEN != '' run: | # populate env vars used by this step REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} @@ -176,7 +191,7 @@ jobs: - name: Get GWA CLI id: getGWA - if: env.RUN_GWA == 'true' + if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | # disabled atm GWA_VERSION=v1.1.2 @@ -187,7 +202,7 @@ jobs: - name: Configure Kong Route id: kongconf - if: env.RUN_GWA == 'true' + if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | #------------------------------------------------------------------------ # commenting this whole section out, current gwa config may work but @@ -237,28 +252,32 @@ jobs: - name: "Send Review Request 2" id: requestreview + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' && env.GHCR_TOKEN != '' run: | # get the openshift dev namespace name OCDEVNAMESPACE=$(oc get configmap $OC_NAMESPACE_NAMES_CONFIG_MAP -o json | jq .data.dev | tr -d '"') REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} CONFIGMAPNAME=$REPONAME-gh-issues-cm EVENT_INFO=$(oc get configmap $CONFIGMAPNAME -o json | jq '.data.github_event_json' | jq -rc) - - # get the issue number from the config map.. the build that - # was triggered by the pr would have cached it there. + #GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN=$GHCR_TOKEN # git_event.txt #EVENT_INFO=$(cat git_event.txt) - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + # extract PR number from GH Action Event Info object + # --------------------------------------------------------- PR_NUMBER=$(echo $EVENT_INFO | jq ".number") echo pr number is $PR_NUMBER PR_URL=$(echo $EVENT_INFO | jq ".pull_request.url" | tr -d '"') echo pr url is $PR_URL + + # configure reviewers - (should extract this from the repo) + # --------------------------------------------------------- REVIEW_URL="$PR_URL/requested_reviewers" echo review url $REVIEW_URL # NicoledeGreef - REVIEWERS='{"reviewers":["franTarkenton", "GuyTheFlower"]}' + REVIEWERS='{"reviewers":["franTarkenton", "GuyTheFlower", "NicoledeGreef"]}' echo reviewers are $REVIEWERS curl --location --request POST "$REVIEW_URL" \ diff --git a/.github/workflows/deploy_prod.yaml b/.github/workflows/deploy_prod.yaml index 6eb4bd3..2863016 100644 --- a/.github/workflows/deploy_prod.yaml +++ b/.github/workflows/deploy_prod.yaml @@ -27,6 +27,13 @@ jobs: run: shell: bash runs-on: ubuntu-20.04 + env: + OPENSHIFT_SERVER_URL: ${{secrets.OPENSHIFT_SERVER_URL}} + OPENSHIFT_TOKEN_DEV: ${{secrets.OPENSHIFT_TOKEN_DEV}} + OPENSHIFT_TOKEN_PROD: ${{secrets.OPENSHIFT_TOKEN_PROD}} + GHCR_USER: ${{ secrets.GHCR_USER }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + steps: # CHECKOUT THE CODE @@ -39,6 +46,7 @@ jobs: - name: OpenShift Action id: GHAOCAuthenticationDev uses: redhat-developer/openshift-actions@v1.1 + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' with: version: '3.11.232' openshift_server_url: ${{ secrets.OPENSHIFT_SERVER_URL }} @@ -48,6 +56,7 @@ jobs: # GETTING THE IMAGE TAG FROM DEV OC - name: Retrieve tag from the artifacts id: retrieveimagetag + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | # outputs: # -------------------------------------------------------- @@ -93,6 +102,7 @@ jobs: # APPLY THE RELEASE TAG TO THE MERGED COMMIT - name: Apply the tag to release id: applyreleasetag + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' && env.GHCR_TOKEN != '' uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GHCR_TOKEN }} # This token is provided by Actions, you do not need to create your own token @@ -108,6 +118,7 @@ jobs: - name: OpenShift Action id: GHAOCAuthenticationPrd uses: redhat-developer/openshift-actions@v1.1 + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_PROD != '' with: version: '3.11.232' openshift_server_url: ${{ secrets.OPENSHIFT_SERVER_URL }} @@ -117,6 +128,7 @@ jobs: # DEPLOY TO PROD - name: Deploy to prod id: proddeploy + if: env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_PROD != '' run: | # ---- get the oc namespace name and set the namespace, verify that the project # that is being pointed to is the correct one or raise an error @@ -148,3 +160,6 @@ jobs: id: prodRouteDeploy run: | echo not enabled atm + + # Add the url that was created by the kong action and add it to the issue + # Add a comment to the issue to say that it has been deployed successfully diff --git a/docs/CDCI-pipeline.md b/docs/CDCI-pipeline.md index e37a8f3..2fa5062 100644 --- a/docs/CDCI-pipeline.md +++ b/docs/CDCI-pipeline.md @@ -22,7 +22,7 @@ the following steps: # Details -The full CD/CI pipeline is handled by github actions. How the pipeline code +The full CD/CI pipeline is initiated by github actions. How the pipeline code gets added to the repo is still yet to be determined. Some idea include: 1. code is injected into the repository when the smk-cli tool is run @@ -37,18 +37,25 @@ cannot be run due to the absence of secrets. Once secrets are added to the repo the next time the action runs it will process what it can based on what secrets are populated. +Ideally all smk based apps should use the same actions. Ideally actions are +linked and not embedded into repositories allowing DataBC to update actions +without having to update each individual repository that has them embedded +into them. + ## Build Build is initiated by a PR to Github's master branch. The build creates a -docker image and stores it as a [github package](https://github.com/orgs/bcgov/packages?repo_name=smk-fap-fcb) +docker image and stores it as a github package. The following is an example +of the package for the project: +[smk-fap-fcp](https://github.com/orgs/bcgov/packages?repo_name=smk-fap-fcb) -Images are labelled using a timestamp. +Images are labelled using a timestamp. [List of releases](https://github.com/bcgov/smk-fap-fcb/releases) The build process calculates an image tag, then caches it in OCP as a configmap for subsequent deployments. If the ocp credentials are not populated then the tag will not get populated. -### Credentials used by Build +### Credentials used by Build / Deploy Github Actions * GHCR_TOKEN: used to authenticate to github for different api calls * GHCR_USER: used to authenticate to github, not actually required for the @@ -56,58 +63,16 @@ tag will not get populated. migration from github packages to github container registry (ghcr.io) * OPENSHIFT_SERVER_URL: the url that is used to communicate with openshift (the url used to authenticate oc cli) -* OPENSHIFT_TOKEN_DEV: service account api key for the oc project. This key - is generated the first time the helm chart is run in ocp4. This is the api key for the dev namespace. Used by steps that deploy to dev. +* OPENSHIFT_TOKEN_DEV: service account api key for the dev oc project. This key + is generated the first time the helm chart is run in ocp4. This is the + api key for the dev namespace. Used by steps that deploy to dev. +* OPENSHIFT_TOKEN_PROD: service account api key for the prod oc project. This key + is generated by the smk init help chart and is the api key for the prod + namespace. Used by steps that deploy to prod. ## Deployment Pre-requisites -All actions should NOT fail even without these parameters being populated, -however they just won't actually do anything. Once these parameters are populated, subsequent PR's should successfully trigger a deployment to Github. - -Deployment actions use all the secrets used by the build, and for the action that completes the final deployment to prod the additional secrets are used: - -* OPENSHIFT_TOKEN_PROD= -* OPENSHIFT_NAMESPACES=, where each entry is equal to the ocp namespace used for different envs. - -example OPENSHIFT_NAMESPACES secret: -``` json -{ - "dev": "glid27-dev", - "test": "glid27-test", - "prod": "glid27-prod" -} -``` - -### Define Helm Chart Values - -Take this template and fill in the correct values, then when complete -put the text into the github secret: **OCP4-HELM-VALUES** - -``` -app_name: -app_image_pull_secret_params: - email: - username: - password: - registry: - imagetag: -``` - - - - - -## Dev Deployment - -### De - -Before any deployments will actually work you need to populate the f - -Dev deployments are triggered by a successful build. All actions beyond the -build will check that the secret: OCP4-SA-KEY. If the key is not populated -the actions will proceed but actually do nothing. - -Deployments are all handled by the helm chart located in the directory -'helm-charts'. Deployments also use the parameter - - +Actions have been configured so that they will not run until the secrets above +have been populated. The actions will actually run, they just won't actually +do anything. Once these parameters are populated, subsequent PR's should +successfully trigger a deployment to Github. diff --git a/docs/build-deploy-notes.md b/docs/build-deploy-notes.md index 69dd4d8..55f0dc8 100644 --- a/docs/build-deploy-notes.md +++ b/docs/build-deploy-notes.md @@ -1,3 +1,6 @@ +This doc contains misc notes I made to help with the development of the cd/ci pipeline +that the repo runs with. + # Testing the Container ## Build an image using Docker @@ -17,8 +20,10 @@ docker build -t $IMAGE_NAME . # Helm deployments -With this chart we are trying to re-use a single chart to deploy multiple SMK based apps that have been generated using the SMK command line tool. Most of the objects defined in the helm chart will be duplicated to create different SMK based apps in the same -repository. +With this chart we are trying to re-use a single chart to deploy multiple SMK +based apps that have been generated using the SMK command line tool. Most of +the objects defined in the helm chart will be duplicated to create different +SMK based apps in the same repository. ## objects created by Helm chart that are re-used for each app diff --git a/helm-charts/templates/_helpers.tpl b/helm-charts/templates/_helpers.tpl index 122b5a6..c42e213 100644 --- a/helm-charts/templates/_helpers.tpl +++ b/helm-charts/templates/_helpers.tpl @@ -61,3 +61,11 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +create image pull secrets for backup image github access +*/}} +{{- define "helm-charts.app_image_pull_secret_params" }} +{{- with .Values.app_image_pull_secret_params }} +{{- printf "{\"auths\":{\"%s\":{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"%s\",\"auth\":\"%s\"}}}" .registry .username .password .email (printf "%s:%s" .username .password | b64enc) | b64enc }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-charts/templates/smk-app-imagepullsecret.yaml b/helm-charts/templates/smk-app-imagepullsecret.yaml new file mode 100644 index 0000000..cd858b7 --- /dev/null +++ b/helm-charts/templates/smk-app-imagepullsecret.yaml @@ -0,0 +1,11 @@ +kind: Secret +apiVersion: v1 +type: kubernetes.io/dockerconfigjson +metadata: + name: {{ .Values.app_image_pull_secret_params.name }} + labels: + app: {{ .Values.app_name }} + env: {{ .Values.env }} + name: {{ .Values.app_image_pull_secret_params.name }} +data: + .dockerconfigjson: {{ template "helm-charts.app_image_pull_secret_params" . }} diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 5958f33..6d01882 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -43,6 +43,9 @@ kong_config: aps_nsp_name: smk-apps app_image_pull_secret_params: - registry: docker.pkg.github.com/bcgov/smk-fap-fcb/smk-fap-fcb - imagetag: 20210112-1958 + name: dummy-image-pull-secret-name + registry: path.to.a.github.docker.package.registry + imagetag: image.tag + email: email.associated.with.registry + password: password.used.to.access.registry From 9ae4fe429b72a7daa75708202838603232b1a683 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Fri, 15 Jan 2021 19:05:53 -0800 Subject: [PATCH 02/12] update docs --- docs/summary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/summary.md b/docs/summary.md index d8d8583..ed0693d 100644 --- a/docs/summary.md +++ b/docs/summary.md @@ -1,9 +1,9 @@ # Overview -This doc summarizes how the various build/deployment pieces work for smk based +This doc describes how the various build/deployment github actions work for smk based apps deployed to openshift. -In summary from the very start the steps would be: +To take a current SMK repository and deploy it to openshift the following are the steps: 1. Acquire a new openshift set of namespaces (dlv|test|prod|tools) 1. Run the namespace init helm chart From d53a84deb4df0da14407d31523a1cdc11271bb94 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Mon, 18 Jan 2021 16:24:46 -0800 Subject: [PATCH 03/12] wrong image deployed to dev * trying to get to the bottom of this issue. Trying a net new build. * adding in reporting on the image that is deployed * testing the service account config / access to image pull --- .github/workflows/build.yaml | 1 - .github/workflows/deploy_dev.yaml | 6 ++++++ .gitignore | 1 + helm-charts/values.yaml | 1 - 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5ac714f..15759ee 100755 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -72,7 +72,6 @@ jobs: echo ::set-output name=TIMESTAMPTAG::${DATESTAMP} echo ::set-output name=REPO::${REPO} - echo event name is $GITHUB_EVENT_NAME echo $DEBUG_DEPLOY diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 6057e08..2d76bff 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -141,6 +141,12 @@ jobs: --set app_image_pull_secret_params.imagetag=$DOCKER_VERSION_TAG \ --set app_image_pull_secret_params.registry=$IMAGE_REGISTRY + # verify / report on the image tag that was deployed + IMAGE_DEPLOYED=$(oc get dc smk-fap-fcb-dc -o json | jq '.spec.template.spec.containers[0].image' | tr -d '"' ) + echo IMAGE_DEPLOYED $IMAGE_DEPLOYED + IMAGE_DEPLOYED_TAG=$(echo ${IMAGE_DEPLOYED} | cut -d':' -f2) + echo IMAGE_DEPLOYED_TAG $IMAGE_DEPLOYED_TAG + # finally need to grant the service account access to the the # image_pull_secret that was just created serviceAccountName=$(oc whoami | cut -d':' -f4) diff --git a/.gitignore b/.gitignore index 6408782..6d90630 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ gwa-cli-linux junk gwa/gwa-config-service-demo.yaml gwa_v*_linux_x64.zip +junk* diff --git a/helm-charts/values.yaml b/helm-charts/values.yaml index 6d01882..7efc219 100644 --- a/helm-charts/values.yaml +++ b/helm-charts/values.yaml @@ -48,4 +48,3 @@ app_image_pull_secret_params: imagetag: image.tag email: email.associated.with.registry password: password.used.to.access.registry - From 996eb0f608604138b06852eb6e97e615c97c2109 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Mon, 18 Jan 2021 17:59:00 -0800 Subject: [PATCH 04/12] fixing the request for reviews - removed myself as a reviewer --- .github/workflows/deploy_dev.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 2d76bff..2603bff 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -24,6 +24,8 @@ jobs: GHCR_USER: ${{ secrets.GHCR_USER }} GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} RUN_GWA: false + REVIEWERS: '["GuyTheFlower", "NicoledeGreef"]' + MENTIONS: '["GuyTheFlower", "NicoledeGreef", "franTarkenton"]' steps: # Checking out code to get access to the openshift deploy template later on @@ -185,10 +187,13 @@ jobs: #echo issue url $issue_url #ISSUE_URL=${{ steps.retrieveimagetag.outputs.ISSUE_URL }} + #format reviewers + MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g') + # ADD COMMENT TO THE ISSUE echo 'adding route to the comment for the issue' GITHUB_TOKEN=${{ secrets.GHCR_TOKEN }} - ISSUE_COMMENT="link to demo deployment of your smk based app is \n https://$ROUTEPATH\n\nVerify that it looks good, if so merge the pull request which will trigger a prod deployment, otherwise close the pull request without merging" + ISSUE_COMMENT="link to demo deployment of your smk based app is \n https://$ROUTEPATH\n\nVerify that it looks good, if so merge the pull request which will trigger a prod deployment, otherwise close the pull request without merging\n\n mentions / notifications: $MENTION_STRING" ISSUE_BODY='{"body": "'$ISSUE_COMMENT'"}' curl $ISSUE_URL -X POST --header "Authorization: Bearer $GITHUB_TOKEN" \ @@ -283,7 +288,7 @@ jobs: REVIEW_URL="$PR_URL/requested_reviewers" echo review url $REVIEW_URL # NicoledeGreef - REVIEWERS='{"reviewers":["franTarkenton", "GuyTheFlower", "NicoledeGreef"]}' + REVIEWERS='{"reviewers":'${REVIEWERS}' }' echo reviewers are $REVIEWERS curl --location --request POST "$REVIEW_URL" \ From 93e4236367b0f46eed0c3e4e1ab9b69b4340926b Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 16:51:11 -0800 Subject: [PATCH 05/12] New steps to actions * added automated GWA flow to generate vanity URLs * added update of issue, so the pr issue has the vanity url in it. --- .github/workflows/deploy_dev.yaml | 159 +++++++++++++++++++----------- .gitignore | 4 +- gwa/createGWAConfig.py | 76 +++++++------- gwa/gwa-config-demo.yaml | 22 ++--- 4 files changed, 147 insertions(+), 114 deletions(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 2603bff..1cd7d13 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -176,17 +176,6 @@ jobs: ISSUE_URL=$(oc get configmap $GITHUB_ISSUES_CONFIGMAP -o json | jq .data.issueURL | tr -d '"') echo ISSUE_URL $ISSUE_URL - # GET THE ISSUE URL from event path - Disabled - #issue_url=$(jq --raw-output .pull_request.issue_url "$GITHUB_EVENT_PATH") - #echo issue url is $issue_url - ## for debugging using existing pull_request - #if [ -z "$issue_url" ] || [ "$issue_url" = "null" ] - # then - # issue_url="https://api.github.com/repos/bcgov/bcdc-smk/issues/19/comments" - #fi - #echo issue url $issue_url - #ISSUE_URL=${{ steps.retrieveimagetag.outputs.ISSUE_URL }} - #format reviewers MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g') @@ -199,12 +188,16 @@ jobs: curl $ISSUE_URL -X POST --header "Authorization: Bearer $GITHUB_TOKEN" \ --header "Accept: application/vnd.github.v3+json" \ --data-raw "$ISSUE_BODY" + + echo ::set-output name=ISSUE_URL::${ISSUE_URL} - name: Get GWA CLI - id: getGWA + id: getGWACLI if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | - # disabled atm + + # Grabbing the GWA command line tool + #---------------------------------------------------- GWA_VERSION=v1.1.2 GWA_CLI_LINK=https://github.com/bcgov/gwa-cli/releases/download/${GWA_VERSION}/gwa_${GWA_VERSION}_linux_x64.zip curl -L -O $GWA_CLI_LINK @@ -215,52 +208,100 @@ jobs: id: kongconf if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | - #------------------------------------------------------------------------ - # commenting this whole section out, current gwa config may work but - # it is incorrect, the gwa should wrap the service vs a route, the - # python code that creates the input file also should get updated as - # the config file it spits out wraps the gwa around a route vs a service. - #------------------------------------------------------------------------ - # # - # # extract kong config info from secrets - # export PATH=`pwd`:$PATH - # ls -l - # gitrepo=${{ steps.retrieveimagetag.outputs.REPONAME }} - # echo gitrepo $gitrepo - # kongSecrets=$(oc get secrets kong-secrets -o json | jq '.data' | jq '.["kong-config"] | @base64d | fromjson') - # GWA_CLIENTID=$(echo $kongSecrets | jq '.gwa_client_id') - # echo client id $GWA_CLIENTID - # GWA_TOKEN=$(echo $kongSecrets | jq '.apikey') - # GWA_NAMESPACE=$(echo $kongSecrets | jq '.gwa_namespace') - # echo namespace $GWA_CLIENTID - # gwa_route_name=${gitrepo}-kong-route - # oc_service_name=${gitrepo}-svc - # oc_route_name="${gitrepo}-rt" - # echo oc_route_name $oc_route_name - # echo oc_service_name $oc_service_name - # echo gwa_route_name $gwa_route_name - # oc_route_host=https://$(oc get routes $oc_route_name -o json | jq '.spec.host' | tr -d '"') - - # gwa-cli-linux init -T \ - # --namespace=$GWA_NAMESPACE \ - # --client-id=$GWA_CLIENTID \ - # --client-secret=$GWA_TOKEN - - # echo init complete - - # python --version - # python3 gwa/createGWAConfig.py \ - # service=$oc_service_name \ - # ocUrl=$oc_route_host \ - # gwa_namespace=$GWA_NAMESPACE \ - # gwa_route_name=$gwa_route_name \ - # url_prefix=$gitrepo > gwa_config.yaml - - # echo gwa config created - - # gwa-cli-linux pg gwa_config.yaml - echo kong config is currently disabled - + + # get the Reponame from previous step + # --------------------------------------------------- + REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} + + # extract kong config info from secret objects + # -------------------------------------------------- + kongSecrets=$(oc get secrets kong-secrets -o json | jq '.data' | jq '.["kong-config"] | @base64d | fromjson') + GWA_CLIENTID=$(echo $kongSecrets | jq '.gwa_client_id') + GWA_TOKEN=$(echo $kongSecrets | jq '.apikey') + GWA_NAMESPACE=$(echo $kongSecrets | jq '.gwa_namespace') + gwa_route_name=${REPONAME}-route + oc_service_name=${REPONAME}-svc + oc_service_port=$(oc get service ${oc_service_name} -o json | jq '.spec.ports[0].port') + oc_project=$(oc project --short=true) + + # init the gwa config + # -------------------------------------------------- + ./gwa-cli-linux init -T \ + --namespace=$GWA_NAMESPACE \ + --client-id=$GWA_CLIENTID \ + --client-secret=$GWA_TOKEN + + # create the gwa config + # -------------------------------------------------- + python3 gwa/createGWAConfig.py \ + OCService=$oc_service_name \ + reponame=$REPONAME \ + OCNamespace=$oc_project \ + servicePort=$oc_service_port \ + kongDomain=api.gov.bc.ca + GWANamespace=$GWA_NAMESPACE \ + GWAenv=dev > gwa_config.yaml + + # publish the gwa config + # -------------------------------------------------- + ./gwa-cli-linux pg gwa_config.yaml + + # verify the url and output it + # -------------------------------------------------- + + # authenticate to api + CREDS64=$(echo -ne "${GWA_CLIENTID}:${GWA_TOKEN}" | base64 -w 0 -) + TOKEN=$(curl 'https://authz-apps-gov-bc-ca.test.apsgw.xyz/auth/realms/aps/protocol/openid-connect/token' \ + -H 'Accept: application/json, text/plain, */*' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -H 'X-Requested-With: XMLHttpRequest' \ + -H "Authorization: Basic ${CREDS64}" \ + -H 'Origin: https://gwa-api-gov-bc-ca.test.apsgw.xyz' \ + -H 'Connection: keep-alive' \ + --data-raw 'grant_type=client_credentials' \ + | jq '.access_token' | tr -d '"') + + # get the services in the namespace + SERVICES=$(curl "https://gwa-api-gov-bc-ca.test.apsgw.xyz/v1/namespaces/${GWA_NAMESPACE}/services" \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0' \ + -H 'Accept: */*' \ + -H 'Accept-Language: en-US,en;q=0.5' --compressed \ + -H "Authorization: Bearer ${TOKEN}" \ + -H 'Connection: keep-alive' ) + echo SERVICES $SERVICES + + # extract the url associated with the service + function getUrl { + python3 - < Date: Wed, 20 Jan 2021 16:57:19 -0800 Subject: [PATCH 06/12] GHA - testing * changed the value that controls if the gwa steps run --- .github/workflows/deploy_dev.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 1cd7d13..c1ae5fe 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -23,7 +23,7 @@ jobs: OPENSHIFT_TOKEN_DEV: ${{secrets.OPENSHIFT_TOKEN_DEV}} GHCR_USER: ${{ secrets.GHCR_USER }} GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} - RUN_GWA: false + RUN_GWA: true REVIEWERS: '["GuyTheFlower", "NicoledeGreef"]' MENTIONS: '["GuyTheFlower", "NicoledeGreef", "franTarkenton"]' steps: @@ -289,6 +289,7 @@ jobs: - name: "Add Dev Vanity Url to Issue" id: addDevVanityUrlToIssue + if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | ISSUEURL=${ steps.updateissue.outputs.ISSUEURL } VANITY_URL=${ steps.kongconf.outputs.VANITY_URL } From e6878907941793cb4f2af001bc73b767b583d2b9 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 17:30:12 -0800 Subject: [PATCH 07/12] cleaning up errors in dev deploy --- .github/workflows/deploy_dev.yaml | 24 +++++++++++----------- gwa/createGWAConfig.py | 33 +++++++++++++------------------ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index c1ae5fe..45b9d5d 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -216,13 +216,13 @@ jobs: # extract kong config info from secret objects # -------------------------------------------------- kongSecrets=$(oc get secrets kong-secrets -o json | jq '.data' | jq '.["kong-config"] | @base64d | fromjson') - GWA_CLIENTID=$(echo $kongSecrets | jq '.gwa_client_id') - GWA_TOKEN=$(echo $kongSecrets | jq '.apikey') - GWA_NAMESPACE=$(echo $kongSecrets | jq '.gwa_namespace') + GWA_CLIENTID=$(echo $kongSecrets | jq '.gwa_client_id' | tr -d '"') + GWA_TOKEN=$(echo $kongSecrets | jq '.apikey' | tr -d '"') + GWA_NAMESPACE=$(echo $kongSecrets | jq '.gwa_namespace' | tr -d '"') gwa_route_name=${REPONAME}-route oc_service_name=${REPONAME}-svc - oc_service_port=$(oc get service ${oc_service_name} -o json | jq '.spec.ports[0].port') - oc_project=$(oc project --short=true) + oc_service_port=$(oc get service ${oc_service_name} -o json | jq '.spec.ports[0].port' | tr -d '"') + oc_project=$(oc project --short=true | tr -d '"') # init the gwa config # -------------------------------------------------- @@ -234,13 +234,13 @@ jobs: # create the gwa config # -------------------------------------------------- python3 gwa/createGWAConfig.py \ - OCService=$oc_service_name \ - reponame=$REPONAME \ - OCNamespace=$oc_project \ - servicePort=$oc_service_port \ - kongDomain=api.gov.bc.ca - GWANamespace=$GWA_NAMESPACE \ - GWAenv=dev > gwa_config.yaml + --OCService $oc_service_name \ + --reponame $REPONAME \ + --OCNamespace $oc_project \ + --servicePort $oc_service_port \ + --kongDomain api.gov.bc.ca \ + --GWANamespace $GWA_NAMESPACE \ + --GWAenv dev > gwa_config.yaml # publish the gwa config # -------------------------------------------------- diff --git a/gwa/createGWAConfig.py b/gwa/createGWAConfig.py index 5fc9016..61a51f8 100644 --- a/gwa/createGWAConfig.py +++ b/gwa/createGWAConfig.py @@ -14,13 +14,13 @@ def __init__(self): def slurpArgs(self): parser = argparse.ArgumentParser(description='Provide parameter used to construct the gwa config file.') - parser.add_argument("OCService", help="openshift service that the route should bind to") - parser.add_argument("reponame", help="unique name for your smk app, usually the name of your repository.") - parser.add_argument("OCNamespace", help="The openshift namespace that the app resides in.") - parser.add_argument("servicePort", help="The port that the open shift service is configured for.") - parser.add_argument("kongDomain", help="The domain suffix that will be created.") - parser.add_argument("GWANamespace", help="the gwa namespace created using gwa tool") - parser.add_argument("GWAenv", help="the gwa namespace created using gwa tool") + parser.add_argument("--OCService", help="openshift service that the route should bind to") + parser.add_argument("--reponame", help="unique name for your smk app, usually the name of your repository.") + parser.add_argument("--OCNamespace", help="The openshift namespace that the app resides in.") + parser.add_argument("--servicePort", help="The port that the open shift service is configured for.") + parser.add_argument("--kongDomain", help="The domain suffix that will be created.") + parser.add_argument("--GWANamespace", help="the gwa namespace created using gwa tool") + parser.add_argument("--GWAenv", help="the gwa namespace created using gwa tool") parser.add_argument("--endpointdir", help="the end point to add to your route", default='/') args = parser.parse_args() @@ -28,7 +28,6 @@ def slurpArgs(self): self.reponame = args.reponame self.OCNamespace = args.OCNamespace self.servicePort = args.servicePort - print(f"self.servicePort: {self.servicePort}") self.kongDomain = args.kongDomain self.GWANamespace = args.GWANamespace self.GWAenv = args.GWAenv @@ -67,23 +66,19 @@ def createYaml(self): ] } yamlString = yaml.dump(yamlData, sys.stdout) - print(yamlString) if __name__ == "__main__": # debug - sys.argv.append("smk-fap-fcp-svc") - sys.argv.append("smk-fap-fcb") - sys.argv.append("b16795-dev") - sys.argv.append("8888") - sys.argv.append("api.gov.bc.ca") - sys.argv.append("smk-apps") - sys.argv.append("dev") - - - + # sys.argv.append("smk-fap-fcp-svc") + # sys.argv.append("smk-fap-fcb") + # sys.argv.append("b16795-dev") + # sys.argv.append("8888") + # sys.argv.append("api.gov.bc.ca") + # sys.argv.append("smk-apps") + # sys.argv.append("dev") gwaConf = GWAConfig() gwaConf.slurpArgs() From b3b2312b4884c67d057ab90588bf8a54b0a636bb Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 17:37:37 -0800 Subject: [PATCH 08/12] fixing typo in deploy actions --- .github/workflows/deploy_dev.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 45b9d5d..dea3479 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -276,12 +276,12 @@ jobs: import json, sys, os jsonObj = json.loads(os.environ['SERVICES']) for obj in jsonObj: - if obj['name'] == os.environ['OCService']: + if obj['name'] == os.environ['oc_service_name']: sys.stdout.write(f"https://{obj['env_host']}") END } - export OCService - echo OCService $OCService + export oc_service_name + echo oc_service_name $oc_service_name export SERVICES VANITY_URL=$(getUrl) echo vanity url $VANITY_URL From b65706972bd6760da436a9d32c343b2d9fb82a33 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 17:52:22 -0800 Subject: [PATCH 09/12] fixing problems with the github action that is updating the issue --- .github/workflows/deploy_dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index dea3479..0f360a7 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -291,8 +291,8 @@ jobs: id: addDevVanityUrlToIssue if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | - ISSUEURL=${ steps.updateissue.outputs.ISSUEURL } - VANITY_URL=${ steps.kongconf.outputs.VANITY_URL } + ISSUEURL=${{ steps.updateissue.outputs.ISSUEURL }} + VANITY_URL=${{ steps.kongconf.outputs.VANITY_URL }} REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g') echo MENTION_STRING $MENTION_STRING From 73af053bba41c3f1567e0eac6261da3f9d33d94d Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 18:08:44 -0800 Subject: [PATCH 10/12] fixing GHA, issues --- .github/workflows/deploy_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index 0f360a7..cbbcaa7 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -291,7 +291,7 @@ jobs: id: addDevVanityUrlToIssue if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | - ISSUEURL=${{ steps.updateissue.outputs.ISSUEURL }} + ISSUE_URL=${{ steps.updateissue.outputs.ISSUE_URL }} VANITY_URL=${{ steps.kongconf.outputs.VANITY_URL }} REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g') From 4e9d8d263a40e51641674a9acb18d2fcb5949edf Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 18:47:49 -0800 Subject: [PATCH 11/12] Fixing how the Kong Vanity URL commenting is taking place --- .github/workflows/deploy_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index cbbcaa7..e3e9676 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -296,7 +296,7 @@ jobs: REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g') echo MENTION_STRING $MENTION_STRING - ISSUECOMMENT="The Vanity URL for the Dev deploy is: $VANITY_URL, \n\n $MENTION_STRING" + ISSUE_COMMENT="The Vanity URL for the Dev deploy is: $VANITY_URL, \n\n $MENTION_STRING" ISSUE_BODY='{"body": "'$ISSUE_COMMENT'"}' GITHUB_TOKEN=${{ secrets.GHCR_TOKEN }} curl $ISSUE_URL -X POST --header "Authorization: Bearer $GITHUB_TOKEN" \ From c3c2f468ef0811b447e286660d180628b74bd235 Mon Sep 17 00:00:00 2001 From: Guy Lafleur Date: Wed, 20 Jan 2021 18:48:07 -0800 Subject: [PATCH 12/12] fixing vanity url commenting on issue --- .github/workflows/deploy_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_dev.yaml b/.github/workflows/deploy_dev.yaml index e3e9676..39d6788 100644 --- a/.github/workflows/deploy_dev.yaml +++ b/.github/workflows/deploy_dev.yaml @@ -291,7 +291,7 @@ jobs: id: addDevVanityUrlToIssue if: env.RUN_GWA == 'true' && env.OPENSHIFT_SERVER_URL != '' && env.OPENSHIFT_TOKEN_DEV != '' run: | - ISSUE_URL=${{ steps.updateissue.outputs.ISSUE_URL }} + ISSUE_URL=${{ steps.updateissue.outputs.ISSUE_URL }}/comments VANITY_URL=${{ steps.kongconf.outputs.VANITY_URL }} REPONAME=${{ steps.retrieveimagetag.outputs.REPONAME }} MENTION_STRING=$(echo $MENTIONS | jq -rc 'map("@" + .) | .[]' | tr '\n' ',' | sed 's/, */, /g' | sed 's/, *$//g')