Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Replace backend with platform #264

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
234 changes: 139 additions & 95 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
name: Build, Test, and Deliver Client


env:
do_sonarscan: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
do_sonarscan:
${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name ==
github.repository }}

on:
pull_request:
Expand Down Expand Up @@ -37,7 +38,7 @@ jobs:
textReportPath: './lib/coverage/coverage.txt'
- run: echo "${MARKDOWN_REPORT}" >> $GITHUB_STEP_SUMMARY
env:
MARKDOWN_REPORT: "${{ steps.coverage-md.outputs.markdownReport }}"
MARKDOWN_REPORT: '${{ steps.coverage-md.outputs.markdownReport }}'
- run: npm audit --omit dev && npm audit --audit-level high
- run: npm run license-check
- run: npm run lint
Expand All @@ -53,32 +54,32 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

remote-store:
needs:
- lib
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./remote-store
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: '18'
cache: 'npm'
cache-dependency-path: './remote-store/package-lock.json'
- uses: actions/download-artifact@v3
with:
name: opentdf-client-lib
path: lib/
- run: npm uninstall @opentdf/client && npm ci && npm i ../lib/opentdf-client-*.tgz
- run: npm install
- run: npm test
- run: npm audit
- run: npm run license-check
- run: npm run lint
- run: npm pack
# remote-store:
# needs:
# - lib
# runs-on: ubuntu-latest
# defaults:
# run:
# working-directory: ./remote-store
# timeout-minutes: 5
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-node@v3
# with:
# node-version: '18'
# cache: 'npm'
# cache-dependency-path: './remote-store/package-lock.json'
# - uses: actions/download-artifact@v3
# with:
# name: opentdf-client-lib
# path: lib/
# - run: npm uninstall @opentdf/client && npm ci && npm i ../lib/opentdf-client-*.tgz
# - run: npm install
# - run: npm test
# - run: npm audit
# - run: npm run license-check
# - run: npm run lint
# - run: npm pack

web-app:
needs:
Expand Down Expand Up @@ -107,25 +108,72 @@ jobs:
- run: npm run lint
- run: npm pack

scripts:
runs-on: ubuntu-latest
defaults:
run:
working-directory: .github/workflows
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
- uses: mig4/setup-bats@v1
with:
bats-version: 1.2.1
- name: 🦇🦇🦇🦇🦇🦇
run: docker run --rm -v "$PWD:/mnt" --workdir "/mnt" bats/bats:1.5.0 *.bats
- name: 🦪 ✔ 🧼🧼🧼
run: >-
docker run --rm -v "$PWD:/mnt" --workdir "/mnt" "koalaman/shellcheck:v0.8.0"
--color=always *.sh
# scripts:
# runs-on: ubuntu-latest
# defaults:
# run:
# working-directory: .github/workflows
# timeout-minutes: 5
# steps:
# - uses: actions/checkout@v3
# - uses: mig4/setup-bats@v1
# with:
# bats-version: 1.2.1
# - name: 🦇🦇🦇🦇🦇🦇
# run: docker run --rm -v "$PWD:/mnt" --workdir "/mnt" bats/bats:1.5.0 *.bats
# - name: 🦪 ✔ 🧼🧼🧼
# run: >-
# docker run --rm -v "$PWD:/mnt" --workdir "/mnt" "koalaman/shellcheck:v0.8.0"
# --color=always *.sh

# backend-roundtrip:
# needs:
# - web-app
# runs-on: ubuntu-latest
# defaults:
# run:
# working-directory: .github/workflows/roundtrip
# timeout-minutes: 45
# steps:
# - uses: actions/checkout@v3
# - uses: actions/setup-node@v3
# with:
# node-version: '18'
# cache: 'npm'
# cache-dependency-path: './web-app/package-lock.json'
# - uses: actions/download-artifact@v3
# with:
# name: opentdf-client-lib
# path: lib/
# - uses: yokawasa/action-setup-kube-tools@v0.11.1
# with:
# setup-tools: |
# kubectl
# helm
# tilt
# # This should be in sync with the minikube-deployed kube version below
# kubectl: '1.30.0'
# helm: '3.14.4'
# tilt: '0.33.13'
# - run: |
# kubectl version --client
# kustomize version
# tilt version
# - name: start minikube
# id: minikube
# uses: medyagh/setup-minikube@master
# with:
# minikube-version: 1.33.0
# # This should be in sync with the setup-tools version above
# kubernetes-version: 1.30.0
# - name: Run tilt ci
# env:
# TEST_SCRIPT: ./wait-and-test.sh backend
# OPENTDF_INGRESS_HOST_PORT: '5432'
# PLAYWRIGHT_TESTS_TO_RUN: roundtrip
# run: tilt ci

backend-roundtrip:
platform-roundtrip:
needs:
- web-app
runs-on: ubuntu-latest
Expand All @@ -144,43 +192,30 @@ jobs:
with:
name: opentdf-client-lib
path: lib/
- uses: yokawasa/action-setup-kube-tools@v0.11.1
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491
with:
setup-tools: |
kubectl
helm
tilt
# This should be in sync with the minikube-deployed kube version below
kubectl: '1.30.0'
helm: '3.14.4'
tilt: '0.33.13'
- run: |
kubectl version --client
kustomize version
tilt version
- name: start minikube
id: minikube
uses: medyagh/setup-minikube@master
with:
minikube-version: 1.33.0
# This should be in sync with the setup-tools version above
kubernetes-version: 1.30.0
- name: Run tilt ci
env:
TEST_SCRIPT: ./wait-and-test.sh
OPENTDF_INGRESS_HOST_PORT: "5432"
go-version: '1.22'
- run: docker compose up -d --wait --wait-timeout 240
- env:
OPENTDF_SERVICES_AUTHORIZATION_URL: 'http://localhost:65432/'
PLAYWRIGHT_TESTS_TO_RUN: roundtrip
run: tilt ci
run: |-
./wait-and-test.sh platform

deliver-ghp:
needs: [lib, web-app, scripts, backend-roundtrip]
needs:
- lib
- web-app
# - scripts
# - backend-roundtrip
- platform-roundtrip
runs-on: ubuntu-latest
timeout-minutes: 5
# To publish from a release or feature branch, remove the ref == condition below
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|| (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/'))
|| (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
(github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name ==
'push' && startsWith(github.ref, 'refs/heads/release/')) || (github.event_name == 'release' &&
startsWith(github.ref, 'refs/tags/'))
outputs:
FULL_VERSION: ${{ steps.guess-build-metadata.outputs.FULL_VERSION }}
DIST_TAG: ${{ steps.guess-build-metadata.outputs.DIST_TAG }}
Expand Down Expand Up @@ -208,22 +243,25 @@ jobs:
echo "DIST_TAG=$(.github/workflows/guess-dist-tag.sh)" >> $GITHUB_OUTPUT
- run: make test
- run: make doc
- run: echo "::notice file=lib/package.json::Will be published to [GitHub Packages](https://github.com/opentdf/client-web/pkgs/npm/client) as ${{ steps.guess-build-metadata.outputs.DIST_TAG }} with version=[${{ steps.guess-build-metadata.outputs.FULL_VERSION }}]"
- run:
echo "::notice file=lib/package.json::Will be published to [GitHub
Packages](https://github.com/opentdf/client-web/pkgs/npm/client) as ${{
steps.guess-build-metadata.outputs.DIST_TAG }} with version=[${{
steps.guess-build-metadata.outputs.FULL_VERSION }}]"
- run: >-
.github/workflows/publish-to.sh
${{ steps.guess-build-metadata.outputs.FULL_VERSION }}
${{ steps.guess-build-metadata.outputs.DIST_TAG }}
.github/workflows/publish-to.sh ${{ steps.guess-build-metadata.outputs.FULL_VERSION }} ${{
steps.guess-build-metadata.outputs.DIST_TAG }}
env:
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: echo "- [Client Library](https://github.com/opentdf/client-web/pkgs/npm/client)">>$GITHUB_STEP_SUMMARY
- run:
echo "- [Client
Library](https://github.com/opentdf/client-web/pkgs/npm/client)">>$GITHUB_STEP_SUMMARY
- name: trigger xtest
run: >-
curl -XPOST
-u "virtru-cloudnative:${{secrets.PERSONAL_ACCESS_TOKEN}}"
-H "Accept: application/vnd.github.everest-preview+json"
-H "Content-Type: application/json"
"https://api.github.com/repos/opentdf/backend/dispatches"
--data '{"event_type":"xtest","client_payload":{"version":"'${FULL_VERSION%%+*}'"}}'
curl -XPOST -u "virtru-cloudnative:${{secrets.PERSONAL_ACCESS_TOKEN}}" -H "Accept:
application/vnd.github.everest-preview+json" -H "Content-Type: application/json"
"https://api.github.com/repos/opentdf/backend/dispatches" --data
'{"event_type":"xtest","client_payload":{"version":"'${FULL_VERSION%%+*}'"}}'
env:
FULL_VERSION: ${{ steps.guess-build-metadata.outputs.FULL_VERSION }}
- name: Publish documentation to gh-pages
Expand All @@ -238,9 +276,9 @@ jobs:
environment: npmjs
needs: deliver-ghp
if: >-
(github.event_name == 'push' && github.ref == 'refs/heads/main')
|| (github.event_name == 'push' && startsWith(github.ref, 'refs/heads/release/'))
|| (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
(github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name ==
'push' && startsWith(github.ref, 'refs/heads/release/')) || (github.event_name == 'release' &&
startsWith(github.ref, 'refs/tags/'))
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v3
Expand All @@ -249,10 +287,16 @@ jobs:
registry-url: 'https://registry.npmjs.org'
- run: make all
- run: >-
.github/workflows/publish-to.sh ${{ needs.deliver-ghp.outputs.FULL_VERSION }}
${{ needs.deliver-ghp.outputs.DIST_TAG }}
.github/workflows/publish-to.sh ${{ needs.deliver-ghp.outputs.FULL_VERSION }} ${{
needs.deliver-ghp.outputs.DIST_TAG }}
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- run: echo "- [Client Library](https://www.npmjs.com/package/@opentdf/client/v/${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
- run: echo "- [Remote Store](https://www.npmjs.com/package/@opentdf/remote-store/v/${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
- run: echo "- [unpkg](https://unpkg.com/browse/@opentdf/client@${{ needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
- run:
echo "- [Client Library](https://www.npmjs.com/package/@opentdf/client/v/${{
needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
- run:
echo "- [Remote Store](https://www.npmjs.com/package/@opentdf/remote-store/v/${{
needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
- run:
echo "- [unpkg](https://unpkg.com/browse/@opentdf/client@${{
needs.deliver-ghp.outputs.FULL_VERSION }})">>$GITHUB_STEP_SUMMARY
6 changes: 3 additions & 3 deletions .github/workflows/large-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Build, Test, and Deliver Client

on:
schedule:
- cron: "0 4 * * 2,4"
- cron: '0 4 * * 2,4'
jobs:
lib:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -106,8 +106,8 @@ jobs:
env:
#path relative to the quickstart Tiltfile
TEST_SCRIPT: ../../wait-and-test.sh
OPENTDF_INGRESS_HOST_PORT: "5432"
OPENTDF_LOAD_FRONTEND: "false"
OPENTDF_INGRESS_HOST_PORT: '5432'
OPENTDF_LOAD_FRONTEND: 'false'
PLAYWRIGHT_TESTS_TO_RUN: huge roundtrip
run: |-
tilt ci --file opentdf/quickstart/Tiltfile
46 changes: 46 additions & 0 deletions .github/workflows/roundtrip/config-demo-idp.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/usr/bin/env bash

set -x

: "${KC_VERSION:=24.0.3}"

if ! which kcadm.sh; then
KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip
echo "DOWNLOADING ${KCADM_URL}"
if ! curl --output kc.zip --fail --location "${KCADM_URL}"; then
echo "[ERROR] Failed to download ${KCADM_URL}"
exit 3
fi
ls -l
if ! unzip ./kc.zip; then
echo "[ERROR] Failed to unzip file from ${KCADM_URL}"
exit 3
fi
ls -l
ls -l "$(pwd)/keycloak-${KC_VERSION}/bin"
PATH=$PATH:"$(pwd)/keycloak-${KC_VERSION}/bin"
export PATH
if ! which kcadm.sh; then
echo "[ERROR] Failed to find kcadm.sh"
exit 3
fi
fi

kcadm.sh config credentials --server http://localhost:65432/auth \
--realm master --user admin --password changeme

kcadm.sh create clients -r opentdf \
-s clientId=browsertest \
-s enabled=true \
-s 'redirectUris=["http://localhost:65432/"]' \
-s consentRequired=false \
-s standardFlowEnabled=true \
-s directAccessGrantsEnabled=true \
-s serviceAccountsEnabled=false \
-s publicClient=true \
-s protocol=openid-connect \
-s 'protocolMappers=[{"name":"aud","protocol":"openid-connect","protocolMapper":"oidc-audience-mapper","consentRequired":false,"config":{"access.token.claim":"true","included.custom.audience":"http://localhost:65432"}}]' \
-s 'attributes={"dpop.bound.access.tokens":"true"}'

kcadm.sh create users -r opentdf -s username=user1 -s enabled=true -s firstName=Alice -s lastName=User
kcadm.sh set-password -r opentdf --username user1 --new-password testuser123
Loading
Loading