diff --git a/.github/workflows/odh-v2-core-bridge.yaml b/.github/workflows/odh-v2-core-bridge.yaml new file mode 100644 index 0000000..5f66724 --- /dev/null +++ b/.github/workflows/odh-v2-core-bridge.yaml @@ -0,0 +1,58 @@ +name: CI/CD odh-v2-core-writer + +on: + push: + paths: + - "infrastructure/raw-data-bridge/**" + - "infrastructure/helm/raw-data-bridge/**" + - ".github/workflows/odh-v2-core-writer.yaml" + workflow_dispatch: + +env: + WORKING_DIRECTORY: infrastructure/raw-data-bridge + KUBERNETES_NAMESPACE: core + KUBERNETES_VALUE_PATH: infrastructure/helm/raw-data-bridge/values.yaml + KUBERNETES_CHART_PATH: infrastructure/helm/raw-data-bridge/raw-data-bridge + PROJECT_NAME: raw-data-bridge + +jobs: + build: + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod' + runs-on: ubuntu-22.04 + concurrency: odh-v2-core-writer-build + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Log in to GitHub Docker Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Manually build and push image + run: cd ${{ env.WORKING_DIRECTORY }} && ./build.sh + + deploy: + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod' + needs: + - build + runs-on: ubuntu-22.04 + concurrency: odh-v2-core-writer-build-deploy + environment: ${{ github.ref == 'refs/heads/main' && 'test' || github.ref == 'refs/heads/prod' && 'prod' || 'unknown' }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Deploy on cluster + uses: noi-techpark/github-actions/helm-deploy@v2 + with: + k8s-name: ${{ env.PROJECT_NAME }} + k8s-namespace: ${{ env.KUBERNETES_NAMESPACE }} + chart-path: ${{ env.KUBERNETES_CHART_PATH }} + values-file: ${{ env.KUBERNETES_VALUE_PATH }} + aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }} + aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }} + aws-eks-cluster-name: aws-main-eu-01 + aws-region: eu-west-1 \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 2de996f..2116c5b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,6 +19,14 @@ "mode": "debug", "program": "${workspaceFolder}/infrastructure/router", "envFile": "${workspaceFolder}/infrastructure/router/.env" + }, + { + "name": "Launch Lookup", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/infrastructure/raw-data-bridge", + "envFile": "${workspaceFolder}/infrastructure/raw-data-bridge/.env" } ] } \ No newline at end of file diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/.helmignore b/infrastructure/helm/raw-data-bridge/raw-data-bridge/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/Chart.yaml b/infrastructure/helm/raw-data-bridge/raw-data-bridge/Chart.yaml new file mode 100644 index 0000000..c295a58 --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: raw-data-bridge +description: Resource Lookup service +type: application +version: 1.0.0 +appVersion: "1.0.0" diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/_helpers.tpl b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/_helpers.tpl new file mode 100644 index 0000000..13b32f2 --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "raw-data-bridge.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "raw-data-bridge.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "raw-data-bridge.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "raw-data-bridge.labels" -}} +helm.sh/chart: {{ include "raw-data-bridge.chart" . }} +{{ include "raw-data-bridge.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "raw-data-bridge.selectorLabels" -}} +app.kubernetes.io/name: {{ include "raw-data-bridge.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/deployment.yaml b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/deployment.yaml new file mode 100644 index 0000000..3f22367 --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "raw-data-bridge.fullname" . }} + labels: + {{- include "raw-data-bridge.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "raw-data-bridge.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "raw-data-bridge.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.health.delay }} + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: {{ .Values.health.delay }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/service.yaml b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/service.yaml new file mode 100644 index 0000000..79b8cae --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "raw-data-bridge.fullname" . }} + labels: + {{- include "raw-data-bridge.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "raw-data-bridge.selectorLabels" . | nindent 4 }} diff --git a/infrastructure/helm/raw-data-bridge/raw-data-bridge/values.yaml b/infrastructure/helm/raw-data-bridge/raw-data-bridge/values.yaml new file mode 100644 index 0000000..08aa622 --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/raw-data-bridge/values.yaml @@ -0,0 +1,32 @@ +# Default values for raw-data-bridge. +replicaCount: 1 + +image: + repository: ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +service: + port: 2000 + type: ClusterIP + +health: + delay: 30 + +env: + - name: APP_MONGO_URI + value: mongodb://user:password@mongodb + - name: APP_LOGLEVEL + value: INFO + +podAnnotations: {} +podLabels: {} +resources: {} +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/infrastructure/helm/raw-data-bridge/values.yaml b/infrastructure/helm/raw-data-bridge/values.yaml new file mode 100644 index 0000000..1b87ccd --- /dev/null +++ b/infrastructure/helm/raw-data-bridge/values.yaml @@ -0,0 +1,18 @@ +image: + repository: ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge + pullPolicy: Always + tag: "v1.0" + +env: + - name: APP_MONGO_URI + value: mongodb://mongodb-headless:27017/ + - name: APP_LOGLEVEL + value: INFO + +# resources: +# limits: +# cpu: 250m # 250m is equivalent to 0.25 CPU core +# memory: 500Mi +# requests: +# cpu: 100m # 100m is equivalent to 0.1 CPU core +# memory: 100Mi diff --git a/infrastructure/raw-data-bridge/.env.example b/infrastructure/raw-data-bridge/.env.example new file mode 100644 index 0000000..6100564 --- /dev/null +++ b/infrastructure/raw-data-bridge/.env.example @@ -0,0 +1,3 @@ +APP_MONGO_URI=mongodb://mongodb + +APP_LOGLEVEL=DEBUG \ No newline at end of file diff --git a/infrastructure/raw-writer/Dockerfile b/infrastructure/raw-data-bridge/Dockerfile similarity index 57% rename from infrastructure/raw-writer/Dockerfile rename to infrastructure/raw-data-bridge/Dockerfile index ba2799c..8b6fda9 100644 --- a/infrastructure/raw-writer/Dockerfile +++ b/infrastructure/raw-data-bridge/Dockerfile @@ -2,18 +2,18 @@ # # SPDX-License-Identifier: CC0-1.0 -FROM golang:1.22.2-bookworm AS base +FROM golang:1.23.4-bookworm AS base FROM base AS build WORKDIR /app COPY go.mod . COPY go.sum . -COPY *.go . -RUN CGO_ENABLED=0 GOOS=linux go build -o writer +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -o raw-data-bridge FROM scratch AS release -COPY --from=build /app/writer /writer -CMD ["/writer"] +COPY --from=build /app/raw-data-bridge /raw-data-bridge +CMD ["/raw-data-bridge"] FROM base AS dev diff --git a/infrastructure/raw-writer/build.sh b/infrastructure/raw-data-bridge/build.sh similarity index 67% rename from infrastructure/raw-writer/build.sh rename to infrastructure/raw-data-bridge/build.sh index 0eb8ae8..78018e9 100755 --- a/infrastructure/raw-writer/build.sh +++ b/infrastructure/raw-data-bridge/build.sh @@ -1,10 +1,10 @@ #!/bin/bash -IMAGE=ghcr.io/noi-techpark/infrastructure-v2/raw-writer +IMAGE=ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge TAGS='latest v1 v1.0' REPO=https://github.com/noi-techpark/infrastructure-v2 docker buildx build --target release -f Dockerfile . -t $IMAGE \ --label "org.opencontainers.image.source=$REPO" \ ---label "org.opencontainers.image.description=Raw data writer service for Open Data Hub ingestion pipeline" \ +--label "org.opencontainers.image.description=Resource Lookup service to retrieve data from Raw Data Table" \ --label "org.opencontainers.image.licenses=AGPL-3.0-or-later" for tag in $TAGS diff --git a/infrastructure/raw-writer/docker-compose.yml b/infrastructure/raw-data-bridge/docker-compose.yml similarity index 75% rename from infrastructure/raw-writer/docker-compose.yml rename to infrastructure/raw-data-bridge/docker-compose.yml index 2b6fa92..f85dc0f 100644 --- a/infrastructure/raw-writer/docker-compose.yml +++ b/infrastructure/raw-data-bridge/docker-compose.yml @@ -11,16 +11,8 @@ services: - pkg:/go/pkg/mod working_dir: /code depends_on: - - rabbitmq - mongodb - rabbitmq: - image: rabbitmq:management-alpine - ports: - - "5672:5672" - - "15672:15672" - attach: false - mongodb: image: mongo ports: diff --git a/infrastructure/raw-data-bridge/go.mod b/infrastructure/raw-data-bridge/go.mod new file mode 100644 index 0000000..4903348 --- /dev/null +++ b/infrastructure/raw-data-bridge/go.mod @@ -0,0 +1,51 @@ +module opendatahub.com/infrav2/raw-data-bridge + +go 1.23.4 + +require ( + github.com/gin-gonic/gin v1.10.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a + github.com/samber/slog-gin v1.14.0 + go.mongodb.org/mongo-driver v1.17.2 + golang.org/x/sync v0.10.0 +) + +require ( + github.com/bytedance/sonic v1.11.9 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.4 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.22.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/infrastructure/raw-data-bridge/go.sum b/infrastructure/raw-data-bridge/go.sum new file mode 100644 index 0000000..b39c8b6 --- /dev/null +++ b/infrastructure/raw-data-bridge/go.sum @@ -0,0 +1,146 @@ +github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= +github.com/bytedance/sonic v1.11.9/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I= +github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= +github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a h1:cmntXRjN/lCskoocnKXZ4gtzbyRZh4E4dF2i24Av/nc= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a/go.mod h1:i/2Gbagc/PBHZD7pjW/22WHvc9MBkhJ9RhVhJ/s9h3c= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/slog-gin v1.14.0 h1:+wfGRudH9xAOaaKlGlXVUyrT9hhFwGX5ChZQOUCszPw= +github.com/samber/slog-gin v1.14.0/go.mod h1:yS2C+cX5tRnPX0MqDby7a3tRFsJuMk7hNwAunyfDxQk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= +go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/infrastructure/raw-data-bridge/main.go b/infrastructure/raw-data-bridge/main.go new file mode 100644 index 0000000..158d5ff --- /dev/null +++ b/infrastructure/raw-data-bridge/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "log" + "log/slog" + "os" + + "github.com/kelseyhightower/envconfig" + "golang.org/x/sync/errgroup" + "opendatahub.com/infrav2/raw-data-bridge/rdt" +) + +var cfg struct { + LogLevel string `default:"INFO"` + MONGO_URI string + DB_PREFIX string +} + +func initConfig() { + err := envconfig.Process("APP", &cfg) + if err != nil { + log.Panic("Unable to initialize config", err) + } +} + +func initLog() { + level := &slog.LevelVar{} + level.UnmarshalText([]byte(cfg.LogLevel)) + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + }))) +} + +func main() { + initConfig() + initLog() + rdt.InitRawDataConnection(cfg.MONGO_URI) + + s := NewServer(context.Background()) + + g, ctx := errgroup.WithContext(context.Background()) + g.Go(func() error { return s.Run(ctx, ":2000") }) + + if err := g.Wait(); err != nil { + slog.Error("failed to run") + } +} diff --git a/infrastructure/raw-data-bridge/rdt/rdt.go b/infrastructure/raw-data-bridge/rdt/rdt.go new file mode 100644 index 0000000..60ed19e --- /dev/null +++ b/infrastructure/raw-data-bridge/rdt/rdt.go @@ -0,0 +1,78 @@ +package rdt + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/noi-techpark/go-opendatahub-ingest/urn" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var ( + ErrDocumentNotFound = errors.New("document not found") + ErrBadURN = errors.New("bad urn format") +) + +var mongoClient *mongo.Client + +type Document struct { + Provider string `json:"provider"` + Timestamp time.Time `json:"timestamp"` + Rawdata string `bson:"rawdata" json:"rawdata"` +} + +func InitRawDataConnection(uri string) { + mclient, err := mongoConnect(uri) + if err != nil { + slog.Error("could not initialize mongo. aborting", "err", err) + panic(err) + } + mongoClient = mclient +} + +func constructTableTarget(urn *urn.URN) (string, string, error) { + provider_tokens := urn.GetNSSWithoutID() + if len(provider_tokens) < 2 { + return "", "", fmt.Errorf("urn format invalid: %s", urn.String()) + } + return provider_tokens[0], strings.Join(provider_tokens[1:], "."), nil +} + +func mongoConnect(uri string) (*mongo.Client, error) { + client, err := mongo.Connect(context.Background(), options.Client().ApplyURI(uri)) + if err != nil { + return nil, err + } + // Send a ping to confirm a successful connection + var result bson.M + if err := client.Database("admin").RunCommand(context.TODO(), bson.D{{"ping", 1}}).Decode(&result); err != nil { + return nil, err + } + return client, nil +} + +func GetDocument(ctx context.Context, urn *urn.URN) (*Document, error) { + db, coll, err := constructTableTarget(urn) + if err != nil { + return nil, ErrBadURN + } + id, err := primitive.ObjectIDFromHex(urn.GetResourceID()) + if err != nil { + return nil, ErrBadURN + } + r := &Document{} + if err := mongoClient.Database(db).Collection(coll).FindOne(ctx, bson.M{"_id": id}).Decode(r); err != nil { + if err == mongo.ErrNoDocuments { + return nil, ErrDocumentNotFound + } + return nil, err + } + return r, nil +} diff --git a/infrastructure/raw-data-bridge/server.go b/infrastructure/raw-data-bridge/server.go new file mode 100644 index 0000000..ac47861 --- /dev/null +++ b/infrastructure/raw-data-bridge/server.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "log/slog" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/noi-techpark/go-opendatahub-ingest/urn" + sloggin "github.com/samber/slog-gin" + "opendatahub.com/infrav2/raw-data-bridge/rdt" +) + +type Server struct { + e *gin.Engine +} + +func NewServer(ctx context.Context) *Server { + gin.SetMode(gin.ReleaseMode) + e := gin.New() + e.Use( + sloggin.NewWithFilters( + slog.Default(), + sloggin.IgnorePath("/health", "/favicon.ico")), + gin.Recovery(), + ) + + e.Use(func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, ResponseType, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + c.Next() + }) + + server := &Server{ + e: e, + } + + if err := server.buildRouter(ctx); err != nil { + slog.Error("error building the server's routes") + } + + return server + +} + +func (s *Server) buildRouter(ctx context.Context) error { + s.e.GET("/health", s.HealthCheck) + s.e.GET("urns/:urn", s.GetDocument) + // s.e.GET("urns/:urn/latest", s.GetDocument) + return nil +} + +func (s *Server) Run(ctx context.Context, port string) error { + return s.e.Run(port) +} + +func (s *Server) HealthCheck(c *gin.Context) { + c.Status(http.StatusOK) +} + +func (s *Server) GetDocument(c *gin.Context) { + requested_urn := c.Param("urn") + u, ok := urn.Parse(requested_urn) + if !ok { + slog.Error("requested payload for malformed urn", "urn", requested_urn) + c.JSON(http.StatusBadRequest, gin.H{ + "error": "invalid urn", + }) + return + } + + doc, err := rdt.GetDocument(context.Background(), u) + if err != nil { + if err == rdt.ErrDocumentNotFound { + c.Status(http.StatusNotFound) + return + } else if err == rdt.ErrBadURN { + slog.Error("requested payload for malformed urn", "urn", requested_urn) + c.JSON(http.StatusBadRequest, gin.H{ + "error": "invalid urn", + }) + return + } + slog.Error("error getting raw data document", "urn", requested_urn, "err", err) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "failed to get document", + }) + return + } + c.JSON(http.StatusOK, doc) +} diff --git a/infrastructure/raw-writer/atmically.go b/infrastructure/raw-writer/atomically.go similarity index 100% rename from infrastructure/raw-writer/atmically.go rename to infrastructure/raw-writer/atomically.go diff --git a/infrastructure/raw-writer/go.mod b/infrastructure/raw-writer/go.mod index 4306561..f103cae 100644 --- a/infrastructure/raw-writer/go.mod +++ b/infrastructure/raw-writer/go.mod @@ -6,8 +6,9 @@ require ( github.com/ThreeDotsLabs/watermill v1.3.5 github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.2 github.com/kelseyhightower/envconfig v1.4.0 - github.com/rabbitmq/amqp091-go v1.9.0 - go.mongodb.org/mongo-driver v1.16.0 + github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a + github.com/rabbitmq/amqp091-go v1.10.0 + go.mongodb.org/mongo-driver v1.17.1 ) require ( @@ -16,7 +17,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -24,7 +26,7 @@ require ( github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/infrastructure/raw-writer/go.sum b/infrastructure/raw-writer/go.sum index 7ee6d13..70edf84 100644 --- a/infrastructure/raw-writer/go.sum +++ b/infrastructure/raw-writer/go.sum @@ -4,7 +4,6 @@ github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.2 h1:80CPTETpNm2BYaxK0Ru0Go6MNok github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.2/go.mod h1:MCNoh0HUg4w0bY64on9BnhUodHeimz8+vMfXrzyuWN8= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -21,27 +20,24 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a h1:cmntXRjN/lCskoocnKXZ4gtzbyRZh4E4dF2i24Av/nc= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a/go.mod h1:i/2Gbagc/PBHZD7pjW/22WHvc9MBkhJ9RhVhJ/s9h3c= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= -github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -50,13 +46,13 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= -go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= @@ -86,8 +82,5 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infrastructure/raw-writer/main.go b/infrastructure/raw-writer/main.go index d807f79..fee485e 100644 --- a/infrastructure/raw-writer/main.go +++ b/infrastructure/raw-writer/main.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "log/slog" + "net/url" "os" "reflect" "strings" @@ -16,6 +17,7 @@ import ( "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" "github.com/ThreeDotsLabs/watermill/message" "github.com/kelseyhightower/envconfig" + "github.com/noi-techpark/go-opendatahub-ingest/urn" "github.com/rabbitmq/amqp091-go" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -149,10 +151,19 @@ func handleMqMsg(msg *message.Message) *mqErr { body["bsontimestamp"] = timestamp // parse provier string to retrieve mongo db and collection names - db, coll, err := parseProvider(body) + provider, err := getProvider(body) if err != nil { return NewMqErr(err.Error(), "json", body) } + db, coll, err := constructTableTarget(provider) + if err != nil { + return NewMqErr(err.Error(), "json", body) + } + + u, ok := urn.RawUrnFromProviderURI(provider) + if !ok { + return NewMqErr("provider generated invalid urn", "json", body) + } inserted_id := primitive.ObjectID{} atom := NewAtom( @@ -167,10 +178,13 @@ func handleMqMsg(msg *message.Message) *mqErr { }), NewQuark( func() (interface{}, error) { + // concat document id to urn + u.AddNSS(inserted_id.Hex()) messagePayload, err := json.Marshal(map[string]any{ "id": inserted_id.Hex(), "db": db, "collection": coll, + "urn": u.String(), }) if err != nil { return nil, err @@ -201,17 +215,26 @@ func parseTimestamp(body *map[string]any) (time.Time, error) { return t, nil } -func parseProvider(body map[string]any) (string, string, error) { +func getProvider(body map[string]any) (string, error) { // provider string is in format "db/collection" providerstr, ok := body["provider"] if !ok || reflect.TypeOf(providerstr).Kind() != reflect.String { - return "", "", errors.New("provider missing or wrong type") + return "", errors.New("provider missing or wrong type") } - provider := strings.SplitN(providerstr.(string), "/", 2) - if provider == nil || len(provider) != 2 { - return "", "", fmt.Errorf("provider format invalid: %s", providerstr) + uri, err := url.ParseRequestURI("opendatahub:" + providerstr.(string)) + if err != nil { + return "", err + } + + return uri.Opaque, nil +} + +func constructTableTarget(provider string) (string, string, error) { + provider_tokens := strings.Split(provider, "/") + if len(provider_tokens) < 2 { + return "", "", fmt.Errorf("provider format invalid: %s", provider) } - return provider[0], provider[1], nil + return provider_tokens[0], strings.Join(provider_tokens[1:], "."), nil } func initMongo() *mongo.Client { diff --git a/infrastructure/router/go.mod b/infrastructure/router/go.mod index 68a795f..0012ab5 100644 --- a/infrastructure/router/go.mod +++ b/infrastructure/router/go.mod @@ -14,7 +14,9 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pkg/errors v0.9.1 // indirect ) diff --git a/infrastructure/router/go.sum b/infrastructure/router/go.sum index 8d78986..9d35f44 100644 --- a/infrastructure/router/go.sum +++ b/infrastructure/router/go.sum @@ -1,14 +1,9 @@ -github.com/ThreeDotsLabs/watermill v1.3.5 h1:50JEPEhMGZQMh08ct0tfO1PsgMOAOhV3zxK2WofkbXg= -github.com/ThreeDotsLabs/watermill v1.3.5/go.mod h1:O/u/Ptyrk5MPTxSeWM5vzTtZcZfxXfO9PK9eXTYiFZY= github.com/ThreeDotsLabs/watermill v1.4.3 h1:cRT1v7jlAgoPyEknvz0IFp3EKdSBRD/0Qbtz6KhexG8= github.com/ThreeDotsLabs/watermill v1.4.3/go.mod h1:lBnrLbxOjeMRgcJbv+UiZr8Ylz8RkJ4m6i/VN/Nk+to= -github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.2 h1:80CPTETpNm2BYaxK0Ru0Go6MNokhiioYYstxeVWJoJI= -github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.2/go.mod h1:MCNoh0HUg4w0bY64on9BnhUodHeimz8+vMfXrzyuWN8= github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.3 h1:fkhmiBtaLn+rz5lbkPD1h8tXHfKy3gX0vMtGmxNtAsk= github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.3/go.mod h1:xy2qXKcJpgrJURRT6YwgRyGL3qIi6/sOHrDI0MO/r5I= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -21,33 +16,23 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a h1:cmntXRjN/lCskoocnKXZ4gtzbyRZh4E4dF2i24Av/nc= +github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20250127084305-71be9a08639a/go.mod h1:i/2Gbagc/PBHZD7pjW/22WHvc9MBkhJ9RhVhJ/s9h3c= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= -github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infrastructure/router/main.go b/infrastructure/router/main.go index c3c344f..d20c61d 100644 --- a/infrastructure/router/main.go +++ b/infrastructure/router/main.go @@ -9,11 +9,13 @@ import ( "log/slog" "os" "reflect" + "strings" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" "github.com/ThreeDotsLabs/watermill/message" "github.com/kelseyhightower/envconfig" + "github.com/noi-techpark/go-opendatahub-ingest/urn" "github.com/rabbitmq/amqp091-go" ) @@ -155,17 +157,18 @@ func handleMq(messages <-chan *message.Message) { } func generateRoutingKey(body map[string]any) (string, error) { - // body must include "db" and "collection" field and we construct routing key - // as db.collection - db, ok := body["db"] - if !ok || reflect.TypeOf(db).Kind() != reflect.String { - return "", errors.New("'db' payload field missing or wrong type") + // body must include "urn" field and we construct routing key + // NSS without last part replacing : -> . + body_urn, ok := body["urn"] + if !ok || reflect.TypeOf(body_urn).Kind() != reflect.String { + return "", errors.New("'urn' payload field missing or wrong type") } - collection, ok := body["collection"] - if !ok || reflect.TypeOf(collection).Kind() != reflect.String { - return "", errors.New("'collection' payload field missing or wrong type") + u, ok := urn.Parse(body_urn.(string)) + if !ok { + return "", errors.New("invalid 'urn' format") } - return fmt.Sprintf("%s.%s", db, collection), nil + + return strings.Join(u.GetNSSWithoutID(), "."), nil } func handleMqMsg(msg *message.Message) *mqErr {