feat(bridge): created a rest server service to abstract the raw data …

…retrival

refactor: usage of standardized urn to identify raw resources. aligned router and writer

.github/workflows/odh-v2-core-bridge.yaml

@@ -0,0 +1,58 @@
+name: CI/CD odh-v2-core-writer
+
+on: 
+  push:
+    paths:
+      - "infrastructure/raw-data-bridge/**"      
+      - "infrastructure/helm/raw-data-bridge/**"
+      - ".github/workflows/odh-v2-core-writer.yaml"
+  workflow_dispatch:
+
+env:
+  WORKING_DIRECTORY: infrastructure/raw-data-bridge
+  KUBERNETES_NAMESPACE: core
+  KUBERNETES_VALUE_PATH: infrastructure/helm/raw-data-bridge/values.yaml
+  KUBERNETES_CHART_PATH: infrastructure/helm/raw-data-bridge/raw-data-bridge
+  PROJECT_NAME: raw-data-bridge
+
+jobs:
+  build:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v4
+
+    - name: Log in to GitHub Docker Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Manually build and push image
+      run: cd ${{ env.WORKING_DIRECTORY }} && ./build.sh
+
+  deploy:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/prod'
+    needs: 
+      - build
+    runs-on: ubuntu-22.04
+    concurrency: odh-v2-core-writer-build-deploy
+    environment: ${{ github.ref == 'refs/heads/main' && 'test' || github.ref == 'refs/heads/prod' && 'prod' || 'unknown' }}
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Deploy on cluster
+        uses: noi-techpark/github-actions/helm-deploy@v2
+        with:
+          k8s-name: ${{ env.PROJECT_NAME }}
+          k8s-namespace: ${{ env.KUBERNETES_NAMESPACE }}
+          chart-path: ${{ env.KUBERNETES_CHART_PATH }}
+          values-file: ${{ env.KUBERNETES_VALUE_PATH }}
+          aws-access-key-id: ${{ secrets[vars.AWS_KEY_ID] }}
+          aws-secret-access-key: ${{ secrets[vars.AWS_KEY_SECRET] }}
+          aws-eks-cluster-name: aws-main-eu-01
+          aws-region: eu-west-1

.vscode/launch.json

@@ -19,6 +19,14 @@
             "mode": "debug",
             "program": "${workspaceFolder}/infrastructure/router",
             "envFile": "${workspaceFolder}/infrastructure/router/.env"
+        },
+        {
+            "name": "Launch Lookup",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "program": "${workspaceFolder}/infrastructure/raw-data-bridge",
+            "envFile": "${workspaceFolder}/infrastructure/raw-data-bridge/.env"
         }
     ]
 }

infrastructure/helm/raw-data-bridge/raw-data-bridge/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

infrastructure/helm/raw-data-bridge/raw-data-bridge/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: raw-data-bridge
+description: Resource Lookup service
+type: application
+version: 1.0.0
+appVersion: "1.0.0"

infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/_helpers.tpl

@@ -0,0 +1,51 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "raw-data-bridge.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "raw-data-bridge.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "raw-data-bridge.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "raw-data-bridge.labels" -}}
+helm.sh/chart: {{ include "raw-data-bridge.chart" . }}
+{{ include "raw-data-bridge.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "raw-data-bridge.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "raw-data-bridge.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/deployment.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "raw-data-bridge.fullname" . }}
+  labels:
+    {{- include "raw-data-bridge.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "raw-data-bridge.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "raw-data-bridge.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          {{- with .Values.env }}
+          env:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: {{ .Values.health.delay }}
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: {{ .Values.health.delay }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

infrastructure/helm/raw-data-bridge/raw-data-bridge/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "raw-data-bridge.fullname" . }}
+  labels:
+    {{- include "raw-data-bridge.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "raw-data-bridge.selectorLabels" . | nindent 4 }}

infrastructure/helm/raw-data-bridge/raw-data-bridge/values.yaml

@@ -0,0 +1,32 @@
+# Default values for raw-data-bridge.
+replicaCount: 1
+
+image:
+  repository: ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: "latest"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+service:
+  port: 2000
+  type: ClusterIP
+
+health:
+  delay: 30
+
+env:
+  - name: APP_MONGO_URI
+    value: mongodb://user:password@mongodb
+  - name: APP_LOGLEVEL
+    value: INFO
+
+podAnnotations: {}
+podLabels: {}
+resources: {}
+nodeSelector: {}
+tolerations: []
+affinity: {}

infrastructure/helm/raw-data-bridge/values.yaml

@@ -0,0 +1,18 @@
+image:
+  repository: ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge
+  pullPolicy: Always
+  tag: "v1.0"
+
+env:
+  - name: APP_MONGO_URI
+    value: mongodb://mongodb-headless:27017/
+  - name: APP_LOGLEVEL
+    value: INFO
+
+# resources:
+#   limits:
+#     cpu: 250m   # 250m is equivalent to 0.25 CPU core
+#     memory: 500Mi
+#   requests:
+#     cpu: 100m   # 100m is equivalent to 0.1 CPU core
+#     memory: 100Mi

infrastructure/raw-data-bridge/.env.example

@@ -0,0 +1,3 @@
+APP_MONGO_URI=mongodb://mongodb
+
+APP_LOGLEVEL=DEBUG

infrastructure/raw-writer/Dockerfile → infrastructure/raw-data-bridge/Dockerfile

@@ -2,18 +2,18 @@
 #
 # SPDX-License-Identifier: CC0-1.0
 
-FROM golang:1.22.2-bookworm AS base
+FROM golang:1.23.4-bookworm AS base
 
 FROM base AS build
 WORKDIR /app
 COPY go.mod .
 COPY go.sum .
-COPY *.go .
-RUN CGO_ENABLED=0 GOOS=linux go build -o writer
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -o raw-data-bridge
 
 FROM scratch AS release
-COPY --from=build /app/writer /writer
-CMD ["/writer"]
+COPY --from=build /app/raw-data-bridge /raw-data-bridge
+CMD ["/raw-data-bridge"]
 
 
 FROM base AS dev

infrastructure/raw-writer/build.sh → infrastructure/raw-data-bridge/build.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
-IMAGE=ghcr.io/noi-techpark/infrastructure-v2/raw-writer
+IMAGE=ghcr.io/noi-techpark/infrastructure-v2/raw-data-bridge
 TAGS='latest v1 v1.0'
 REPO=https://github.com/noi-techpark/infrastructure-v2
 docker buildx build --target release -f Dockerfile . -t $IMAGE \
 --label "org.opencontainers.image.source=$REPO" \
---label "org.opencontainers.image.description=Raw data writer service for Open Data Hub ingestion pipeline" \
+--label "org.opencontainers.image.description=Resource Lookup service to retrieve data from Raw Data Table" \
 --label "org.opencontainers.image.licenses=AGPL-3.0-or-later"
 
 for tag in $TAGS

infrastructure/raw-writer/docker-compose.yml → infrastructure/raw-data-bridge/docker-compose.yml

@@ -11,16 +11,8 @@ services:
       - pkg:/go/pkg/mod
     working_dir: /code
     depends_on:
-      - rabbitmq
       - mongodb
 
-  rabbitmq:
-    image: rabbitmq:management-alpine
-    ports:
-      - "5672:5672"
-      - "15672:15672"
-    attach: false
-
   mongodb:
     image: mongo
     ports: