Merge pull request #37 from samply/develop

Release version v0.1.1

.github/workflows/rust.yml

@@ -22,13 +22,16 @@ jobs:
 
   build-rust:
     name: Build (Rust)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     strategy:
       matrix:
         arch:
           - amd64
           - arm64
+        features:
+          - sockets
+          - ""
 
     steps:
       - name: Set arch ${{ matrix.arch }}
@@ -61,44 +64,76 @@ jobs:
             echo "profilestr=--profile $PROFILE" >> $GITHUB_ENV
           fi
       - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v2
-        with:
-          key: ${{ matrix.arch }}-${{ env.PROFILE }}
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           override: true
           target: ${{ env.rustarch }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          key: ${{ matrix.arch }}-${{ env.PROFILE }}
+          prefix-key: v1-rust-${{ matrix.features && format('features_{0}', matrix.features) || 'nofeatures' }} # Increase to invalidate old caches.
       - name: Build (${{ matrix.arch }})
         uses: actions-rs/cargo@v1
         with:
           use-cross: ${{ env.is_cross }}
           command: build
-          args: --target ${{ env.rustarch }} ${{ env.profilestr }}
+          args: --target ${{ env.rustarch }} ${{ matrix.features && format('--features {0}', matrix.features) }} ${{ env.profilestr }}
       - name: Upload Artifact
         uses: actions/upload-artifact@v3
         with:
-          name: binaries-${{ matrix.arch }}
+          name: binaries-${{ matrix.arch }}-${{ matrix.features }}
           path: |
             target/${{ env.rustarch }}/${{ env.PROFILE }}/connect
 
   test:
     name: Run tests
     needs: [ build-rust ]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
+
+    strategy:
+      matrix:
+        features:
+          - ""
+          - "sockets"
 
     steps:
-      - name: Not implemented yet
-        run: echo "This will be implemented soonish"
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: binaries-amd64-${{ matrix.features }}
+          path: artifacts/binaries-amd64/
+      - name: Set diffrent image tag
+        run: |
+          if [[ ${{ format('"{0}"', matrix.features) }} == 'sockets' ]]; then
+            echo "TAG=develop-sockets" >> $GITHUB_ENV
+          fi
+      - name: Start containers
+        run: ./dev/start ci
+      # - name: Show logs
+      #   working-directory: ./dev
+      #   run: |
+      #     sleep 3
+      #     docker compose logs
+      - name: Run tests
+        run: cargo test ${{ format('--features "{0}"', matrix.features) }}
 
   docker:
     needs: [ build-rust, pre-check, test ]
+
+    strategy:
+      matrix:
+        features:
+          - ""
+          - "sockets"
+
     # This workflow defines how a maven package is built, tested and published.
     # Visit: https://github.com/samply/github-workflows/blob/develop/.github/workflows/docker-ci.yml, for more information
     uses: samply/github-workflows/.github/workflows/docker-ci.yml@main
     with:
       # The Docker Hub Repository you want eventually push to, e.g samply/share-client
       image-name: "samply/beam-connect"
+      image-tag-suffix: ${{ matrix.features && format('-{0}', matrix.features) }}
       # Define special prefixes for docker tags. They will prefix each images tag.
       # image-tag-prefix: "foo"
       # Define the build context of your image, typically default '.' will be enough
@@ -107,9 +142,8 @@ jobs:
       build-file: './Dockerfile.ci'
       # NOTE: This doesn't work currently
       # A list of build arguments, passed to the docker build
-#      build-args: |
-#        PROFILE=${{ env.PROFILE }}
-#        COMPONENT=broker
+      build-args: |
+        FEATURE=-${{ matrix.features }}
       # Define the target platforms of the docker build (default "linux/amd64,linux/arm64/v8")
       # build-platforms: "linux/amd64,linux/arm64"
       # If your actions generate an artifact in a previous build step, you can tell this workflow to download it

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "connect"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 license = "Apache-2.0"
 
@@ -18,30 +18,36 @@ inherits = "release"
 strip = false
 
 [dependencies]
-shared = { git = "https://github.com/samply/beam", branch="develop" }
+beam-lib = { git = "https://github.com/samply/beam", branch="develop", features = ["strict-ids"] }
 
-#axum = "0.5.12"
 tokio = { version = "1", features = ["macros","rt-multi-thread","signal"] }
-hyper = { version = "0", features = ["full"] }
-tower-http = { version = "0", features = ["trace"] }
-tower = "*"
+hyper = { version = "0.14", features = ["full"] }
 
 # HTTP client with proxy support
-hyper-tls = "0.5.0"
-hyper-proxy = "0.9.1"
-mz-http-proxy = { version = "0.1.0", features = ["hyper"] }
+reqwest = { version = "0.11.19", features = ["json", "stream"] }
 
-log = "*"
-pretty_env_logger = "*"
+tracing = "0.1"
 
 serde = "*"
 serde_json = "*"
-hyper_serde = "0.13"
 
-clap = {version = "4", features = ["derive"]}
+clap = { version = "4", features = ["derive", "env"] }
 
 thiserror = "*"
-http-serde = "1.1.2"
+http-serde = "1.1"
+tokio-native-tls = "0.3.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+anyhow = "1"
+openssl = "*" # Already used by native_tls which does not reexport it. This is used for b64 en/decode
+
+[features]
+sockets = ["beam-lib/sockets"]
 
 [build-dependencies]
 build-data = "0"
+
+[dev-dependencies]
+once_cell = "1"
+futures-util = "0.3.28"
+paste = "1.0.12"
+tokio-tungstenite = "0.20.0"

Cross.toml

@@ -1,5 +1,5 @@
 [target.aarch64-unknown-linux-gnu]
-image = "ghcr.io/lablans/cross-test:aarch64-unknown-linux-gnu"
+pre-build = ["dpkg --add-architecture arm64 && apt-get update && apt-get install --assume-yes libssl-dev:arm64 && rm -rf /var/lib/apt/lists/*"]
 
 [target.x86_64-unknown-linux-gnu]
-image = "ghcr.io/lablans/cross-test:x86_64-unknown-linux-gnu"
+pre-build = ["dpkg --add-architecture amd64 && apt-get update && apt-get install --assume-yes libssl-dev:amd64 && rm -rf /var/lib/apt/lists/*"]

Dockerfile.ci

@@ -3,11 +3,17 @@
 ARG IMGNAME=gcr.io/distroless/cc
 
 FROM alpine AS chmodder
+ARG FEATURE
 ARG TARGETARCH
-COPY /artifacts/binaries-$TARGETARCH/connect /app/
+COPY /artifacts/binaries-$TARGETARCH$FEATURE/connect /app/
 RUN chmod +x /app/*
 
-FROM ${IMGNAME}
+# FROM ${IMGNAME}
+FROM ubuntu:latest
+RUN apt update
+RUN apt install -y ca-certificates
+RUN apt install -y ssl-cert
+RUN make-ssl-cert generate-default-snakeoil
 #ARG COMPONENT
 ARG TARGETARCH
 #COPY /artifacts/binaries-$TARGETARCH/$COMPONENT /usr/local/bin/

README.md

@@ -34,14 +34,17 @@ The following command line parameters are required:
  * `PROXY_URL`: The URL of the local Samply.Proxy which is used to connect to the Samply.Broker
  * `APP_ID`: The BeamId of the Beam.Connect application 
  * `LOCAL_TARGETS_FILE`: The path to the local service resolution file (see [Routing Section](#Request-Routing)).
- * `DISCOVERY_URL`: The URL that is queried to receive the central service discovery this may also be a local file (see [Routing Section](#Request-Routing)).
+ * `DISCOVERY_URL`: The URL (or local file) to be is queried to receive the central service discovery (see [Routing Section](#Request-Routing)).
 
 The following command line parameter is only used in Receiver mode (see [Usage Section](#usage)):
  * `PROXY_APIKEY`: In Receiver Mode, the API key with which this Beam.Connector is registered for listening at the Samply.Broker
 
 The following command line parameter is optional, as it uses a default value:
  * `BIND_ADDR`: The interface and port Beam.Connect is listening on. Defaults to `0.0.0.0:8062`.
 
+If the following flag is optional.
+ * `NO_AUTH`:  Samply.Beam.Connect does not require a `Proxy Authorization` header, i.e. it forwards requests without (client) authentication
+
 All parameters can be given as environment variables instead.
 
 ### Run using Docker
@@ -58,9 +61,10 @@ docker run -e PROXY_URL='<PROXY_URL>' \
            -e DISCOVERY_URL='<DISCOVERY_URL>' \
            -e PROXY_APIKEY='<PROXY_APIKEY>' \
            -e BIND_ADDR='<BIND_ADDR>' \
+           -e NO_AUTH='true' \
            samply/beam-connect
 ```
-Again, the last environment variable `PROXY_APIKEY` is only required for usage in Receiver Mode and `BIND_ADDR` is optional.
+Again, the environment variable `PROXY_APIKEY` is only required for usage in Receiver Mode. `BIND_ADDR` and `NO_AUTH` are optional.
 
 ### Use Beam.Connect to forward a HTTP request
 We give an example [cURL](https://curl.se/) request showing the usage of Beam.Connect to access an internal service within University Hospital #23 (`uk23`):
@@ -87,7 +91,14 @@ A mishap in communication will be returned as appropriate HTTP replies.
 
 As described in the [command line parameter list](#run-as-an-application), the central cite discovery is fetched from a given URL or local json file. However, to spare the local services from the need to express outward facing connections themselves, Samply.Beam.Connect exports this received information as a local REST endpoint: `GET http://<beam_connect_url>:<beam_connect_port>/sites`. Note, that the information is only fetched at startup and remains static for the program's lifetime.
 
+#### HTTPS support
+
+Https is supported but requires setting up the following parameters:
+* `SSL_CERT_PEM`: Location to the pem file used for incoming SSL connections.
+* `SSL_CERT_KEY`: Location to the corresponding key file for the SSL connections.
+* `TLS_CA_CERTIFICATES_DIR`: May need to be set if the local target uses a self signed certificate which is not trusted by beam-connect. In this case the certificate of the target must be placed inside `TLS_CA_CERTIFICATES_DIR` as a pem file in order to be trusted.
+
 ## Notes
-At the moment Samply.Beam.Connect does not implement streaming and does not support HTTPS connections. In the intended usage scenario, both Samply.Beam.Connect and Samply.Beam.Proxy are positioned right next to each other in the same privileged network and thus speak plain HTTP. Of course, for outgoing traffic, the Samply.Proxy signs and encrypts the payloads on its own.
+At the moment Samply.Beam.Connect does not implement streaming. In the intended usage scenario, both Samply.Beam.Connect and Samply.Beam.Proxy are positioned right next to each other in the same privileged network and thus speak plain HTTP or [HTTPS if configured](#https). Of course, for outgoing traffic, the Samply.Proxy signs and encrypts the payloads on its own.
 
 In Receiving Mode, Beam.Connect only relays requests to allow-listed resources to mitigate possible misuse.

dev/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.7"
 services:
   vault:
-    image: vault
+    image: hashicorp/vault
     ports:
       - 127.0.0.1:8200:8200
     environment:
@@ -21,7 +21,6 @@ services:
       BROKER_URL: ${BROKER_URL}
       PKI_ADDRESS: http://vault:8200
       no_proxy: vault
-      NO_PROXY: vault
       PRIVKEY_FILE: /run/secrets/dummy.pem
       BIND_ADDR: 0.0.0.0:8080
       RUST_LOG: ${RUST_LOG}
@@ -37,12 +36,10 @@ services:
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY1_ID}
-      APP_0_ID: ${APP1_ID_SHORT}
-      APP_0_KEY: ${APP_KEY}
+      APP_app1_KEY: ${APP_KEY}
       PRIVKEY_FILE: /run/secrets/proxy1.pem
       BIND_ADDR: 0.0.0.0:8081
       RUST_LOG: ${RUST_LOG}
-      NO_PROXY: broker
       no_proxy: broker
     secrets:
       - proxy1.pem
@@ -53,7 +50,6 @@ services:
     build:
       context: ../
       dockerfile: Dockerfile.ci
-    image: samply/beam-connect:${TAG}
     ports:
       - 8062:8062
     volumes:
@@ -64,19 +60,18 @@ services:
       PROXY_APIKEY: ${APP_KEY}
       DISCOVERY_URL: "./map/example_central_test.json"
       RUST_LOG: ${RUST_LOG}
-      NO_PROXY: proxy1
-      no_proxy: proxy1
+      no_proxy: proxy1,my.example.com
   connect2:
    depends_on:
      - proxy2
    build:
      context: ../
      dockerfile: Dockerfile.ci
-   image: samply/beam-connect:${TAG}
    ports:
      - 8063:8063
    volumes:
      - ../examples/:/map
+     - ssl-cert:/custom-cert
    environment:
      PROXY_URL: "http://proxy2:8082"
      BIND_ADDR: 0.0.0.0:8063
@@ -85,8 +80,8 @@ services:
      DISCOVERY_URL: "./map/example_central_test.json"
      LOCAL_TARGETS_FILE: "./map/example_local_test.json"
      RUST_LOG: ${RUST_LOG}
-     NO_PROXY: proxy2
-     no_proxy: proxy2
+     no_proxy: proxy2,my.example.com
+     TLS_CA_CERTIFICATES_DIR: /custom-cert
   proxy2:
     depends_on: [broker]
     image: samply/beam-proxy:${TAG}
@@ -95,16 +90,28 @@ services:
     environment:
       BROKER_URL: ${BROKER_URL}
       PROXY_ID: ${PROXY2_ID}
-      APP_0_ID: ${APP2_ID_SHORT}
-      APP_0_KEY: ${APP_KEY}
+      APP_app2_KEY: ${APP_KEY}
       PRIVKEY_FILE: /run/secrets/proxy2.pem
       BIND_ADDR: 0.0.0.0:8082
       RUST_LOG: ${RUST_LOG}
-      NO_PROXY: broker
       no_proxy: broker
     secrets:
       - proxy2.pem
       - root.crt.pem
+  ws-echo:
+    image: jmalloc/echo-server
+    environment:
+      PORT: 80
+  echo:
+    image: mendhak/http-https-echo
+    container_name: my.example.com # We set this so that we can connect via this common name so that the ssl cert CN matches
+    environment:
+        - HTTP_PORT=80
+        - HTTPS_PORT=443
+    volumes:
+      - ssl-cert:/app/custom-cert
+    entrypoint: ["sh", "-c", "cp ./fullchain.pem ./custom-cert/cert.pem && node ./index.js"]
+    user: "0:0"
 secrets:
   pki.secret:
     file: ./pki/pki.secret
@@ -116,3 +123,6 @@ secrets:
     file: ./pki/dummy.priv.pem
   root.crt.pem:
     file: ./pki/root.crt.pem
+
+volumes:
+  ssl-cert: