Merge pull request #10 from SB-IM/release/0.1.0

Release/0.1.0: WebRTC broadcasting

.dockerignore

@@ -0,0 +1,3 @@
+.github
+.idea
+config

.github/workflows/build.yaml

@@ -14,12 +14,18 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
         with:
-          images: ${{ secrets.DOCKERHUB_USERNAME }}/orchid
-          tag-sha: true
-          tag-custom: latest
+          images: ghcr.io/sb-im/skywalker
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
+          flavor: |
+            latest=true
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -35,14 +41,17 @@ jobs:
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
-          username: ${{ secrets.GITHUB_USERNAME }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.CONTAINER_USER }}
+          password: ${{ secrets.CONTAINER_TOKEN }}
+      - name: Prepare ssh key
+        run: echo "${{ secrets.SSH_KEY }}" > ssh-key
       - name: Build and push
         uses: docker/build-push-action@v2
         with:
-          platforms: linux/amd64,linux/arm64
+          ssh: github=ssh-key
+          file: docker/Dockerfile
           push: true
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

.gitignore

@@ -1,3 +1,5 @@
 .idea
 config/config.dev.yaml
 .env
+config/config.docker.yaml
+skywalker

Makefile

@@ -1,16 +1,23 @@
+# Project name.
+PROJECT_NAME:=skywalker
+
 # SSH private key set up.
 CURRENT_USER?=william
 PRIVATE_KEY_FILE?=id_ed25519
 PRIVATE_KEY_PATH?=github=/home/$(CURRENT_USER)/.ssh/$(PRIVATE_KEY_FILE)
-PROJECT_DIR?=/home/${CURRENT_USER}/go/src/github.com/SB-IM/skywalker
+PROJECT_DIR?=/home/$(CURRENT_USER)/go/src/github.com/SB-IM/skywalker
 
 # Project image repo.
-IMAGE_BROADCAST?=ghcr.io/sb-im/skywalker:broadcast-latest
+IMAGE?=ghcr.io/sb-im/skywalker:latest-dev
 
 .PHONY: run
 run:
 	@DEBUG_MQTT_CLIENT=false go run -race ./cmd --debug broadcast -c config/config.dev.yaml
 
+.PHONY: build
+build:
+	@go build -race -o $(PROJECT_NAME) ./cmd
+
 .PHONY: lint
 lint:
 	@golangci-lint run ./...
@@ -19,21 +26,25 @@ lint:
 image:
 	@docker build \
 	--ssh $(PRIVATE_KEY_PATH) \
-	-t $(IMAGE_BROADCAST) \
-	-f docker/Dockerfile.broadcast.dev .
+	-t $(IMAGE) \
+	-f docker/Dockerfile.dev .
+
+.PHONY: push
+push:
+	@docker push $(IMAGE)
 
 # Note: '--env-file' value is relative to '-f' value's directory.
 .PHONY: up
-up: down
-	@docker-compose -f docker/docker-compose.dev.yaml up --build -d
+up: down image
+	@docker-compose -f docker/docker-compose.dev.yaml up -d
 
 .PHONY: down
 down:
-	@docker-compose -f docker/docker-compose.dev.yaml down
+	@docker-compose -f docker/docker-compose.dev.yaml down --remove-orphans
 
 .PHONY: broker
 broker:
-	@docker run -d --rm --name mosquitto -p 1883:1883 -p 9001:9001 -v ${PROJECT_DIR}/config/mosquitto.conf:/mosquitto/config/mosquitto.conf eclipse-mosquitto:2
+	@docker run -d --rm --name mosquitto -p 1883:1883 -p 9001:9001 -v $(PROJECT_DIR)/config/mosquitto.conf:/mosquitto/config/mosquitto.conf eclipse-mosquitto:2
 
 .PHONY: stop-broker
 stop-broker:
@@ -50,3 +61,7 @@ log-livestream:
 .PHONY: log-broadcast
 log-broadcast:
 	@docker-compose -f docker/docker-compose.dev.yaml logs -f broadcast
+
+.PHONY: e2e-broadcast
+e2e-broadcast:
+	@go run ./e2e/broadcast

README.md

@@ -0,0 +1,30 @@
+# Skywalker
+
+Skywalker walks above the cloud.
+
+## What is Skywalker project?
+
+Skywalker is cloud service managing and controlling all edge devices, processing IOT data, etc. It also manages IOT users and provides friendly APIs to front-end.
+
+Currently, Skywalker includes:
+
+- `Broadcast`: forwards video streams from edge devices.
+
+## How to run?
+
+All sub-processes are executed through sub commands under `skywalker` command.
+
+Make sure you have the following tools installed:
+
+- `Go`
+- `GNU make`
+- `golangci-lint` (Optional)
+- `Docker`
+- `Docker-compose` (Optional)
+- `mosquitto` (Optional, can be on Docker)
+
+### Run `broadcast`
+
+```bash
+$ make
+```

cmd/broadcast/cmd.go

@@ -13,6 +13,7 @@ import (
 	"github.com/urfave/cli/v2/altsrc"
 
 	"github.com/SB-IM/skywalker/internal/broadcast"
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
 )
 
 // Command returns a broadcast command.
@@ -25,19 +26,19 @@ func Command() *cli.Command {
 
 		mc mqtt.Client
 
-		mqttConfigOptions     mqttclient.ConfigOptions
-		topicConfigOptions    broadcast.TopicConfigOptions
-		webRTCConfigOptions   broadcast.WebRTCConfigOptions
-		wsServerConfigOptions broadcast.WSServerConfigOptions
+		mqttConfigOptions       mqttclient.ConfigOptions
+		mqttClientConfigOptions cfg.MQTTClientConfigOptions
+		webRTCConfigOptions     cfg.WebRTCConfigOptions
+		serverConfigOptions     cfg.ServerConfigOptions
 	)
 
 	flags := func() (flags []cli.Flag) {
 		for _, v := range [][]cli.Flag{
 			loadConfigFlag(),
 			mqttFlags(&mqttConfigOptions),
-			topicFlags(&topicConfigOptions),
+			mqttClientFlags(&mqttClientConfigOptions),
 			webRTCFlags(&webRTCConfigOptions),
-			wsFlags(&wsServerConfigOptions),
+			serverFlags(&serverConfigOptions),
 		} {
 			flags = append(flags, v...)
 		}
@@ -71,10 +72,10 @@ func Command() *cli.Command {
 			return nil
 		},
 		Action: func(c *cli.Context) error {
-			svc := broadcast.New(ctx, broadcast.ConfigOptions{
-				WebRTCConfigOptions:   webRTCConfigOptions,
-				TopicConfigOptions:    topicConfigOptions,
-				WSServerConfigOptions: wsServerConfigOptions,
+			svc := broadcast.New(ctx, &cfg.ConfigOptions{
+				WebRTCConfigOptions:     webRTCConfigOptions,
+				MQTTClientConfigOptions: mqttClientConfigOptions,
+				ServerConfigOptions:     serverConfigOptions,
 			})
 			err := svc.Broadcast()
 			if err != nil {
@@ -135,7 +136,7 @@ func mqttFlags(mqttConfigOptions *mqttclient.ConfigOptions) []cli.Flag {
 	}
 }
 
-func topicFlags(topicConfigOptions *broadcast.TopicConfigOptions) []cli.Flag {
+func mqttClientFlags(topicConfigOptions *cfg.MQTTClientConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "topic-offer",
@@ -145,16 +146,44 @@ func topicFlags(topicConfigOptions *broadcast.TopicConfigOptions) []cli.Flag {
 			Destination: &topicConfigOptions.OfferTopic,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "topic-answer",
-			Usage:       "MQTT topic for WebRTC SDP answer signaling",
+			Name:        "topic-answer-suffix",
+			Usage:       "MQTT topic suffix for WebRTC SDP answer signaling",
 			Value:       "/edge/livestream/signal/answer",
 			DefaultText: "/edge/livestream/signal/answer",
-			Destination: &topicConfigOptions.AnswerTopic,
+			Destination: &topicConfigOptions.AnswerTopicSuffix,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "topic-candidate-send-suffix",
+			Usage:       "MQTT topic suffix for WebRTC candidate sending, and the sending topic of edge is /edge/livestream/signal/candidate/send",
+			Value:       "/edge/livestream/signal/candidate/recv",
+			DefaultText: "/edge/livestream/signal/candidate/recv",
+			Destination: &topicConfigOptions.CandidateSendTopicSuffix,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "topic-candidate-recv-suffix",
+			Usage:       "MQTT topic suffix for WebRTC candidate receiving, and the receiving topic of edge is /edge/livestream/signal/candidate/recv", //nolint:lll
+			Value:       "/edge/livestream/signal/candidate/send",
+			DefaultText: "/edge/livestream/signal/candidate/send",
+			Destination: &topicConfigOptions.CandidateRecvTopicSuffix,
+		}),
+		altsrc.NewUintFlag(&cli.UintFlag{
+			Name:        "qos",
+			Usage:       "MQTT client qos for WebRTC SDP signaling",
+			Value:       0,
+			DefaultText: "0",
+			Destination: &topicConfigOptions.Qos,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:        "retained",
+			Usage:       "MQTT client setting retainsion for WebRTC SDP signaling",
+			Value:       false,
+			DefaultText: "false",
+			Destination: &topicConfigOptions.Retained,
 		}),
 	}
 }
 
-func webRTCFlags(webRTCConfigOptions *broadcast.WebRTCConfigOptions) []cli.Flag {
+func webRTCFlags(webRTCConfigOptions *cfg.WebRTCConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "ice-server",
@@ -163,31 +192,45 @@ func webRTCFlags(webRTCConfigOptions *broadcast.WebRTCConfigOptions) []cli.Flag
 			DefaultText: "stun:stun.l.google.com:19302",
 			Destination: &webRTCConfigOptions.ICEServer,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "ice-server-username",
+			Usage:       "ICE server username",
+			Value:       "",
+			DefaultText: "",
+			Destination: &webRTCConfigOptions.Username,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "ice-server-credential",
+			Usage:       "ICE server credential",
+			Value:       "",
+			DefaultText: "",
+			Destination: &webRTCConfigOptions.Credential,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:        "enable-frontend",
+			Usage:       "Enable webRTC frontend server",
+			Value:       false,
+			DefaultText: "false",
+			Destination: &webRTCConfigOptions.EnableFrontend,
+		}),
 	}
 }
 
-func wsFlags(wsServerConfigOptions *broadcast.WSServerConfigOptions) []cli.Flag {
+func serverFlags(serverConfigOptions *cfg.ServerConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "ws-host",
-			Usage:       "Host of WebSocket server",
+			Name:        "host",
+			Usage:       "Host of webRTC signaling server",
 			Value:       "0.0.0.0",
 			DefaultText: "0.0.0.0",
-			Destination: &wsServerConfigOptions.Host,
+			Destination: &serverConfigOptions.Host,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:        "ws-port",
-			Usage:       "Port of WebSocket server",
+			Name:        "port",
+			Usage:       "Port of webRTC signaling server",
 			Value:       8080,
 			DefaultText: "8080",
-			Destination: &wsServerConfigOptions.Port,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "ws-path",
-			Usage:       "HTTP path of broadcast service",
-			Value:       "/ws/webrtc",
-			DefaultText: "/ws/webrtc",
-			Destination: &wsServerConfigOptions.Path,
+			Destination: &serverConfigOptions.Port,
 		}),
 	}
 }

cmd/main.go

@@ -17,7 +17,7 @@ func init() {
 
 func main() {
 	if err := run(os.Args); err != nil {
-		log.Fatal().Err(err)
+		log.Err(err).Send()
 	}
 }
 

config/config.example.yaml

@@ -4,14 +4,22 @@ mqtt-clientID: mqtt_edge
 mqtt-username: user
 mqtt-password: password
 
-# MQTT topics  for webRTC signaling config
+# MQTT client config for webRTC signaling
 topic-offer: /edge/livestream/signal/offer
-topic-answer: /edge/livestream/signal/answer
+topic-answer-suffix: /edge/livestream/signal/answer
+# topic-candidate-send is opposite to edge's topic-candidate-recv-suffix topic
+topic-candidate-send-suffix: /edge/livestream/signal/candidate/recv
+# topic-candidate-recv-suffix is opposite to edge's topic-candidate-send-suffix topic
+topic-candidate-recv-suffix: /edge/livestream/signal/candidate/send
+qos: 0
+retained: false
 
 # WebRTC config
 ice-server: stun:stun.l.google.com:19302
+ice-server-username: ""
+ice-server-credential: ""
+enable-frontend: false
 
-# WebSocket config
-ws-host: 0.0.0.0
-ws-port: 8080
-ws-path: /ws/webrtc
+# Signal server config
+host: 0.0.0.0
+port: 8080

docker/Dockerfile.broadcast → docker/Dockerfile

@@ -23,17 +23,19 @@ RUN --mount=type=ssh,id=github git config --global url."git@github.com:".instead
 
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=linux go build -tags timetzdata -ldflags="-w -s" -a -installsuffix cgo -o broadcast ./cmd
+RUN CGO_ENABLED=0 GOOS=linux go build -tags timetzdata -ldflags="-w -s" -a -installsuffix cgo -o skywalker ./cmd
 
 FROM scratch
 
 COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=0 /src/broadcast /
+COPY --from=0 /src/skywalker /
 
 ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
-ENTRYPOINT [ "/broadcast" ]
+EXPOSE 8080
+
+ENTRYPOINT [ "/skywalker" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]

docker/Dockerfile.broadcast.dev → docker/Dockerfile.dev

@@ -17,12 +17,14 @@ RUN --mount=type=ssh,id=github git config --global url."git@github.com:".instead
 
 COPY . .
 
-RUN GOOS=linux go build -race -tags timetzdata -o broadcast ./cmd
+RUN GOOS=linux go build -race -tags timetzdata -o skywalker ./cmd
 
 ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
-ENTRYPOINT [ "/src/broadcast" ]
+VOLUME [ "/src/e2e" ]
+
+ENTRYPOINT [ "/src/skywalker" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]