From d4eb8c5bbc50a3eedf49587a094746c12885b272 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 1 Apr 2021 15:05:13 +0800
Subject: [PATCH 01/52] Full publisher session

---
 go.mod                          |  4 ++--
 go.sum                          |  6 ++++++
 internal/broadcast/broadcast.go | 30 ++++++++++++++++++++----------
 internal/broadcast/publisher.go |  8 +++++---
 internal/broadcast/session.go   | 17 +++++++++++++----
 internal/broadcast/webrtc.go    | 22 ++++++++++++++++++----
 6 files changed, 64 insertions(+), 23 deletions(-)

diff --git a/go.mod b/go.mod
index deda9a1..26fc948 100644
--- a/go.mod
+++ b/go.mod
@@ -6,9 +6,9 @@ require (
 	github.com/SB-IM/logging v0.2.1
 	github.com/SB-IM/mqtt-client v0.1.0
 	github.com/SB-IM/pb v0.0.0-20210327132643-2a2ed393cb2b
-	github.com/eclipse/paho.mqtt.golang v1.3.2
+	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/pion/rtcp v1.2.6
-	github.com/pion/webrtc/v3 v3.0.19
+	github.com/pion/webrtc/v3 v3.0.20
 	github.com/rs/zerolog v1.21.0
 	github.com/urfave/cli/v2 v2.3.0
 	google.golang.org/protobuf v1.26.0
diff --git a/go.sum b/go.sum
index 1466bc6..a8afc5f 100644
--- a/go.sum
+++ b/go.sum
@@ -18,6 +18,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eclipse/paho.mqtt.golang v1.3.2 h1:ICzfxSyrR8bOsh9l8JBBOwO1tc2C26oEyody0ml0L6E=
 github.com/eclipse/paho.mqtt.golang v1.3.2/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
+github.com/eclipse/paho.mqtt.golang v1.3.3 h1:Fh1zsLniMFJByLqKrSB9ZRjkbpU0k1Xne23ZqEE/O08=
+github.com/eclipse/paho.mqtt.golang v1.3.3/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -106,6 +108,8 @@ github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko
 github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sctp v1.7.11 h1:UCnj7MsobLKLuP/Hh+JMiI/6W5Bs/VF45lWKgHFjSIE=
 github.com/pion/sctp v1.7.11/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
+github.com/pion/sctp v1.7.12 h1:GsatLufywVruXbZZT1CKg+Jr8ZTkwiPnmUC/oO9+uuY=
+github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
 github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
 github.com/pion/sdp/v3 v3.0.4/go.mod h1:bNiSknmJE0HYBprTHXKPQ3+JjacTv5uap92ueJZKsRk=
 github.com/pion/srtp/v2 v2.0.2 h1:664iGzVmaY7KYS5M0gleY0DscRo9ReDfTxQrq4UgGoU=
@@ -123,6 +127,8 @@ github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
 github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
 github.com/pion/webrtc/v3 v3.0.19 h1:h3EOMuMNYkJ0X2w1iKNGGBLFN7M/Max0vNUF9IUXqBc=
 github.com/pion/webrtc/v3 v3.0.19/go.mod h1:P/aoizAjeMUh61uAH58BRypn97IKjcLtIAm/mHqovJw=
+github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw=
+github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index 50238b8..f3e8bdf 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -42,6 +42,7 @@ func New(ctx context.Context, config ConfigOptions) *Service {
 
 // Broadcast broadcasts video streams following publisher -> transfer -> subscribers flow direction.
 func (svc *Service) Broadcast() error {
+	// Start subscriber signalling handler.
 	// Register Websockets handler.
 	http.HandleFunc(svc.config.WSServerConfigOptions.Path, svc.handleSubscription())
 	go func() {
@@ -56,12 +57,12 @@ func (svc *Service) Broadcast() error {
 	}()
 
 	// Start publisher signaling worker.
-	publisher := newPublisher(svc.client, svc.logger, svc.config.TopicConfigOptions)
+	publisher := newPublisher(svc.client, &svc.logger, svc.config.TopicConfigOptions)
 	publisher.Signal()
 
 	// Use a loop to start endless broadcasting sessions.
 	for {
-		s := newSession(&publisher.publisherChans, svc.logger, svc.config.WebRTCConfigOptions)
+		s := newSession(&publisher.publisherChans, &svc.logger, svc.config.WebRTCConfigOptions)
 
 		// You can get id and track source only when half of session (publisher session) completes.
 		// Therefore, you must start session first.
@@ -78,10 +79,12 @@ func (svc *Service) Broadcast() error {
 // handleSubscription is called every time after publisher session and at the start of subscriber session.
 // If the running order is wrong, it blocks forever.
 func (svc *Service) handleSubscription() http.HandlerFunc {
+	logger := svc.logger.With().Str("component", "subscriber").Logger()
+
 	return func(w http.ResponseWriter, r *http.Request) {
 		c, err := websocket.Accept(w, r, nil)
 		if err != nil {
-			panic(err)
+			logger.Panic().Err(err).Msg("webSocket failed to establish connection")
 		}
 		defer c.Close(websocket.StatusInternalError, "")
 
@@ -90,13 +93,16 @@ func (svc *Service) handleSubscription() http.HandlerFunc {
 
 		var offer pb.SessionDescription
 		if err := wsjson.Read(ctx, c, &offer); err != nil {
-			panic(err)
+			logger.Err(err).Msg("could not read message")
+			return
 		}
-		fmt.Printf("Received: %+v\n", offer.Id)
+		logger.Debug().Str("offer", offer.String()).Msg("received offer")
 
+		var (
+			offerChan  chan *pb.SessionDescription
+			answerChan chan *pb.SessionDescription
+		)
 		// The subscriber's sdp id must be equal to session's id.
-		var offerChan chan *pb.SessionDescription
-		var answerChan chan *pb.SessionDescription
 		if inner, ok := svc.sessions[machineID(offer.Id)]; ok {
 			// The subscriber's sdp track source must be equal to session's track source.
 			if subscriber, ok := inner[offer.TrackSource]; ok {
@@ -104,10 +110,14 @@ func (svc *Service) handleSubscription() http.HandlerFunc {
 				answerChan = subscriber.answerChan
 			}
 		}
-		if offerChan == nil {
-			if err := wsjson.Write(ctx, c, "wrong id"); err != nil {
-				panic(err)
+		// If offerChan or answerChan is nil, it means offer.Id or offer.TrackSource does not exists in any session.
+		// And this request message is invalid.
+		if offerChan == nil || answerChan == nil {
+			if err := wsjson.Write(ctx, c, "wrong id or track_source"); err != nil {
+				logger.Err(err).Msg("could not write message")
 			}
+			logger.Debug().Msg("offerChan or answerChan is nil")
+			return
 		}
 		offerChan <- &offer
 
diff --git a/internal/broadcast/publisher.go b/internal/broadcast/publisher.go
index 7d3382c..717eefe 100644
--- a/internal/broadcast/publisher.go
+++ b/internal/broadcast/publisher.go
@@ -21,19 +21,21 @@ type publisher struct {
 	config         TopicConfigOptions
 }
 
-func newPublisher(client mqtt.Client, logger zerolog.Logger, config TopicConfigOptions) *publisher {
+func newPublisher(client mqtt.Client, logger *zerolog.Logger, config TopicConfigOptions) *publisher {
 	return &publisher{
 		client: client,
-		publisherChans: publisherChans{
+		publisherChans: publisherChans{ // The channel buffer size limits concurrency
 			OfferChan:  make(chan *pb.SessionDescription, 1),
 			AnswerChan: make(chan *pb.SessionDescription, 1),
 		},
-		logger: logger,
+		logger: *logger,
 		config: config,
 	}
 }
 
 func (p *publisher) Signal() {
+	p.logger = p.logger.With().Str("component", "publisher").Logger()
+
 	// The receiving topic is the same for each edge device, but message payload is different.
 	// The id and trackSource in payload determine the following publishing topic.
 	// Receive remote description with MQTT.
diff --git a/internal/broadcast/session.go b/internal/broadcast/session.go
index a52fc7a..90dbb23 100644
--- a/internal/broadcast/session.go
+++ b/internal/broadcast/session.go
@@ -1,6 +1,8 @@
 package broadcast
 
 import (
+	"fmt"
+
 	pb "github.com/SB-IM/pb/signal"
 	"github.com/rs/zerolog"
 )
@@ -36,14 +38,14 @@ type session struct {
 	config WebRTCConfigOptions
 }
 
-func newSession(publisherChans *publisherChans, logger zerolog.Logger, config WebRTCConfigOptions) *session {
+func newSession(publisherChans *publisherChans, logger *zerolog.Logger, config WebRTCConfigOptions) *session {
 	return &session{
 		publisherChans: publisherChans,
 		subscriberChans: &subscriberChans{
 			offerChan:  make(chan *pb.SessionDescription, 1),
 			answerChan: make(chan *pb.SessionDescription, 1),
 		},
-		logger: logger,
+		logger: *logger,
 		config: config,
 	}
 }
@@ -54,14 +56,21 @@ func (s *session) start(middle middleFunc) error {
 	if err != nil {
 		return err
 	}
-	s.createPublisher(localTrack)
+	s.logger.Debug().Msg("created local track")
+
+	if err := s.createPublisher(localTrack); err != nil {
+		return fmt.Errorf("failed to crate publisher: %w", err)
+	}
+	s.logger.Debug().Str("id", string(s.id)).Int32("track_source", int32(s.trackSource)).Msg("created a publisher")
 
 	middle(s.id, s.trackSource, s.subscriberChans)
 
 	go func() {
 		// Use a loop to start endless subscriber sessions.
 		for {
-			s.createSubscriber(localTrack)
+			if err := s.createSubscriber(localTrack); err != nil {
+				s.logger.Err(err).Msg("failed to create subscriber")
+			}
 		}
 	}()
 
diff --git a/internal/broadcast/webrtc.go b/internal/broadcast/webrtc.go
index 81cffd5..92b43b2 100644
--- a/internal/broadcast/webrtc.go
+++ b/internal/broadcast/webrtc.go
@@ -22,6 +22,13 @@ const (
 	rtcpPLIInterval = time.Second * 3
 )
 
+func (a actor) string() string {
+	if a == peerPublisher {
+		return "publisher"
+	}
+	return "subscriber"
+}
+
 // createLocalTrack creates a local video track shared between publisher and subscriber peers.
 // localTrack is a transfer that transfers video track between publisher and subscriber peers.
 // For localTrack, there can only be one publisher peer, but subscribers can be many.
@@ -74,7 +81,7 @@ func (s *session) createPublisher(videoTrack *webrtc.TrackLocalStaticRTP) error
 	if err := s.createPeerConnection(peerConnection, peerPublisher); err != nil {
 		return fmt.Errorf("failed to create peer connection: %w", err)
 	}
-	s.logger.Debug().Msg("created PeerConnection for publisher")
+	s.logger.Debug().Str("id", string(s.id)).Int32("track_source", int32(s.trackSource)).Msg("created PeerConnection for publisher")
 
 	return nil
 }
@@ -118,19 +125,26 @@ func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, ac
 		// Set up session identity.
 		s.id = machineID(offer.Id)
 		s.trackSource = offer.TrackSource
+
 	} else {
 		offer = <-s.subscriberChans.offerChan
 	}
+	logger := s.logger.With().
+		Str("actor", actor.string()).
+		Str("id", offer.Id).
+		Int32("track_source",
+			int32(offer.TrackSource)).
+		Logger()
 
 	// Set the handler for ICE connection state
 	// This will notify you when the peer has connected/disconnected
 	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		s.logger.Debug().Str("state", connectionState.String()).Msg("ICE connection state has changed")
+		logger.Debug().Str("state", connectionState.String()).Msg("ICE connection state has changed")
 		if connectionState == webrtc.ICEConnectionStateFailed {
 			if err := peerConnection.Close(); err != nil {
-				s.logger.Panic().Err(err).Msg("could not close PeerConnection")
+				logger.Panic().Err(err).Msg("could not close PeerConnection")
 			}
-			s.logger.Debug().Msg("PeerConnection has been closed")
+			logger.Debug().Msg("PeerConnection has been closed")
 		}
 	})
 

From 293c056dc402d9e4d0c3028fa1d51a7a4010142c Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 1 Apr 2021 15:11:07 +0800
Subject: [PATCH 02/52] Chore: fix some golint

---
 cmd/broadcast/cmd.go            | 2 +-
 internal/broadcast/broadcast.go | 6 +++---
 internal/broadcast/webrtc.go    | 9 ++++-----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/cmd/broadcast/cmd.go b/cmd/broadcast/cmd.go
index ec920a6..8d107ca 100644
--- a/cmd/broadcast/cmd.go
+++ b/cmd/broadcast/cmd.go
@@ -71,7 +71,7 @@ func Command() *cli.Command {
 			return nil
 		},
 		Action: func(c *cli.Context) error {
-			svc := broadcast.New(ctx, broadcast.ConfigOptions{
+			svc := broadcast.New(ctx, &broadcast.ConfigOptions{
 				WebRTCConfigOptions:   webRTCConfigOptions,
 				TopicConfigOptions:    topicConfigOptions,
 				WSServerConfigOptions: wsServerConfigOptions,
diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index f3e8bdf..1650faf 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -28,10 +28,10 @@ type Service struct {
 	sessions map[machineID]subscriber
 	client   mqtt.Client
 	logger   zerolog.Logger
-	config   ConfigOptions
+	config   *ConfigOptions
 }
 
-func New(ctx context.Context, config ConfigOptions) *Service {
+func New(ctx context.Context, config *ConfigOptions) *Service {
 	return &Service{
 		sessions: make(map[machineID]subscriber),
 		client:   mqttclient.FromContext(ctx),
@@ -42,7 +42,7 @@ func New(ctx context.Context, config ConfigOptions) *Service {
 
 // Broadcast broadcasts video streams following publisher -> transfer -> subscribers flow direction.
 func (svc *Service) Broadcast() error {
-	// Start subscriber signalling handler.
+	// Start subscriber signaling handler.
 	// Register Websockets handler.
 	http.HandleFunc(svc.config.WSServerConfigOptions.Path, svc.handleSubscription())
 	go func() {
diff --git a/internal/broadcast/webrtc.go b/internal/broadcast/webrtc.go
index 92b43b2..7eb97aa 100644
--- a/internal/broadcast/webrtc.go
+++ b/internal/broadcast/webrtc.go
@@ -62,7 +62,7 @@ func (s *session) createPublisher(videoTrack *webrtc.TrackLocalStaticRTP) error
 	// Set a handler for when a new remote track starts, this just distributes all our packets
 	// to connected peers
 	peerConnection.OnTrack(func(t *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
-		go sendRTCP(peerConnection, t, s.logger)
+		go sendRTCP(peerConnection, t, &s.logger)
 
 		rtpBuf := make([]byte, 1400)
 		for {
@@ -104,7 +104,7 @@ func (s *session) createSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 	if err != nil {
 		return fmt.Errorf("could not add track: %w", err)
 	}
-	go processRTCP(rtpSender, s.logger)
+	go processRTCP(rtpSender, &s.logger)
 
 	if err := s.createPeerConnection(peerConnection, peerSubscriber); err != nil {
 		return fmt.Errorf("failed to create peer connection: %w", err)
@@ -125,7 +125,6 @@ func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, ac
 		// Set up session identity.
 		s.id = machineID(offer.Id)
 		s.trackSource = offer.TrackSource
-
 	} else {
 		offer = <-s.subscriberChans.offerChan
 	}
@@ -182,7 +181,7 @@ func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, ac
 
 // sendRTCP sends a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
 // This can be less wasteful by processing incoming RTCP events, then we would emit a NACK/PLI when a viewer requests it
-func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote, logger zerolog.Logger) {
+func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote, logger *zerolog.Logger) {
 	ticker := time.NewTicker(rtcpPLIInterval)
 	for range ticker.C {
 		if rtcpSendErr := peerConnection.WriteRTCP([]rtcp.Packet{
@@ -198,7 +197,7 @@ func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRe
 // processRTCP reads incoming RTCP packets
 // Before these packets are returned they are processed by interceptors.
 // For things like NACK this needs to be called.
-func processRTCP(rtpSender *webrtc.RTPSender, logger zerolog.Logger) {
+func processRTCP(rtpSender *webrtc.RTPSender, logger *zerolog.Logger) {
 	rtcpBuf := make([]byte, 1500)
 	for {
 		if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {

From 1e57c581652f1dfc93671fee0b1faee1e334373a Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 1 Apr 2021 20:10:51 +0800
Subject: [PATCH 03/52] Full subscriber session

---
 Makefile                        |   4 +-
 docker/Dockerfile.broadcast     |   2 +
 docker/docker-compose.dev.yaml  |  10 ++--
 e2e/broadcast/index.html        |  14 +++++
 e2e/broadcast/index.js          |  62 +++++++++++++++++++
 go.mod                          |   4 +-
 go.sum                          |  24 ++++----
 internal/broadcast/broadcast.go | 102 ++++++++++++++++++++------------
 internal/broadcast/webrtc.go    |  14 +++--
 9 files changed, 170 insertions(+), 66 deletions(-)
 create mode 100644 e2e/broadcast/index.html
 create mode 100644 e2e/broadcast/index.js

diff --git a/Makefile b/Makefile
index 2653c3a..5f91568 100644
--- a/Makefile
+++ b/Makefile
@@ -24,8 +24,8 @@ image:
 
 # Note: '--env-file' value is relative to '-f' value's directory.
 .PHONY: up
-up: down
-	@docker-compose -f docker/docker-compose.dev.yaml up --build -d
+up: down image
+	@docker-compose -f docker/docker-compose.dev.yaml up -d
 
 .PHONY: down
 down:
diff --git a/docker/Dockerfile.broadcast b/docker/Dockerfile.broadcast
index 9359d8d..2f70195 100644
--- a/docker/Dockerfile.broadcast
+++ b/docker/Dockerfile.broadcast
@@ -34,6 +34,8 @@ ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
+EXPOSE 8080
+
 ENTRYPOINT [ "/broadcast" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]
diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 87970cf..5ab3504 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -1,9 +1,7 @@
 version: "3.9"
 services:
   livestream:
-    build:
-      context: ../../sphinx
-      dockerfile: docker/Dockerfile.livestream.dev
+    image: ghcr.io/sb-im/sphinx:livestream-amd64-latest
     container_name: livestream
     command:
       - --debug
@@ -21,15 +19,15 @@ services:
     restart: on-failure
 
   broadcast:
-    build:
-      context: ../
-      dockerfile: docker/Dockerfile.broadcast.dev
+    image: ghcr.io/sb-im/skywalker:broadcast-latest
     container_name: broadcast
     command:
       - --debug
       - broadcast
       - -c
       - /config/config.yaml
+    ports:
+      - "8080:8080"
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
     volumes:
diff --git a/e2e/broadcast/index.html b/e2e/broadcast/index.html
new file mode 100644
index 0000000..745c254
--- /dev/null
+++ b/e2e/broadcast/index.html
@@ -0,0 +1,14 @@
+<html>
+
+<head>
+    <title>broadcast</title>
+</head>
+
+<body>
+    <h3> Video </h3>
+    <div id="remoteVideos"></div> <br />
+</body>
+
+<script type="text/javascript" src="index.js"></script>
+
+</html>
diff --git a/e2e/broadcast/index.js b/e2e/broadcast/index.js
new file mode 100644
index 0000000..f84475f
--- /dev/null
+++ b/e2e/broadcast/index.js
@@ -0,0 +1,62 @@
+(() => {
+  let pc = new RTCPeerConnection();
+  pc.ontrack = function (event) {
+    var el = document.createElement(event.track.kind);
+    el.srcObject = event.streams[0];
+    el.autoplay = true;
+    el.controls = true;
+
+    document.getElementById("remoteVideos").appendChild(el);
+  };
+
+  pc.addTransceiver("video");
+
+  function dial() {
+    const conn = new WebSocket(`ws://localhost:8080/ws/webrtc`);
+
+    conn.addEventListener("close", (ev) => {
+      console.log(
+        `WebSocket Disconnected code: ${ev.code}, reason: ${ev.reason}`
+      );
+    });
+    conn.addEventListener("open", (ev) => {
+      console.info("websocket connected");
+
+      for (let i = 0; i < 2; i++) {
+        pc.createOffer()
+          .then((offer) => {
+            pc.setLocalDescription(offer);
+
+            // Standard message schema.
+            msg = {
+              id: "fa955cc6881b4b45b49ffbf2d81e7223",
+              track_source: i,
+              sdp: offer,
+            };
+
+            str = JSON.stringify(msg);
+            console.log(`sent offer ${i} ${str}`);
+
+            conn.send(str);
+          })
+          .catch(alert);
+      }
+    });
+
+    // This is where we handle messages received.
+    conn.addEventListener("message", (ev) => {
+      // Receiving SDP answer
+      console.log(`Received SDP answer from server: ${ev.data}`);
+
+      // .then((res) => res.json())
+      // .then((res) => pc.setRemoteDescription(res))
+      try {
+        answer = JSON.parse(ev.data);
+        pc.setRemoteDescription(answer.sdp);
+      } catch (err) {
+        console.error(err);
+      }
+    });
+  }
+  dial();
+})();
diff --git a/go.mod b/go.mod
index 26fc948..647085f 100644
--- a/go.mod
+++ b/go.mod
@@ -3,9 +3,9 @@ module github.com/SB-IM/skywalker
 go 1.16
 
 require (
-	github.com/SB-IM/logging v0.2.1
+	github.com/SB-IM/logging v0.2.2
 	github.com/SB-IM/mqtt-client v0.1.0
-	github.com/SB-IM/pb v0.0.0-20210327132643-2a2ed393cb2b
+	github.com/SB-IM/pb v0.1.3
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/pion/rtcp v1.2.6
 	github.com/pion/webrtc/v3 v3.0.20
diff --git a/go.sum b/go.sum
index a8afc5f..6e90a53 100644
--- a/go.sum
+++ b/go.sum
@@ -1,22 +1,22 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/SB-IM/logging v0.1.0 h1:Auo5aT0NGSsSaq+6+bMWrxLHChmG9Zc4LyF7F9tKQ7k=
 github.com/SB-IM/logging v0.1.0/go.mod h1:VtWxAA+Qto3I28BcaS8eRgdxkOjT9ROwBqlSwL9n6Fw=
-github.com/SB-IM/logging v0.2.0 h1:GTzxo9MhmgpOtYF1C8P4BeWdMogemM9Zgc6KSjJrGLo=
-github.com/SB-IM/logging v0.2.0/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
-github.com/SB-IM/logging v0.2.1 h1:Ae9oONCjbswLMEGNIqXtX+i566lplwXPTU/T5mvSJPs=
-github.com/SB-IM/logging v0.2.1/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
+github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
+github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
 github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXCdI=
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
-github.com/SB-IM/pb v0.0.0-20210327132643-2a2ed393cb2b h1:cOSjAr8Wp9Fm1U7ogGYtscdR89ULvXx9gizkOmYkzBo=
-github.com/SB-IM/pb v0.0.0-20210327132643-2a2ed393cb2b/go.mod h1:o6mbn4gu2nalbtL13qr33fefuUIoEZTRBfQTvl7Z0vM=
+github.com/SB-IM/pb v0.1.0 h1:Naa4g0ETHosjM8CvvH7/7dmPheRGgcS9PSEqwJ99sgI=
+github.com/SB-IM/pb v0.1.0/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
+github.com/SB-IM/pb v0.1.2 h1:sW1SW9sxBIwEkEdumFFJotIuAToz52NdOb64M2KWBzI=
+github.com/SB-IM/pb v0.1.2/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
+github.com/SB-IM/pb v0.1.3 h1:Sq1wCcrcX5smwLvEtfw1cN04fHM2T4wiN5FggThLEU8=
+github.com/SB-IM/pb v0.1.3/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/eclipse/paho.mqtt.golang v1.3.2 h1:ICzfxSyrR8bOsh9l8JBBOwO1tc2C26oEyody0ml0L6E=
 github.com/eclipse/paho.mqtt.golang v1.3.2/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
 github.com/eclipse/paho.mqtt.golang v1.3.3 h1:Fh1zsLniMFJByLqKrSB9ZRjkbpU0k1Xne23ZqEE/O08=
 github.com/eclipse/paho.mqtt.golang v1.3.3/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
@@ -49,8 +49,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1 h1:jAbXjIeW2ZSW2AwFxlGTDoc2CjI2XujLkV3ArsZFCvc=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -106,8 +106,6 @@ github.com/pion/rtcp v1.2.6/go.mod h1:52rMNPWFsjr39z9B9MhnkqhPLoeHTv1aN63o/42bWE
 github.com/pion/rtp v1.6.2 h1:iGBerLX6JiDjB9NXuaPzHyxHFG9JsIEdgwTC0lp5n/U=
 github.com/pion/rtp v1.6.2/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko=
 github.com/pion/sctp v1.7.10/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
-github.com/pion/sctp v1.7.11 h1:UCnj7MsobLKLuP/Hh+JMiI/6W5Bs/VF45lWKgHFjSIE=
-github.com/pion/sctp v1.7.11/go.mod h1:EhpTUQu1/lcK3xI+eriS6/96fWetHGCvBi9MSsnaBN0=
 github.com/pion/sctp v1.7.12 h1:GsatLufywVruXbZZT1CKg+Jr8ZTkwiPnmUC/oO9+uuY=
 github.com/pion/sctp v1.7.12/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s=
 github.com/pion/sdp/v3 v3.0.4 h1:2Kf+dgrzJflNCSw3TV5v2VLeI0s/qkzy2r5jlR0wzf8=
@@ -125,8 +123,6 @@ github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
 github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
 github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
 github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
-github.com/pion/webrtc/v3 v3.0.19 h1:h3EOMuMNYkJ0X2w1iKNGGBLFN7M/Max0vNUF9IUXqBc=
-github.com/pion/webrtc/v3 v3.0.19/go.mod h1:P/aoizAjeMUh61uAH58BRypn97IKjcLtIAm/mHqovJw=
 github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw=
 github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index 1650faf..97d2934 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -67,9 +67,16 @@ func (svc *Service) Broadcast() error {
 		// You can get id and track source only when half of session (publisher session) completes.
 		// Therefore, you must start session first.
 		if err := s.start(func(id machineID, t pb.TrackSource, s *subscriberChans) {
-			inner := make(subscriber)
-			inner[t] = s
-			svc.sessions[id] = inner
+			// This is where get buggy. You can't make new inner map for each session,
+			// because by this way you erase previous inner map which has track_source key value.
+			// In face, the inner map should be shared between two sessions which have the same machine id.
+			if inner, ok := svc.sessions[id]; ok {
+				inner[t] = s
+			} else {
+				inner := make(subscriber)
+				inner[t] = s
+				svc.sessions[id] = inner
+			}
 		}); err != nil {
 			return fmt.Errorf("session failed: %w", err)
 		}
@@ -82,50 +89,71 @@ func (svc *Service) handleSubscription() http.HandlerFunc {
 	logger := svc.logger.With().Str("component", "subscriber").Logger()
 
 	return func(w http.ResponseWriter, r *http.Request) {
-		c, err := websocket.Accept(w, r, nil)
+		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
+			OriginPatterns: []string{"*"},
+		})
 		if err != nil {
 			logger.Panic().Err(err).Msg("webSocket failed to establish connection")
 		}
-		defer c.Close(websocket.StatusInternalError, "")
+		defer c.Close(websocket.StatusNormalClosure, "")
 
 		ctx, cancel := context.WithCancel(r.Context())
 		defer cancel()
 
-		var offer pb.SessionDescription
-		if err := wsjson.Read(ctx, c, &offer); err != nil {
-			logger.Err(err).Msg("could not read message")
-			return
-		}
-		logger.Debug().Str("offer", offer.String()).Msg("received offer")
-
-		var (
-			offerChan  chan *pb.SessionDescription
-			answerChan chan *pb.SessionDescription
-		)
-		// The subscriber's sdp id must be equal to session's id.
-		if inner, ok := svc.sessions[machineID(offer.Id)]; ok {
-			// The subscriber's sdp track source must be equal to session's track source.
-			if subscriber, ok := inner[offer.TrackSource]; ok {
-				offerChan = subscriber.offerChan
-				answerChan = subscriber.answerChan
+		// Continually reading from WebSocket connection.
+		// To reduce complexity, we don't use channel pipeline transporting reading and writing messages.
+		for {
+			var offer pb.SessionDescription
+			if err := wsjson.Read(ctx, c, &offer); err != nil {
+				logger.Err(err).Msg("could not read message")
+				return
 			}
-		}
-		// If offerChan or answerChan is nil, it means offer.Id or offer.TrackSource does not exists in any session.
-		// And this request message is invalid.
-		if offerChan == nil || answerChan == nil {
-			if err := wsjson.Write(ctx, c, "wrong id or track_source"); err != nil {
-				logger.Err(err).Msg("could not write message")
+			logger.Debug().
+				Str("offer", offer.String()).
+				Str("offer.id", offer.Id).
+				Int32("offer.track_source", int32(offer.TrackSource)).
+				Msg("received offer")
+
+			var (
+				offerChan  chan *pb.SessionDescription
+				answerChan chan *pb.SessionDescription
+			)
+			// The subscriber's sdp id must be equal to session's id.
+			if inner, ok := svc.sessions[machineID(offer.Id)]; ok {
+				logger.Debug().
+					Str("offer.id", offer.Id).
+					Int32("offer.track_source", int32(offer.TrackSource)).
+					Msg("got machine id")
+				// The subscriber's sdp track source must be equal to session's track source.
+				if subscriber, ok := inner[offer.TrackSource]; ok {
+					offerChan = subscriber.offerChan
+					answerChan = subscriber.answerChan
+					logger.Debug().
+						Str("offer.id", offer.Id).
+						Int32("offer.track_source", int32(offer.TrackSource)).
+						Msg("got subscriber channels")
+				}
 			}
-			logger.Debug().Msg("offerChan or answerChan is nil")
-			return
-		}
-		offerChan <- &offer
+			// If offerChan or answerChan is nil, it means offer.Id or offer.TrackSource does not exists in any session.
+			// And this request message is invalid.
+			if offerChan == nil || answerChan == nil {
+				logger.Debug().
+					Str("offer.id", offer.Id).
+					Int32("offer.track_source", int32(offer.TrackSource)).
+					Msg("offerChan or answerChan is nil")
+
+				if err := wsjson.Write(ctx, c, "wrong id or track_source"); err != nil {
+					logger.Err(err).Msg("could not write message")
+				}
+				continue
+			}
+			// TODO: Timeout for sending and receiving in case of blocking side.
+			offerChan <- &offer
 
-		answer := <-answerChan
-		if err := wsjson.Write(ctx, c, answer); err != nil {
-			panic(err)
+			answer := <-answerChan
+			if err := wsjson.Write(ctx, c, answer); err != nil {
+				logger.Err(err).Msg("could not write message")
+			}
 		}
-
-		c.Close(websocket.StatusNormalClosure, "")
 	}
 }
diff --git a/internal/broadcast/webrtc.go b/internal/broadcast/webrtc.go
index 7eb97aa..5d5836c 100644
--- a/internal/broadcast/webrtc.go
+++ b/internal/broadcast/webrtc.go
@@ -118,6 +118,7 @@ func (s *session) createSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 // It's a generic function for both publisher and subscriber.
 func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, actor actor) error {
 	// Receive remote offer.
+	// TODO: Timeout for sending and receiving in case of blocking side.
 	var offer *pb.SessionDescription
 	if actor == peerPublisher {
 		offer = <-s.publisherChans.OfferChan
@@ -168,7 +169,8 @@ func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, ac
 	// in a production application you should exchange ICE Candidates via OnICECandidate
 	<-gatherComplete
 
-	// Send answer local description.
+	// Send answer of local description.
+	// This is a universal answer for both publisher and subscriber in protobuf format.
 	sdp := webrtcSdp2pbSdp(peerConnection.LocalDescription())
 	if actor == peerPublisher {
 		s.publisherChans.AnswerChan <- sdp
@@ -209,14 +211,16 @@ func processRTCP(rtpSender *webrtc.RTPSender, logger *zerolog.Logger) {
 
 func pbSdp2webrtcSdp(sdp *pb.SessionDescription) webrtc.SessionDescription {
 	return webrtc.SessionDescription{
-		Type: webrtc.SDPType(sdp.Type),
-		SDP:  string(sdp.Description),
+		Type: webrtc.NewSDPType(sdp.Sdp.Type),
+		SDP:  sdp.Sdp.Sdp,
 	}
 }
 
 func webrtcSdp2pbSdp(sdp *webrtc.SessionDescription) *pb.SessionDescription {
 	return &pb.SessionDescription{
-		Type:        int32(sdp.Type),
-		Description: []byte(sdp.SDP),
+		Sdp: &pb.SDP{
+			Type: sdp.Type.String(),
+			Sdp:  sdp.SDP,
+		},
 	}
 }

From e98acc16c5d9a923374d2d2ab81c9973fa7988cc Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 1 Apr 2021 20:28:04 +0800
Subject: [PATCH 04/52] Fix send log

---
 cmd/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/main.go b/cmd/main.go
index 2ad3ac4..8a97018 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -17,7 +17,7 @@ func init() {
 
 func main() {
 	if err := run(os.Args); err != nil {
-		log.Fatal().Err(err)
+		log.Fatal().Err(err).Send()
 	}
 }
 

From 0f7b33723177691235e56eaa32cbfab5ce569259 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Fri, 2 Apr 2021 10:37:15 +0800
Subject: [PATCH 05/52] Remove fatal level

---
 cmd/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/main.go b/cmd/main.go
index 8a97018..907b567 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -17,7 +17,7 @@ func init() {
 
 func main() {
 	if err := run(os.Args); err != nil {
-		log.Fatal().Err(err).Send()
+		log.Err(err).Send()
 	}
 }
 

From d926898482297b2be32946b4af1e5f0e6b499bc2 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Fri, 2 Apr 2021 19:47:59 +0800
Subject: [PATCH 06/52] Rewrite broadcast(#1, #2)

---
 cmd/broadcast/cmd.go                        |  44 +++---
 e2e/broadcast/index.js                      |  62 --------
 e2e/broadcast/main.go                       |  11 ++
 e2e/broadcast/{ => static}/index.html       |   6 +-
 e2e/broadcast/static/index.js               |  58 +++++++
 go.mod                                      |   2 +-
 go.sum                                      |  50 +-----
 internal/broadcast/broadcast.go             | 163 ++++----------------
 internal/broadcast/{ => cfg}/cfg.go         |  12 +-
 internal/broadcast/publisher.go             |  83 ----------
 internal/broadcast/publisher/publisher.go   | 137 ++++++++++++++++
 internal/broadcast/session.go               |  78 ----------
 internal/broadcast/session/session.go       |  15 ++
 internal/broadcast/subscriber/subscriber.go |  83 ++++++++++
 internal/broadcast/{ => webrtc}/webrtc.go   | 139 +++++++----------
 15 files changed, 430 insertions(+), 513 deletions(-)
 delete mode 100644 e2e/broadcast/index.js
 create mode 100644 e2e/broadcast/main.go
 rename e2e/broadcast/{ => static}/index.html (60%)
 create mode 100644 e2e/broadcast/static/index.js
 rename internal/broadcast/{ => cfg}/cfg.go (60%)
 delete mode 100644 internal/broadcast/publisher.go
 create mode 100644 internal/broadcast/publisher/publisher.go
 delete mode 100644 internal/broadcast/session.go
 create mode 100644 internal/broadcast/session/session.go
 create mode 100644 internal/broadcast/subscriber/subscriber.go
 rename internal/broadcast/{ => webrtc}/webrtc.go (51%)

diff --git a/cmd/broadcast/cmd.go b/cmd/broadcast/cmd.go
index 8d107ca..93b8f3b 100644
--- a/cmd/broadcast/cmd.go
+++ b/cmd/broadcast/cmd.go
@@ -13,6 +13,7 @@ import (
 	"github.com/urfave/cli/v2/altsrc"
 
 	"github.com/SB-IM/skywalker/internal/broadcast"
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
 )
 
 // Command returns a broadcast command.
@@ -25,10 +26,10 @@ func Command() *cli.Command {
 
 		mc mqtt.Client
 
-		mqttConfigOptions     mqttclient.ConfigOptions
-		topicConfigOptions    broadcast.TopicConfigOptions
-		webRTCConfigOptions   broadcast.WebRTCConfigOptions
-		wsServerConfigOptions broadcast.WSServerConfigOptions
+		mqttConfigOptions   mqttclient.ConfigOptions
+		topicConfigOptions  cfg.TopicConfigOptions
+		webRTCConfigOptions cfg.WebRTCConfigOptions
+		serverConfigOptions cfg.ServerConfigOptions
 	)
 
 	flags := func() (flags []cli.Flag) {
@@ -37,7 +38,7 @@ func Command() *cli.Command {
 			mqttFlags(&mqttConfigOptions),
 			topicFlags(&topicConfigOptions),
 			webRTCFlags(&webRTCConfigOptions),
-			wsFlags(&wsServerConfigOptions),
+			serverFlags(&serverConfigOptions),
 		} {
 			flags = append(flags, v...)
 		}
@@ -71,10 +72,10 @@ func Command() *cli.Command {
 			return nil
 		},
 		Action: func(c *cli.Context) error {
-			svc := broadcast.New(ctx, &broadcast.ConfigOptions{
-				WebRTCConfigOptions:   webRTCConfigOptions,
-				TopicConfigOptions:    topicConfigOptions,
-				WSServerConfigOptions: wsServerConfigOptions,
+			svc := broadcast.New(ctx, &cfg.ConfigOptions{
+				WebRTCConfigOptions: webRTCConfigOptions,
+				TopicConfigOptions:  topicConfigOptions,
+				ServerConfigOptions: serverConfigOptions,
 			})
 			err := svc.Broadcast()
 			if err != nil {
@@ -135,7 +136,7 @@ func mqttFlags(mqttConfigOptions *mqttclient.ConfigOptions) []cli.Flag {
 	}
 }
 
-func topicFlags(topicConfigOptions *broadcast.TopicConfigOptions) []cli.Flag {
+func topicFlags(topicConfigOptions *cfg.TopicConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "topic-offer",
@@ -154,7 +155,7 @@ func topicFlags(topicConfigOptions *broadcast.TopicConfigOptions) []cli.Flag {
 	}
 }
 
-func webRTCFlags(webRTCConfigOptions *broadcast.WebRTCConfigOptions) []cli.Flag {
+func webRTCFlags(webRTCConfigOptions *cfg.WebRTCConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "ice-server",
@@ -166,28 +167,21 @@ func webRTCFlags(webRTCConfigOptions *broadcast.WebRTCConfigOptions) []cli.Flag
 	}
 }
 
-func wsFlags(wsServerConfigOptions *broadcast.WSServerConfigOptions) []cli.Flag {
+func serverFlags(serverConfigOptions *cfg.ServerConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "ws-host",
-			Usage:       "Host of WebSocket server",
+			Name:        "host",
+			Usage:       "Host of webRTC signaling server",
 			Value:       "0.0.0.0",
 			DefaultText: "0.0.0.0",
-			Destination: &wsServerConfigOptions.Host,
+			Destination: &serverConfigOptions.Host,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:        "ws-port",
-			Usage:       "Port of WebSocket server",
+			Name:        "port",
+			Usage:       "Port of webRTC signaling server",
 			Value:       8080,
 			DefaultText: "8080",
-			Destination: &wsServerConfigOptions.Port,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "ws-path",
-			Usage:       "HTTP path of broadcast service",
-			Value:       "/ws/webrtc",
-			DefaultText: "/ws/webrtc",
-			Destination: &wsServerConfigOptions.Path,
+			Destination: &serverConfigOptions.Port,
 		}),
 	}
 }
diff --git a/e2e/broadcast/index.js b/e2e/broadcast/index.js
deleted file mode 100644
index f84475f..0000000
--- a/e2e/broadcast/index.js
+++ /dev/null
@@ -1,62 +0,0 @@
-(() => {
-  let pc = new RTCPeerConnection();
-  pc.ontrack = function (event) {
-    var el = document.createElement(event.track.kind);
-    el.srcObject = event.streams[0];
-    el.autoplay = true;
-    el.controls = true;
-
-    document.getElementById("remoteVideos").appendChild(el);
-  };
-
-  pc.addTransceiver("video");
-
-  function dial() {
-    const conn = new WebSocket(`ws://localhost:8080/ws/webrtc`);
-
-    conn.addEventListener("close", (ev) => {
-      console.log(
-        `WebSocket Disconnected code: ${ev.code}, reason: ${ev.reason}`
-      );
-    });
-    conn.addEventListener("open", (ev) => {
-      console.info("websocket connected");
-
-      for (let i = 0; i < 2; i++) {
-        pc.createOffer()
-          .then((offer) => {
-            pc.setLocalDescription(offer);
-
-            // Standard message schema.
-            msg = {
-              id: "fa955cc6881b4b45b49ffbf2d81e7223",
-              track_source: i,
-              sdp: offer,
-            };
-
-            str = JSON.stringify(msg);
-            console.log(`sent offer ${i} ${str}`);
-
-            conn.send(str);
-          })
-          .catch(alert);
-      }
-    });
-
-    // This is where we handle messages received.
-    conn.addEventListener("message", (ev) => {
-      // Receiving SDP answer
-      console.log(`Received SDP answer from server: ${ev.data}`);
-
-      // .then((res) => res.json())
-      // .then((res) => pc.setRemoteDescription(res))
-      try {
-        answer = JSON.parse(ev.data);
-        pc.setRemoteDescription(answer.sdp);
-      } catch (err) {
-        console.error(err);
-      }
-    });
-  }
-  dial();
-})();
diff --git a/e2e/broadcast/main.go b/e2e/broadcast/main.go
new file mode 100644
index 0000000..5acba86
--- /dev/null
+++ b/e2e/broadcast/main.go
@@ -0,0 +1,11 @@
+package main
+
+import (
+	"log"
+	"net/http"
+)
+
+func main() {
+	http.Handle("/", http.FileServer(http.Dir("./static")))
+	log.Fatal(http.ListenAndServe(":7070", nil))
+}
diff --git a/e2e/broadcast/index.html b/e2e/broadcast/static/index.html
similarity index 60%
rename from e2e/broadcast/index.html
rename to e2e/broadcast/static/index.html
index 745c254..bc94727 100644
--- a/e2e/broadcast/index.html
+++ b/e2e/broadcast/static/index.html
@@ -5,8 +5,12 @@
 </head>
 
 <body>
-    <h3> Video </h3>
+    Video<br />
     <div id="remoteVideos"></div> <br />
+    Logs1<br />
+    <div id="log1"></div> <br />
+    Logs2<br />
+    <div id="log2"></div> <br />
 </body>
 
 <script type="text/javascript" src="index.js"></script>
diff --git a/e2e/broadcast/static/index.js b/e2e/broadcast/static/index.js
new file mode 100644
index 0000000..85409b4
--- /dev/null
+++ b/e2e/broadcast/static/index.js
@@ -0,0 +1,58 @@
+(() => {
+  for (let i = 0; i < 2; i++) {
+    let pc = new RTCPeerConnection({
+      iceServers: [
+        {
+          urls: "stun:stun.l.google.com:19302",
+        },
+      ],
+    });
+
+    let log = (msg) => {
+      document.getElementById(`log${i}`).innerHTML += msg + "<br>";
+    };
+
+    pc.ontrack = function (event) {
+      var el = document.createElement(event.track.kind);
+      el.srcObject = event.streams[0];
+      el.autoplay = true;
+      el.controls = true;
+
+      document.getElementById("remoteVideos").appendChild(el);
+    };
+
+    pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
+
+    pc.addTransceiver("video");
+
+    pc.createOffer()
+      .then((offer) => {
+        pc.setLocalDescription(offer);
+        console.log(`Sending offer ${i}: ${offer}`);
+
+        sdp = {
+          id: "fa955cc6881b4b45b49ffbf2d81e7223",
+          track_source: i,
+          sdp: offer,
+        };
+
+        return fetch("http://localhost:8080/v1/broadcast/signal", {
+          method: "post",
+          headers: {
+            Accept: "application/json",
+            "Content-Type": "application/json",
+            'Access-Control-Allow-Origin':'*'
+          },
+          mode: 'no-cors',
+          body: JSON.stringify(sdp),
+        });
+      })
+      .then((res) => res.json())
+      .then((res) => {
+        console.log(`Received answer ${i}: ${res}`);
+
+        pc.setRemoteDescription(res);
+      })
+      .catch(log);
+  }
+})();
diff --git a/go.mod b/go.mod
index 647085f..8733cbd 100644
--- a/go.mod
+++ b/go.mod
@@ -7,10 +7,10 @@ require (
 	github.com/SB-IM/mqtt-client v0.1.0
 	github.com/SB-IM/pb v0.1.3
 	github.com/eclipse/paho.mqtt.golang v1.3.3
+	github.com/gorilla/mux v1.8.0
 	github.com/pion/rtcp v1.2.6
 	github.com/pion/webrtc/v3 v3.0.20
 	github.com/rs/zerolog v1.21.0
 	github.com/urfave/cli/v2 v2.3.0
 	google.golang.org/protobuf v1.26.0
-	nhooyr.io/websocket v1.8.6
 )
diff --git a/go.sum b/go.sum
index 6e90a53..07c810f 100644
--- a/go.sum
+++ b/go.sum
@@ -5,10 +5,6 @@ github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
 github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
 github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXCdI=
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
-github.com/SB-IM/pb v0.1.0 h1:Naa4g0ETHosjM8CvvH7/7dmPheRGgcS9PSEqwJ99sgI=
-github.com/SB-IM/pb v0.1.0/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
-github.com/SB-IM/pb v0.1.2 h1:sW1SW9sxBIwEkEdumFFJotIuAToz52NdOb64M2KWBzI=
-github.com/SB-IM/pb v0.1.2/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
 github.com/SB-IM/pb v0.1.3 h1:Sq1wCcrcX5smwLvEtfw1cN04fHM2T4wiN5FggThLEU8=
 github.com/SB-IM/pb v0.1.3/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -22,26 +18,7 @@ github.com/eclipse/paho.mqtt.golang v1.3.3 h1:Fh1zsLniMFJByLqKrSB9ZRjkbpU0k1Xne2
 github.com/eclipse/paho.mqtt.golang v1.3.3/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
-github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
-github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
-github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
-github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
-github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
-github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
-github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
-github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -56,30 +33,18 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8=
-github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
-github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -140,16 +105,10 @@ github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZ
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
-github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
-github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
-github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -182,7 +141,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -195,7 +153,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -222,10 +179,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
-nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index 97d2934..2d095f4 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -1,159 +1,58 @@
-// broadcast is an event driven video broadcasting system.
-// It's composed of a model of publisher, transfer and subscribers.
-// The connection between local peer and remote publisher peer can be called a half session and is valid and should be established first.
-// The connection between local peer and remote subscriber peer can also be called a half session but may not be valid.
-// A whole session is made up of three peers. In fact, every publisher fires an event that starts a unique session.
-// And every subscriber fires an event that starts their own half session.
 package broadcast
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
+	"time"
 
 	mqttclient "github.com/SB-IM/mqtt-client"
-	pb "github.com/SB-IM/pb/signal"
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	"nhooyr.io/websocket"
-	"nhooyr.io/websocket/wsjson"
-)
 
-type subscriber map[pb.TrackSource]*subscriberChans
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+	"github.com/SB-IM/skywalker/internal/broadcast/publisher"
+	"github.com/SB-IM/skywalker/internal/broadcast/session"
+	"github.com/SB-IM/skywalker/internal/broadcast/subscriber"
+)
 
 // Service consists of many sessions.
 type Service struct {
-	sessions map[machineID]subscriber
-	client   mqtt.Client
-	logger   zerolog.Logger
-	config   *ConfigOptions
+	client mqtt.Client
+	logger zerolog.Logger
+	config cfg.ConfigOptions
 }
 
-func New(ctx context.Context, config *ConfigOptions) *Service {
+func New(ctx context.Context, config *cfg.ConfigOptions) *Service {
 	return &Service{
-		sessions: make(map[machineID]subscriber),
-		client:   mqttclient.FromContext(ctx),
-		logger:   *log.Ctx(ctx),
-		config:   config,
+		client: mqttclient.FromContext(ctx),
+		logger: *log.Ctx(ctx),
+		config: *config,
 	}
 }
 
-// Broadcast broadcasts video streams following publisher -> transfer -> subscribers flow direction.
-func (svc *Service) Broadcast() error {
-	// Start subscriber signaling handler.
-	// Register Websockets handler.
-	http.HandleFunc(svc.config.WSServerConfigOptions.Path, svc.handleSubscription())
-	go func() {
-		// Start HTTP server.
-		svc.logger.Info().Str("host",
-			svc.config.WSServerConfigOptions.Host).Int("port",
-			svc.config.WSServerConfigOptions.Port,
-		).Msg("starting HTTP server for WebSocket")
-		svc.logger.Fatal().Err(http.ListenAndServe(
-			svc.config.WSServerConfigOptions.Host+":"+strconv.Itoa(svc.config.WSServerConfigOptions.Port),
-			nil))
-	}()
-
-	// Start publisher signaling worker.
-	publisher := newPublisher(svc.client, &svc.logger, svc.config.TopicConfigOptions)
-	publisher.Signal()
+func (s *Service) Broadcast() error {
+	sessions := make(session.Sessions)
+	pub := publisher.New(s.client, sessions, &s.logger, cfg.PublisherConfigOptions{
+		TopicConfigOptions:  s.config.TopicConfigOptions,
+		WebRTCConfigOptions: s.config.WebRTCConfigOptions,
+	})
+	pub.Signal()
 
-	// Use a loop to start endless broadcasting sessions.
-	for {
-		s := newSession(&publisher.publisherChans, &svc.logger, svc.config.WebRTCConfigOptions)
+	sub := subscriber.New(sessions, &s.logger, s.config.WebRTCConfigOptions)
+	handler := sub.Signal()
 
-		// You can get id and track source only when half of session (publisher session) completes.
-		// Therefore, you must start session first.
-		if err := s.start(func(id machineID, t pb.TrackSource, s *subscriberChans) {
-			// This is where get buggy. You can't make new inner map for each session,
-			// because by this way you erase previous inner map which has track_source key value.
-			// In face, the inner map should be shared between two sessions which have the same machine id.
-			if inner, ok := svc.sessions[id]; ok {
-				inner[t] = s
-			} else {
-				inner := make(subscriber)
-				inner[t] = s
-				svc.sessions[id] = inner
-			}
-		}); err != nil {
-			return fmt.Errorf("session failed: %w", err)
-		}
-	}
+	server := s.newServer(handler)
+	return server.ListenAndServe()
 }
 
-// handleSubscription is called every time after publisher session and at the start of subscriber session.
-// If the running order is wrong, it blocks forever.
-func (svc *Service) handleSubscription() http.HandlerFunc {
-	logger := svc.logger.With().Str("component", "subscriber").Logger()
-
-	return func(w http.ResponseWriter, r *http.Request) {
-		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
-			OriginPatterns: []string{"*"},
-		})
-		if err != nil {
-			logger.Panic().Err(err).Msg("webSocket failed to establish connection")
-		}
-		defer c.Close(websocket.StatusNormalClosure, "")
-
-		ctx, cancel := context.WithCancel(r.Context())
-		defer cancel()
-
-		// Continually reading from WebSocket connection.
-		// To reduce complexity, we don't use channel pipeline transporting reading and writing messages.
-		for {
-			var offer pb.SessionDescription
-			if err := wsjson.Read(ctx, c, &offer); err != nil {
-				logger.Err(err).Msg("could not read message")
-				return
-			}
-			logger.Debug().
-				Str("offer", offer.String()).
-				Str("offer.id", offer.Id).
-				Int32("offer.track_source", int32(offer.TrackSource)).
-				Msg("received offer")
-
-			var (
-				offerChan  chan *pb.SessionDescription
-				answerChan chan *pb.SessionDescription
-			)
-			// The subscriber's sdp id must be equal to session's id.
-			if inner, ok := svc.sessions[machineID(offer.Id)]; ok {
-				logger.Debug().
-					Str("offer.id", offer.Id).
-					Int32("offer.track_source", int32(offer.TrackSource)).
-					Msg("got machine id")
-				// The subscriber's sdp track source must be equal to session's track source.
-				if subscriber, ok := inner[offer.TrackSource]; ok {
-					offerChan = subscriber.offerChan
-					answerChan = subscriber.answerChan
-					logger.Debug().
-						Str("offer.id", offer.Id).
-						Int32("offer.track_source", int32(offer.TrackSource)).
-						Msg("got subscriber channels")
-				}
-			}
-			// If offerChan or answerChan is nil, it means offer.Id or offer.TrackSource does not exists in any session.
-			// And this request message is invalid.
-			if offerChan == nil || answerChan == nil {
-				logger.Debug().
-					Str("offer.id", offer.Id).
-					Int32("offer.track_source", int32(offer.TrackSource)).
-					Msg("offerChan or answerChan is nil")
-
-				if err := wsjson.Write(ctx, c, "wrong id or track_source"); err != nil {
-					logger.Err(err).Msg("could not write message")
-				}
-				continue
-			}
-			// TODO: Timeout for sending and receiving in case of blocking side.
-			offerChan <- &offer
-
-			answer := <-answerChan
-			if err := wsjson.Write(ctx, c, answer); err != nil {
-				logger.Err(err).Msg("could not write message")
-			}
-		}
+func (s *Service) newServer(handler http.Handler) *http.Server {
+	return &http.Server{
+		Handler: handler,
+		Addr:    s.config.Host + ":" + strconv.Itoa(s.config.Port),
+		// Good practice: enforce timeouts for servers you create!
+		WriteTimeout: 15 * time.Second,
+		ReadTimeout:  15 * time.Second,
 	}
 }
diff --git a/internal/broadcast/cfg.go b/internal/broadcast/cfg/cfg.go
similarity index 60%
rename from internal/broadcast/cfg.go
rename to internal/broadcast/cfg/cfg.go
index 63b9918..83e1c43 100644
--- a/internal/broadcast/cfg.go
+++ b/internal/broadcast/cfg/cfg.go
@@ -1,9 +1,14 @@
-package broadcast
+package cfg
 
 type ConfigOptions struct {
 	WebRTCConfigOptions
 	TopicConfigOptions
-	WSServerConfigOptions
+	ServerConfigOptions
+}
+
+type PublisherConfigOptions struct {
+	TopicConfigOptions
+	WebRTCConfigOptions
 }
 
 type WebRTCConfigOptions struct {
@@ -15,8 +20,7 @@ type TopicConfigOptions struct {
 	AnswerTopic string
 }
 
-type WSServerConfigOptions struct {
+type ServerConfigOptions struct {
 	Host string
 	Port int
-	Path string
 }
diff --git a/internal/broadcast/publisher.go b/internal/broadcast/publisher.go
deleted file mode 100644
index 717eefe..0000000
--- a/internal/broadcast/publisher.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package broadcast
-
-import (
-	"strconv"
-
-	pb "github.com/SB-IM/pb/signal"
-	mqtt "github.com/eclipse/paho.mqtt.golang"
-	"github.com/rs/zerolog"
-	"google.golang.org/protobuf/proto"
-)
-
-type publisherChans struct {
-	OfferChan  chan *pb.SessionDescription
-	AnswerChan chan *pb.SessionDescription
-}
-
-type publisher struct {
-	client         mqtt.Client
-	publisherChans publisherChans
-	logger         zerolog.Logger
-	config         TopicConfigOptions
-}
-
-func newPublisher(client mqtt.Client, logger *zerolog.Logger, config TopicConfigOptions) *publisher {
-	return &publisher{
-		client: client,
-		publisherChans: publisherChans{ // The channel buffer size limits concurrency
-			OfferChan:  make(chan *pb.SessionDescription, 1),
-			AnswerChan: make(chan *pb.SessionDescription, 1),
-		},
-		logger: *logger,
-		config: config,
-	}
-}
-
-func (p *publisher) Signal() {
-	p.logger = p.logger.With().Str("component", "publisher").Logger()
-
-	// The receiving topic is the same for each edge device, but message payload is different.
-	// The id and trackSource in payload determine the following publishing topic.
-	// Receive remote description with MQTT.
-	t := p.client.Subscribe(p.config.OfferTopic, 1, func(c mqtt.Client, msg mqtt.Message) {
-		var offer pb.SessionDescription
-		if err := proto.Unmarshal(msg.Payload(), &offer); err != nil {
-			p.logger.Fatal().Err(err).Msg("could not unmarshal sdp")
-		}
-		p.logger.Debug().
-			Str("id", offer.Id).
-			Str("topic", p.config.OfferTopic).
-			Int32("track_source", int32(offer.TrackSource)).
-			Msg("received offer from edge")
-		p.publisherChans.OfferChan <- &offer
-
-		answer := <-p.publisherChans.AnswerChan
-		payload, err := proto.Marshal(answer)
-		if err != nil {
-			p.logger.Fatal().Err(err).Msg("could not encode sdp")
-		}
-
-		answerTopic := p.config.AnswerTopic + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
-		// The publishing topic is unique to each edge device and is determined by above receiving message payload.
-		t := p.client.Publish(answerTopic, 1, true, payload)
-		// Handle the token in a go routine so this loop keeps sending messages regardless of delivery status
-		go func() {
-			<-t.Done()
-			if t.Error() != nil {
-				p.logger.Fatal().Err(t.Error()).Msgf("could not publish to %s", answerTopic)
-			} else {
-				p.logger.Debug().Str("topic", answerTopic).Msg("sent answer to edge")
-			}
-		}()
-	})
-	// the connection handler is called in a goroutine so blocking here would hot cause an issue. However as blocking
-	// in other handlers does cause problems its best to just assume we should not block
-	go func() {
-		<-t.Done()
-		if t.Error() != nil {
-			p.logger.Fatal().Err(t.Error()).Msgf("could not subscribe to %s", p.config.OfferTopic)
-		} else {
-			p.logger.Info().Msgf("subscribed to %s", p.config.OfferTopic)
-		}
-	}()
-}
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
new file mode 100644
index 0000000..c1e4c28
--- /dev/null
+++ b/internal/broadcast/publisher/publisher.go
@@ -0,0 +1,137 @@
+package publisher
+
+import (
+	"fmt"
+	"strconv"
+
+	pb "github.com/SB-IM/pb/signal"
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/pion/webrtc/v3"
+	"github.com/rs/zerolog"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+	"github.com/SB-IM/skywalker/internal/broadcast/session"
+	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
+)
+
+// Publisher stands for a publisher webRTC peer.
+type Publisher struct {
+	client mqtt.Client
+	logger zerolog.Logger
+	config cfg.PublisherConfigOptions
+
+	// sessions must be created before used by publisher and is shared between publishers and subscribers.
+	// It's mainly written and maintained by publishers
+	sessions session.Sessions
+}
+
+// New returns a new Publisher.
+func New(
+	client mqtt.Client,
+	sessions session.Sessions,
+	logger *zerolog.Logger,
+	config cfg.PublisherConfigOptions,
+) *Publisher {
+	l := logger.With().Str("component", "Publisher").Logger()
+	return &Publisher{
+		client:   client,
+		logger:   l,
+		config:   config,
+		sessions: sessions,
+	}
+}
+
+// Signal performs webRTC signaling for all publisher peers.
+func (p *Publisher) Signal() {
+	// The receiving topic is the same for each edge device, but message payload is different.
+	// The id and trackSource in payload determine the following publishing topic.
+	// Receive remote SDP with MQTT.
+	t := p.client.Subscribe(p.config.OfferTopic, 1, p.handleMessage())
+	// the connection handler is called in a goroutine so blocking here would hot cause an issue. However as blocking
+	// in other handlers does cause problems its best to just assume we should not block
+	go func() {
+		<-t.Done()
+		if t.Error() != nil {
+			p.logger.Err(t.Error()).Msgf("could not subscribe to %s", p.config.OfferTopic)
+		} else {
+			p.logger.Info().Msgf("subscribed to %s", p.config.OfferTopic)
+		}
+	}()
+}
+
+// handleMessage handles MQTT subscription message.
+func (p *Publisher) handleMessage() mqtt.MessageHandler {
+	return func(c mqtt.Client, m mqtt.Message) {
+		var offer pb.SessionDescription
+		if err := proto.Unmarshal(m.Payload(), &offer); err != nil {
+			p.logger.Err(err).Msg("could not unmarshal sdp")
+			return
+		}
+		p.logger.Debug().
+			Str("id", offer.Id).
+			Str("topic", p.config.OfferTopic).
+			Int32("track_source", int32(offer.TrackSource)).
+			Msg("received offer from edge")
+
+		answer, videoTrack, err := p.signalPeerConnection(&offer)
+		if err != nil {
+			p.logger.Err(err).Msg("failed to signal peer connection")
+			return
+		}
+
+		payload, err := proto.Marshal(answer)
+		if err != nil {
+			p.logger.Err(err).Msg("could not encode sdp")
+			return
+		}
+
+		// The publishing topic is unique to each edge device and is determined by above receiving message payload.
+		answerTopic := p.config.AnswerTopic + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
+		t := c.Publish(answerTopic, 1, true, payload)
+		<-t.Done()
+		if t.Error() != nil {
+			p.logger.Err(t.Error()).Msgf("could not publish to %s", answerTopic)
+			return
+		}
+		p.logger.Debug().Str("topic", answerTopic).Msg("sent answer to edge")
+
+		// Register session on signaling success.
+		p.registerSession(session.MachineID(offer.Id), offer.TrackSource, videoTrack)
+	}
+}
+
+// signalPeerConnection creates video track and performs webRTC signaling.
+func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription) (
+	*pb.SessionDescription,
+	*webrtc.TrackLocalStaticRTP,
+	error,
+) {
+	videoTrack, err := webrtcx.CreateLocalTrack()
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not create webRTC local video track: %w", err)
+	}
+	w := webrtcx.New(videoTrack, p.config.WebRTCConfigOptions, &p.logger)
+
+	// TODO: handle blocking case with timeout for channels.
+	w.OfferChan <- offer
+	if err := w.CreatePublisher(); err != nil {
+		return nil, nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
+	}
+
+	return <-w.AnswerChan, videoTrack, nil
+}
+
+func (p *Publisher) registerSession(
+	id session.MachineID,
+	trackSource pb.TrackSource,
+	videoTrack *webrtc.TrackLocalStaticRTP,
+) {
+	if s, ok := p.sessions[id]; ok {
+		s[trackSource] = videoTrack
+	} else {
+		s := make(session.Session)
+		s[trackSource] = videoTrack
+		p.sessions[id] = s
+	}
+}
diff --git a/internal/broadcast/session.go b/internal/broadcast/session.go
deleted file mode 100644
index 90dbb23..0000000
--- a/internal/broadcast/session.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package broadcast
-
-import (
-	"fmt"
-
-	pb "github.com/SB-IM/pb/signal"
-	"github.com/rs/zerolog"
-)
-
-type machineID string
-
-type middleFunc func(id machineID, trackSource pb.TrackSource, subscriber *subscriberChans)
-
-type subscriberChans struct {
-	offerChan  chan *pb.SessionDescription
-	answerChan chan *pb.SessionDescription
-}
-
-// session consists of three peers, one publisher, one local transfer and multiple subscribers.
-// A session is driven by pub/sub events.
-type session struct {
-	// id is the id of a publisher and also id of this session.
-	// A subscriber subscribes video track identified by this unique id.
-	id machineID
-
-	// trackSource is the source of video track, either drone or monitor.
-	trackSource pb.TrackSource
-
-	// publisherChans contains edge device publisher delivering channels.
-	// The channels must be initialized before use.
-	publisherChans *publisherChans
-
-	// subscriberChans contains browser subscriber delivering channels.
-	// The channels must be initialized before use.
-	subscriberChans *subscriberChans
-
-	logger zerolog.Logger
-	config WebRTCConfigOptions
-}
-
-func newSession(publisherChans *publisherChans, logger *zerolog.Logger, config WebRTCConfigOptions) *session {
-	return &session{
-		publisherChans: publisherChans,
-		subscriberChans: &subscriberChans{
-			offerChan:  make(chan *pb.SessionDescription, 1),
-			answerChan: make(chan *pb.SessionDescription, 1),
-		},
-		logger: *logger,
-		config: config,
-	}
-}
-
-// start starts a session broadcasting video track from one publisher peer to at least one subscriber peer.
-func (s *session) start(middle middleFunc) error {
-	localTrack, err := createLocalTrack()
-	if err != nil {
-		return err
-	}
-	s.logger.Debug().Msg("created local track")
-
-	if err := s.createPublisher(localTrack); err != nil {
-		return fmt.Errorf("failed to crate publisher: %w", err)
-	}
-	s.logger.Debug().Str("id", string(s.id)).Int32("track_source", int32(s.trackSource)).Msg("created a publisher")
-
-	middle(s.id, s.trackSource, s.subscriberChans)
-
-	go func() {
-		// Use a loop to start endless subscriber sessions.
-		for {
-			if err := s.createSubscriber(localTrack); err != nil {
-				s.logger.Err(err).Msg("failed to create subscriber")
-			}
-		}
-	}()
-
-	return nil
-}
diff --git a/internal/broadcast/session/session.go b/internal/broadcast/session/session.go
new file mode 100644
index 0000000..5364a97
--- /dev/null
+++ b/internal/broadcast/session/session.go
@@ -0,0 +1,15 @@
+package session
+
+import (
+	pb "github.com/SB-IM/pb/signal"
+	"github.com/pion/webrtc/v3"
+)
+
+// MachineID is unique id for edge device.
+type MachineID string
+
+// Session maps track source to local video track.
+type Session map[pb.TrackSource]*webrtc.TrackLocalStaticRTP
+
+// Sessions maps machine id to session.
+type Sessions map[MachineID]Session
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
new file mode 100644
index 0000000..c641100
--- /dev/null
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -0,0 +1,83 @@
+package subscriber
+
+import (
+	"encoding/json"
+	"net/http"
+
+	pb "github.com/SB-IM/pb/signal"
+	"github.com/gorilla/mux"
+	"github.com/rs/zerolog"
+
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+	"github.com/SB-IM/skywalker/internal/broadcast/session"
+	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
+)
+
+// Subscriber stands for a subscriber webRTC peer.
+type Subscriber struct {
+	config cfg.WebRTCConfigOptions
+	logger zerolog.Logger
+
+	// sessions must be created before used by publisher and is shared between publishers ans subscribers.
+	// It's only read by subscriber.
+	sessions session.Sessions
+}
+
+// New returns a new Subscriber.
+func New(sessions session.Sessions, logger *zerolog.Logger, config cfg.WebRTCConfigOptions) *Subscriber {
+	l := logger.With().Str("component", "Subscriber").Logger()
+	return &Subscriber{
+		sessions: sessions,
+		config:   config,
+		logger:   l,
+	}
+}
+
+// Signal performs webRTC signaling for all subscriber peers.
+func (s *Subscriber) Signal() http.Handler {
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).
+		Methods(http.MethodPost).
+		Headers("Content-Type", "application/json")
+
+	return r
+}
+
+// handleSignal handles HTTP subscriber.
+func (s *Subscriber) handleSignal() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// w.Header().Set("Access-Control-Allow-Origin", "*")
+		// w.Header().Set("Access-Control-Allow-Methods", "*")
+		// w.Header().Set("Access-Control-Allow-Headers", "*")
+
+		var offer pb.SessionDescription
+		if err := json.NewDecoder(r.Body).Decode(&offer); err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		ses, ok := s.sessions[session.MachineID(offer.Id)]
+		if !ok {
+			http.Error(w, "wrong id", http.StatusBadRequest)
+			return
+		}
+		videoTrack, ok := ses[offer.TrackSource]
+		if !ok {
+			http.Error(w, "wrong track_source", http.StatusBadRequest)
+			return
+		}
+
+		wcx := webrtcx.New(videoTrack, s.config, &s.logger)
+		// TODO: handle blocking case with timeout for channels.
+		wcx.OfferChan <- &offer
+		if err := wcx.CreateSubscriber(); err != nil {
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+
+		answer := <-wcx.AnswerChan
+		if err := json.NewEncoder(w).Encode(&answer); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	}
+}
diff --git a/internal/broadcast/webrtc.go b/internal/broadcast/webrtc/webrtc.go
similarity index 51%
rename from internal/broadcast/webrtc.go
rename to internal/broadcast/webrtc/webrtc.go
index 5d5836c..9d8ee65 100644
--- a/internal/broadcast/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -1,4 +1,4 @@
-package broadcast
+package webrtc
 
 import (
 	"errors"
@@ -10,46 +10,42 @@ import (
 	"github.com/pion/rtcp"
 	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
-)
 
-// actor distinguishes between publisher and subscriber.
-type actor int
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+)
 
 const (
-	peerPublisher actor = iota
-	peerSubscriber
-
 	rtcpPLIInterval = time.Second * 3
 )
 
-func (a actor) string() string {
-	if a == peerPublisher {
-		return "publisher"
-	}
-	return "subscriber"
+type WebRTC struct {
+	logger     zerolog.Logger
+	config     cfg.WebRTCConfigOptions
+	videoTrack *webrtc.TrackLocalStaticRTP
+	OfferChan  chan *pb.SessionDescription
+	AnswerChan chan *pb.SessionDescription
 }
 
-// createLocalTrack creates a local video track shared between publisher and subscriber peers.
-// localTrack is a transfer that transfers video track between publisher and subscriber peers.
-// For localTrack, there can only be one publisher peer, but subscribers can be many.
-func createLocalTrack() (*webrtc.TrackLocalStaticRTP, error) {
-	videoTrack, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
-	if err != nil {
-		return nil, fmt.Errorf("coult not create TrackLocalStaticRTP: %w", err)
+// New returns a new WebRTC.
+func New(videoTrack *webrtc.TrackLocalStaticRTP, config cfg.WebRTCConfigOptions, logger *zerolog.Logger) *WebRTC {
+	return &WebRTC{
+		logger:     *logger,
+		config:     config,
+		videoTrack: videoTrack,
+		OfferChan:  make(chan *pb.SessionDescription, 1), // Make 1 buffer so offer sending never blocks
+		AnswerChan: make(chan *pb.SessionDescription, 1), // Make 1 buffer so answer sending never blocks
 	}
-	return videoTrack, nil
 }
 
-// createPublisher creates a session between local and remote webRTC peer which is a publisher.
-// It receives and transfers video track from remote publisher peer to local peer and is fired by remote publisher peer by RPC call.
-func (s *session) createPublisher(videoTrack *webrtc.TrackLocalStaticRTP) error {
-	peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{s.config.ICEServer},
-			},
-		},
-	})
+// CreateLocalTrack creates a TrackLocalStaticRTP and is only used by publisher.
+func CreateLocalTrack() (*webrtc.TrackLocalStaticRTP, error) {
+	return webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
+}
+
+// CreatePublisher creates a webRTC publisher peer.
+// Caller must send offer first by OfferChan or this function blocks waiting for receiving offer forever.
+func (w *WebRTC) CreatePublisher() error {
+	peerConnection, err := w.newPeerConnection()
 	if err != nil {
 		return fmt.Errorf("could not create PeerConnection: %w", err)
 	}
@@ -62,37 +58,35 @@ func (s *session) createPublisher(videoTrack *webrtc.TrackLocalStaticRTP) error
 	// Set a handler for when a new remote track starts, this just distributes all our packets
 	// to connected peers
 	peerConnection.OnTrack(func(t *webrtc.TrackRemote, _ *webrtc.RTPReceiver) {
-		go sendRTCP(peerConnection, t, &s.logger)
-
+		go w.sendRTCP(peerConnection, t)
 		rtpBuf := make([]byte, 1400)
 		for {
 			i, _, readErr := t.Read(rtpBuf)
 			if readErr != nil {
-				s.logger.Panic().Err(err).Msg("could not read buffer")
+				w.logger.Panic().Err(err).Msg("could not read buffer")
 			}
-
 			// ErrClosedPipe means we don't have any subscribers, this is ok if no peers have connected yet
-			if _, err = videoTrack.Write(rtpBuf[:i]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
-				s.logger.Panic().Err(err).Msg("could not write video track")
+			if _, err = w.videoTrack.Write(rtpBuf[:i]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
+				w.logger.Panic().Err(err).Msg("could not write video track")
 			}
 		}
 	})
 
-	if err := s.createPeerConnection(peerConnection, peerPublisher); err != nil {
+	if err := w.signalPeerConnection(peerConnection); err != nil {
 		return fmt.Errorf("failed to create peer connection: %w", err)
 	}
-	s.logger.Debug().Str("id", string(s.id)).Int32("track_source", int32(s.trackSource)).Msg("created PeerConnection for publisher")
+	w.logger.Debug().Msg("created peer connection for publisher")
 
 	return nil
 }
 
-// createSubscriber creates a session between local and remote webRTC peer which is a subscriber.
-// It broadcasts video track from local to remote subscriber peer and is fired by remote subscriber peer by RCP call.
-func (s *session) createSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error {
+// CreateSubscriber creates a webRTC subscriber peer.
+// Caller must send offer first by OfferChan or this function blocks waiting for receiving offer forever.
+func (w *WebRTC) CreateSubscriber() error {
 	peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
 			{
-				URLs: []string{s.config.ICEServer},
+				URLs: []string{w.config.ICEServer},
 			},
 		},
 	})
@@ -100,51 +94,32 @@ func (s *session) createSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 		return fmt.Errorf("could not create PeerConnection: %w", err)
 	}
 
-	rtpSender, err := peerConnection.AddTrack(videoTrack)
+	rtpSender, err := peerConnection.AddTrack(w.videoTrack)
 	if err != nil {
 		return fmt.Errorf("could not add track: %w", err)
 	}
-	go processRTCP(rtpSender, &s.logger)
+	go w.processRTCP(rtpSender)
 
-	if err := s.createPeerConnection(peerConnection, peerSubscriber); err != nil {
+	if err := w.signalPeerConnection(peerConnection); err != nil {
 		return fmt.Errorf("failed to create peer connection: %w", err)
 	}
-	s.logger.Debug().Msg("created PeerConnection for subscriber")
+	w.logger.Debug().Msg("created peer connection for subscriber")
 
 	return nil
 }
 
-// createPeerConnection creates peer connection to the remote webRTC peer who sends SDP offer.
-// It's a generic function for both publisher and subscriber.
-func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, actor actor) error {
-	// Receive remote offer.
-	// TODO: Timeout for sending and receiving in case of blocking side.
-	var offer *pb.SessionDescription
-	if actor == peerPublisher {
-		offer = <-s.publisherChans.OfferChan
-
-		// Set up session identity.
-		s.id = machineID(offer.Id)
-		s.trackSource = offer.TrackSource
-	} else {
-		offer = <-s.subscriberChans.offerChan
-	}
-	logger := s.logger.With().
-		Str("actor", actor.string()).
-		Str("id", offer.Id).
-		Int32("track_source",
-			int32(offer.TrackSource)).
-		Logger()
+func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) error {
+	offer := <-w.OfferChan
 
 	// Set the handler for ICE connection state
 	// This will notify you when the peer has connected/disconnected
 	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
-		logger.Debug().Str("state", connectionState.String()).Msg("ICE connection state has changed")
+		w.logger.Debug().Str("state", connectionState.String()).Msg("ICE connection state has changed")
 		if connectionState == webrtc.ICEConnectionStateFailed {
 			if err := peerConnection.Close(); err != nil {
-				logger.Panic().Err(err).Msg("could not close PeerConnection")
+				w.logger.Panic().Err(err).Msg("could not close peer connection")
 			}
-			logger.Debug().Msg("PeerConnection has been closed")
+			w.logger.Debug().Msg("peer connection has been closed")
 		}
 	})
 
@@ -172,18 +147,24 @@ func (s *session) createPeerConnection(peerConnection *webrtc.PeerConnection, ac
 	// Send answer of local description.
 	// This is a universal answer for both publisher and subscriber in protobuf format.
 	sdp := webrtcSdp2pbSdp(peerConnection.LocalDescription())
-	if actor == peerPublisher {
-		s.publisherChans.AnswerChan <- sdp
-	} else {
-		s.subscriberChans.answerChan <- sdp
-	}
+	w.AnswerChan <- sdp
 
 	return nil
 }
 
+func (w *WebRTC) newPeerConnection() (*webrtc.PeerConnection, error) {
+	return webrtc.NewPeerConnection(webrtc.Configuration{
+		ICEServers: []webrtc.ICEServer{
+			{
+				URLs: []string{w.config.ICEServer},
+			},
+		},
+	})
+}
+
 // sendRTCP sends a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
 // This can be less wasteful by processing incoming RTCP events, then we would emit a NACK/PLI when a viewer requests it
-func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote, logger *zerolog.Logger) {
+func (w *WebRTC) sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote) {
 	ticker := time.NewTicker(rtcpPLIInterval)
 	for range ticker.C {
 		if rtcpSendErr := peerConnection.WriteRTCP([]rtcp.Packet{
@@ -191,7 +172,7 @@ func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRe
 				MediaSSRC: uint32(remoteTrack.SSRC()),
 			},
 		}); rtcpSendErr != nil {
-			logger.Panic().Err(rtcpSendErr).Send()
+			w.logger.Panic().Err(rtcpSendErr).Send()
 		}
 	}
 }
@@ -199,11 +180,11 @@ func sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRe
 // processRTCP reads incoming RTCP packets
 // Before these packets are returned they are processed by interceptors.
 // For things like NACK this needs to be called.
-func processRTCP(rtpSender *webrtc.RTPSender, logger *zerolog.Logger) {
+func (w *WebRTC) processRTCP(rtpSender *webrtc.RTPSender) {
 	rtcpBuf := make([]byte, 1500)
 	for {
 		if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
-			logger.Err(rtcpErr).Send()
+			w.logger.Err(rtcpErr).Send()
 			return
 		}
 	}

From a036852ba82cf335c2b08541ab4662026b160a3f Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 10:02:16 +0800
Subject: [PATCH 07/52] Update static js

---
 e2e/broadcast/static/index.js | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/e2e/broadcast/static/index.js b/e2e/broadcast/static/index.js
index 85409b4..035f709 100644
--- a/e2e/broadcast/static/index.js
+++ b/e2e/broadcast/static/index.js
@@ -9,7 +9,7 @@
     });
 
     let log = (msg) => {
-      document.getElementById(`log${i}`).innerHTML += msg + "<br>";
+      document.getElementById(`log${i + 1}`).innerHTML += msg + "<br>";
     };
 
     pc.ontrack = function (event) {
@@ -28,30 +28,27 @@
     pc.createOffer()
       .then((offer) => {
         pc.setLocalDescription(offer);
-        console.log(`Sending offer ${i}: ${offer}`);
 
         sdp = {
           id: "fa955cc6881b4b45b49ffbf2d81e7223",
           track_source: i,
           sdp: offer,
         };
+        console.log(`Sending offer ${i}: ${JSON.stringify(sdp)}`);
 
-        return fetch("http://localhost:8080/v1/broadcast/signal", {
+        return fetch("/v1/broadcast/signal", {
           method: "post",
           headers: {
             Accept: "application/json",
             "Content-Type": "application/json",
-            'Access-Control-Allow-Origin':'*'
           },
-          mode: 'no-cors',
           body: JSON.stringify(sdp),
         });
       })
       .then((res) => res.json())
       .then((res) => {
-        console.log(`Received answer ${i}: ${res}`);
-
         pc.setRemoteDescription(res);
+        console.log(`Received answer ${i}: ${JSON.stringify(res)}`);
       })
       .catch(log);
   }

From ff3b2708e3677694e94a5ec8fe79f726dac8eb4f Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 10:38:44 +0800
Subject: [PATCH 08/52] Update broadcast e2e

---
 Makefile                      | 4 ++++
 e2e/broadcast/main.go         | 2 +-
 e2e/broadcast/static/index.js | 3 ++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 5f91568..de04839 100644
--- a/Makefile
+++ b/Makefile
@@ -50,3 +50,7 @@ log-livestream:
 .PHONY: log-broadcast
 log-broadcast:
 	@docker-compose -f docker/docker-compose.dev.yaml logs -f broadcast
+
+.PHONY: e2e-broadcast
+e2e-broadcast:
+	@go run ./e2e/broadcast
diff --git a/e2e/broadcast/main.go b/e2e/broadcast/main.go
index 5acba86..b21af73 100644
--- a/e2e/broadcast/main.go
+++ b/e2e/broadcast/main.go
@@ -6,6 +6,6 @@ import (
 )
 
 func main() {
-	http.Handle("/", http.FileServer(http.Dir("./static")))
+	http.Handle("/", http.FileServer(http.Dir("e2e/broadcast/static")))
 	log.Fatal(http.ListenAndServe(":7070", nil))
 }
diff --git a/e2e/broadcast/static/index.js b/e2e/broadcast/static/index.js
index 035f709..fba9ee0 100644
--- a/e2e/broadcast/static/index.js
+++ b/e2e/broadcast/static/index.js
@@ -36,13 +36,14 @@
         };
         console.log(`Sending offer ${i}: ${JSON.stringify(sdp)}`);
 
-        return fetch("/v1/broadcast/signal", {
+        return fetch(`http://${window.location.hostname}:8080/v1/broadcast/signal`, {
           method: "post",
           headers: {
             Accept: "application/json",
             "Content-Type": "application/json",
           },
           body: JSON.stringify(sdp),
+          mode: "no-cors"
         });
       })
       .then((res) => res.json())

From 9e3b5fa620de5a490e5c00f95ec4fe8560ccfb89 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 15:26:57 +0800
Subject: [PATCH 09/52] Minor update, add more debug log

---
 internal/broadcast/broadcast.go               |  1 +
 internal/broadcast/publisher/publisher.go     | 24 ++++++++------
 internal/broadcast/subscriber/subscriber.go   | 24 ++++++++------
 .../broadcast/subscriber/subscriber_test.go   | 32 +++++++++++++++++++
 4 files changed, 62 insertions(+), 19 deletions(-)
 create mode 100644 internal/broadcast/subscriber/subscriber_test.go

diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index 2d095f4..45a42fd 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -44,6 +44,7 @@ func (s *Service) Broadcast() error {
 	handler := sub.Signal()
 
 	server := s.newServer(handler)
+	s.logger.Debug().Str("host", s.config.Host).Int("port", s.config.Port).Msg("starting HTTP server")
 	return server.ListenAndServe()
 }
 
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index c1e4c28..ec11939 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -68,21 +68,24 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 			p.logger.Err(err).Msg("could not unmarshal sdp")
 			return
 		}
-		p.logger.Debug().
+
+		logger := p.logger.With().
+			Str("offer_topic", p.config.OfferTopic).
 			Str("id", offer.Id).
-			Str("topic", p.config.OfferTopic).
 			Int32("track_source", int32(offer.TrackSource)).
-			Msg("received offer from edge")
+			Logger()
+		logger.Debug().Str("offer", offer.String()).Msg("received offer from edge")
 
-		answer, videoTrack, err := p.signalPeerConnection(&offer)
+		answer, videoTrack, err := p.signalPeerConnection(&offer, &logger)
 		if err != nil {
-			p.logger.Err(err).Msg("failed to signal peer connection")
+			logger.Err(err).Msg("failed to signal peer connection")
 			return
 		}
+		logger.Debug().Msg("Successfully signaled peer connection")
 
 		payload, err := proto.Marshal(answer)
 		if err != nil {
-			p.logger.Err(err).Msg("could not encode sdp")
+			logger.Err(err).Msg("could not encode sdp")
 			return
 		}
 
@@ -94,7 +97,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 			p.logger.Err(t.Error()).Msgf("could not publish to %s", answerTopic)
 			return
 		}
-		p.logger.Debug().Str("topic", answerTopic).Msg("sent answer to edge")
+		logger.Debug().Str("answer_topic", answerTopic).Str("answer", answer.String()).Msg("sent answer to edge")
 
 		// Register session on signaling success.
 		p.registerSession(session.MachineID(offer.Id), offer.TrackSource, videoTrack)
@@ -102,7 +105,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 }
 
 // signalPeerConnection creates video track and performs webRTC signaling.
-func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription) (
+func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *zerolog.Logger) (
 	*pb.SessionDescription,
 	*webrtc.TrackLocalStaticRTP,
 	error,
@@ -111,13 +114,16 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription) (
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create webRTC local video track: %w", err)
 	}
-	w := webrtcx.New(videoTrack, p.config.WebRTCConfigOptions, &p.logger)
+	logger.Debug().Msg("created video track")
+
+	w := webrtcx.New(videoTrack, p.config.WebRTCConfigOptions, logger)
 
 	// TODO: handle blocking case with timeout for channels.
 	w.OfferChan <- offer
 	if err := w.CreatePublisher(); err != nil {
 		return nil, nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
 	}
+	logger.Debug().Msg("created publisher")
 
 	return <-w.AnswerChan, videoTrack, nil
 }
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index c641100..69df898 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -36,48 +36,52 @@ func New(sessions session.Sessions, logger *zerolog.Logger, config cfg.WebRTCCon
 // Signal performs webRTC signaling for all subscriber peers.
 func (s *Subscriber) Signal() http.Handler {
 	r := mux.NewRouter()
-	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).
-		Methods(http.MethodPost).
-		Headers("Content-Type", "application/json")
-
+	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).Methods(http.MethodPost)
+	s.logger.Debug().Msg("registered signal HTTP handler")
 	return r
 }
 
 // handleSignal handles HTTP subscriber.
 func (s *Subscriber) handleSignal() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		// w.Header().Set("Access-Control-Allow-Origin", "*")
-		// w.Header().Set("Access-Control-Allow-Methods", "*")
-		// w.Header().Set("Access-Control-Allow-Headers", "*")
-
 		var offer pb.SessionDescription
 		if err := json.NewDecoder(r.Body).Decode(&offer); err != nil {
+			s.logger.Err(err).Msg("could not decode request json body")
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
+		logger := s.logger.With().Str("id", offer.Id).Int32("track_source", int32(offer.TrackSource)).Logger()
+		logger.Debug().Str("offer", offer.String()).Msg("received offer from subscriber")
 
 		ses, ok := s.sessions[session.MachineID(offer.Id)]
 		if !ok {
+			logger.Warn().Msg("no machine in found in existing sessions")
 			http.Error(w, "wrong id", http.StatusBadRequest)
 			return
 		}
 		videoTrack, ok := ses[offer.TrackSource]
 		if !ok {
+			logger.Warn().Msg("no track source found in existing sessions")
 			http.Error(w, "wrong track_source", http.StatusBadRequest)
 			return
 		}
 
-		wcx := webrtcx.New(videoTrack, s.config, &s.logger)
+		wcx := webrtcx.New(videoTrack, s.config, &logger)
 		// TODO: handle blocking case with timeout for channels.
 		wcx.OfferChan <- &offer
 		if err := wcx.CreateSubscriber(); err != nil {
+			logger.Err(err).Msg("failed to create subscriber")
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
 		}
+		logger.Debug().Msg("successfully created subscriber")
 
 		answer := <-wcx.AnswerChan
-		if err := json.NewEncoder(w).Encode(&answer); err != nil {
+		w.Header().Add("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(&answer.Sdp); err != nil {
+			logger.Err(err).Msg("could not encode json response body")
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 		}
+		logger.Debug().Msg("sent answer to subscriber")
 	}
 }
diff --git a/internal/broadcast/subscriber/subscriber_test.go b/internal/broadcast/subscriber/subscriber_test.go
new file mode 100644
index 0000000..3b14429
--- /dev/null
+++ b/internal/broadcast/subscriber/subscriber_test.go
@@ -0,0 +1,32 @@
+package subscriber
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+	"github.com/rs/zerolog"
+)
+
+func TestSignalBadRequest(t *testing.T) {
+	logger := zerolog.Nop()
+	sub := New(nil, &logger, cfg.WebRTCConfigOptions{})
+
+	body := `{"id":"fa955cc6881b4b45b49ffbf2d81e7223","track_source":0,"sdp":{"type":"offer","sdp":"v=0\r\no=- 4497186609034332336 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS\r\nm=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 124 119 123 118 114 115 116\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:laPQ\r\na=ice-pwd:IEAqRBhJbIHYRq6fMDh7YZH9\r\na=ice-options:trickle\r\na=fingerprint:sha-256 28:5E:F4:E4:1F:A2:02:19:E3:5C:7C:B2:E2:8A:28:98:88:73:9F:0E:CD:17:E2:D0:98:99:6B:89:72:E8:58:5A\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 urn:3gpp:video-orientation\r\na=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\na=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\na=sendrecv\r\na=msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=rtcp-mux\r\na=rtcp-rsize\r\na=rtpmap:96 VP8/90000\r\na=rtcp-fb:96 goog-remb\r\na=rtcp-fb:96 transport-cc\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96 nack pli\r\na=rtpmap:97 rtx/90000\r\na=fmtp:97 apt=96\r\na=rtpmap:98 VP9/90000\r\na=rtcp-fb:98 goog-remb\r\na=rtcp-fb:98 transport-cc\r\na=rtcp-fb:98 ccm fir\r\na=rtcp-fb:98 nack\r\na=rtcp-fb:98 nack pli\r\na=fmtp:98 profile-id=0\r\na=rtpmap:99 rtx/90000\r\na=fmtp:99 apt=98\r\na=rtpmap:100 VP9/90000\r\na=rtcp-fb:100 goog-remb\r\na=rtcp-fb:100 transport-cc\r\na=rtcp-fb:100 ccm fir\r\na=rtcp-fb:100 nack\r\na=rtcp-fb:100 nack pli\r\na=fmtp:100 profile-id=2\r\na=rtpmap:101 rtx/90000\r\na=fmtp:101 apt=100\r\na=rtpmap:102 H264/90000\r\na=rtcp-fb:102 goog-remb\r\na=rtcp-fb:102 transport-cc\r\na=rtcp-fb:102 ccm fir\r\na=rtcp-fb:102 nack\r\na=rtcp-fb:102 nack pli\r\na=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r\na=rtpmap:121 rtx/90000\r\na=fmtp:121 apt=102\r\na=rtpmap:127 H264/90000\r\na=rtcp-fb:127 goog-remb\r\na=rtcp-fb:127 transport-cc\r\na=rtcp-fb:127 ccm fir\r\na=rtcp-fb:127 nack\r\na=rtcp-fb:127 nack pli\r\na=fmtp:127 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r\na=rtpmap:120 rtx/90000\r\na=fmtp:120 apt=127\r\na=rtpmap:125 H264/90000\r\na=rtcp-fb:125 goog-remb\r\na=rtcp-fb:125 transport-cc\r\na=rtcp-fb:125 ccm fir\r\na=rtcp-fb:125 nack\r\na=rtcp-fb:125 nack pli\r\na=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\na=rtpmap:107 rtx/90000\r\na=fmtp:107 apt=125\r\na=rtpmap:108 H264/90000\r\na=rtcp-fb:108 goog-remb\r\na=rtcp-fb:108 transport-cc\r\na=rtcp-fb:108 ccm fir\r\na=rtcp-fb:108 nack\r\na=rtcp-fb:108 nack pli\r\na=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\r\na=rtpmap:109 rtx/90000\r\na=fmtp:109 apt=108\r\na=rtpmap:124 H264/90000\r\na=rtcp-fb:124 goog-remb\r\na=rtcp-fb:124 transport-cc\r\na=rtcp-fb:124 ccm fir\r\na=rtcp-fb:124 nack\r\na=rtcp-fb:124 nack pli\r\na=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032\r\na=rtpmap:119 rtx/90000\r\na=fmtp:119 apt=124\r\na=rtpmap:123 H264/90000\r\na=rtcp-fb:123 goog-remb\r\na=rtcp-fb:123 transport-cc\r\na=rtcp-fb:123 ccm fir\r\na=rtcp-fb:123 nack\r\na=rtcp-fb:123 nack pli\r\na=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032\r\na=rtpmap:118 rtx/90000\r\na=fmtp:118 apt=123\r\na=rtpmap:114 red/90000\r\na=rtpmap:115 rtx/90000\r\na=fmtp:115 apt=114\r\na=rtpmap:116 ulpfec/90000\r\na=ssrc-group:FID 3334187606 1513158839\r\na=ssrc:3334187606 cname:2rzanSazYyXNIwZj\r\na=ssrc:3334187606 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:3334187606 mslabel:-\r\na=ssrc:3334187606 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 cname:2rzanSazYyXNIwZj\r\na=ssrc:1513158839 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 mslabel:-\r\na=ssrc:1513158839 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\n"}}`
+	req, err := http.NewRequest(http.MethodPost, "/v1/broadcast/signal", strings.NewReader(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Add("Content-Type", "application/json")
+
+	rr := httptest.NewRecorder()
+
+	r := sub.Signal()
+	r.ServeHTTP(rr, req)
+
+	if rr.Code != http.StatusBadRequest {
+		t.Errorf("handler returned wrong status code, got: %v want: %v", rr.Code, http.StatusBadRequest)
+	}
+}

From abc85090873cc35eec08c809c2a8ca86d7c94995 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 19:38:36 +0800
Subject: [PATCH 10/52] Minor update

---
 docker/docker-compose.dev.yaml              |  2 +-
 e2e/broadcast/static/index.html             | 44 +++++++++++++++-
 e2e/broadcast/static/index.js               | 56 ---------------------
 internal/broadcast/subscriber/subscriber.go | 12 +++--
 internal/broadcast/webrtc/webrtc.go         | 16 ++----
 5 files changed, 58 insertions(+), 72 deletions(-)
 delete mode 100644 e2e/broadcast/static/index.js

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 5ab3504..00fb512 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -22,7 +22,6 @@ services:
     image: ghcr.io/sb-im/skywalker:broadcast-latest
     container_name: broadcast
     command:
-      - --debug
       - broadcast
       - -c
       - /config/config.yaml
@@ -30,6 +29,7 @@ services:
       - "8080:8080"
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
+      - DEBUG=true
     volumes:
       - ../config/config.dev.yaml:/config/config.yaml
     depends_on:
diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index bc94727..e3bac6b 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -13,6 +13,48 @@
     <div id="log2"></div> <br />
 </body>
 
-<script type="text/javascript" src="index.js"></script>
+<script type="text/javascript">
+    for (let i = 0; i < 2; i++) {
+        let pc = new RTCPeerConnection()
+        pc.ontrack = function (event) {
+            var el = document.createElement(event.track.kind)
+            el.srcObject = event.streams[0]
+            el.autoplay = true
+            el.controls = true
+
+            document.getElementById('remoteVideos').appendChild(el)
+        }
+
+        let log = (msg) => {
+            document.getElementById(`log${i + 1}`).innerHTML += msg + "<br>";
+        };
+
+        pc.addTransceiver('video');
+
+        pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
+
+        pc.createOffer()
+            .then(offer => {
+                pc.setLocalDescription(offer)
+
+                sdp = {
+                    id: "fa955cc6881b4b45b49ffbf2d81e7223",
+                    track_source: i,
+                    sdp: offer,
+                };
+                return fetch(`/v1/broadcast/signal`, {
+                    method: 'post',
+                    headers: {
+                        'Accept': 'application/json, text/plain, */*',
+                        'Content-Type': 'application/json'
+                    },
+                    body: JSON.stringify(sdp)
+                })
+            })
+            .then(res => res.json())
+            .then(res => pc.setRemoteDescription(res))
+            .catch(alert)
+    }
+</script>
 
 </html>
diff --git a/e2e/broadcast/static/index.js b/e2e/broadcast/static/index.js
deleted file mode 100644
index fba9ee0..0000000
--- a/e2e/broadcast/static/index.js
+++ /dev/null
@@ -1,56 +0,0 @@
-(() => {
-  for (let i = 0; i < 2; i++) {
-    let pc = new RTCPeerConnection({
-      iceServers: [
-        {
-          urls: "stun:stun.l.google.com:19302",
-        },
-      ],
-    });
-
-    let log = (msg) => {
-      document.getElementById(`log${i + 1}`).innerHTML += msg + "<br>";
-    };
-
-    pc.ontrack = function (event) {
-      var el = document.createElement(event.track.kind);
-      el.srcObject = event.streams[0];
-      el.autoplay = true;
-      el.controls = true;
-
-      document.getElementById("remoteVideos").appendChild(el);
-    };
-
-    pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
-
-    pc.addTransceiver("video");
-
-    pc.createOffer()
-      .then((offer) => {
-        pc.setLocalDescription(offer);
-
-        sdp = {
-          id: "fa955cc6881b4b45b49ffbf2d81e7223",
-          track_source: i,
-          sdp: offer,
-        };
-        console.log(`Sending offer ${i}: ${JSON.stringify(sdp)}`);
-
-        return fetch(`http://${window.location.hostname}:8080/v1/broadcast/signal`, {
-          method: "post",
-          headers: {
-            Accept: "application/json",
-            "Content-Type": "application/json",
-          },
-          body: JSON.stringify(sdp),
-          mode: "no-cors"
-        });
-      })
-      .then((res) => res.json())
-      .then((res) => {
-        pc.setRemoteDescription(res);
-        console.log(`Received answer ${i}: ${JSON.stringify(res)}`);
-      })
-      .catch(log);
-  }
-})();
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 69df898..cdea616 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -3,6 +3,7 @@ package subscriber
 import (
 	"encoding/json"
 	"net/http"
+	"os"
 
 	pb "github.com/SB-IM/pb/signal"
 	"github.com/gorilla/mux"
@@ -37,6 +38,10 @@ func New(sessions session.Sessions, logger *zerolog.Logger, config cfg.WebRTCCon
 func (s *Subscriber) Signal() http.Handler {
 	r := mux.NewRouter()
 	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).Methods(http.MethodPost)
+
+	if os.Getenv("DEBUG") == "true" {
+		r.Handle("/", http.FileServer(http.Dir("e2e/broadcast/static")))
+	}
 	s.logger.Debug().Msg("registered signal HTTP handler")
 	return r
 }
@@ -77,11 +82,12 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 		logger.Debug().Msg("successfully created subscriber")
 
 		answer := <-wcx.AnswerChan
-		w.Header().Add("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(&answer.Sdp); err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(answer.Sdp); err != nil {
 			logger.Err(err).Msg("could not encode json response body")
 			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
 		}
-		logger.Debug().Msg("sent answer to subscriber")
+		logger.Debug().Str("answer", answer.Sdp.String()).Msg("sent answer to subscriber")
 	}
 }
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 9d8ee65..edbea5e 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -83,13 +83,7 @@ func (w *WebRTC) CreatePublisher() error {
 // CreateSubscriber creates a webRTC subscriber peer.
 // Caller must send offer first by OfferChan or this function blocks waiting for receiving offer forever.
 func (w *WebRTC) CreateSubscriber() error {
-	peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
-		ICEServers: []webrtc.ICEServer{
-			{
-				URLs: []string{w.config.ICEServer},
-			},
-		},
-	})
+	peerConnection, err := w.newPeerConnection()
 	if err != nil {
 		return fmt.Errorf("could not create PeerConnection: %w", err)
 	}
@@ -124,17 +118,17 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 	})
 
 	if err := peerConnection.SetRemoteDescription(pbSdp2webrtcSdp(offer)); err != nil {
-		return fmt.Errorf("could not set remote describption: %w", err)
+		return fmt.Errorf("could not set remote description: %w", err)
 	}
 
+	// Create channel that is blocked until ICE Gathering is complete
+	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+
 	answer, err := peerConnection.CreateAnswer(nil)
 	if err != nil {
 		return fmt.Errorf("could not create answer: %w", err)
 	}
 
-	// Create channel that is blocked until ICE Gathering is complete
-	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
-
 	if err = peerConnection.SetLocalDescription(answer); err != nil {
 		return fmt.Errorf("could not set local description: %w", err)
 	}

From c87277580856d93deb46bf8adb07469919eefe7c Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 20:48:51 +0800
Subject: [PATCH 11/52] Concurrency safe map

---
 internal/broadcast/broadcast.go             | 14 +++++++-------
 internal/broadcast/publisher/publisher.go   | 11 ++++++-----
 internal/broadcast/session/session.go       |  3 ---
 internal/broadcast/subscriber/subscriber.go |  9 +++++----
 4 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index 45a42fd..ffd8a65 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	mqttclient "github.com/SB-IM/mqtt-client"
@@ -13,15 +14,15 @@ import (
 
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
 	"github.com/SB-IM/skywalker/internal/broadcast/publisher"
-	"github.com/SB-IM/skywalker/internal/broadcast/session"
 	"github.com/SB-IM/skywalker/internal/broadcast/subscriber"
 )
 
 // Service consists of many sessions.
 type Service struct {
-	client mqtt.Client
-	logger zerolog.Logger
-	config cfg.ConfigOptions
+	client   mqtt.Client
+	logger   zerolog.Logger
+	config   cfg.ConfigOptions
+	sessions sync.Map
 }
 
 func New(ctx context.Context, config *cfg.ConfigOptions) *Service {
@@ -33,14 +34,13 @@ func New(ctx context.Context, config *cfg.ConfigOptions) *Service {
 }
 
 func (s *Service) Broadcast() error {
-	sessions := make(session.Sessions)
-	pub := publisher.New(s.client, sessions, &s.logger, cfg.PublisherConfigOptions{
+	pub := publisher.New(s.client, &s.sessions, &s.logger, cfg.PublisherConfigOptions{
 		TopicConfigOptions:  s.config.TopicConfigOptions,
 		WebRTCConfigOptions: s.config.WebRTCConfigOptions,
 	})
 	pub.Signal()
 
-	sub := subscriber.New(sessions, &s.logger, s.config.WebRTCConfigOptions)
+	sub := subscriber.New(&s.sessions, &s.logger, s.config.WebRTCConfigOptions)
 	handler := sub.Signal()
 
 	server := s.newServer(handler)
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index ec11939..5d50a2a 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -3,6 +3,7 @@ package publisher
 import (
 	"fmt"
 	"strconv"
+	"sync"
 
 	pb "github.com/SB-IM/pb/signal"
 	mqtt "github.com/eclipse/paho.mqtt.golang"
@@ -23,13 +24,13 @@ type Publisher struct {
 
 	// sessions must be created before used by publisher and is shared between publishers and subscribers.
 	// It's mainly written and maintained by publishers
-	sessions session.Sessions
+	sessions *sync.Map
 }
 
 // New returns a new Publisher.
 func New(
 	client mqtt.Client,
-	sessions session.Sessions,
+	sessions *sync.Map,
 	logger *zerolog.Logger,
 	config cfg.PublisherConfigOptions,
 ) *Publisher {
@@ -133,11 +134,11 @@ func (p *Publisher) registerSession(
 	trackSource pb.TrackSource,
 	videoTrack *webrtc.TrackLocalStaticRTP,
 ) {
-	if s, ok := p.sessions[id]; ok {
-		s[trackSource] = videoTrack
+	if s, ok := p.sessions.Load(id); ok {
+		s.(session.Session)[trackSource] = videoTrack
 	} else {
 		s := make(session.Session)
 		s[trackSource] = videoTrack
-		p.sessions[id] = s
+		p.sessions.Store(id, s)
 	}
 }
diff --git a/internal/broadcast/session/session.go b/internal/broadcast/session/session.go
index 5364a97..e76582e 100644
--- a/internal/broadcast/session/session.go
+++ b/internal/broadcast/session/session.go
@@ -10,6 +10,3 @@ type MachineID string
 
 // Session maps track source to local video track.
 type Session map[pb.TrackSource]*webrtc.TrackLocalStaticRTP
-
-// Sessions maps machine id to session.
-type Sessions map[MachineID]Session
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index cdea616..15370bc 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"os"
+	"sync"
 
 	pb "github.com/SB-IM/pb/signal"
 	"github.com/gorilla/mux"
@@ -21,11 +22,11 @@ type Subscriber struct {
 
 	// sessions must be created before used by publisher and is shared between publishers ans subscribers.
 	// It's only read by subscriber.
-	sessions session.Sessions
+	sessions *sync.Map
 }
 
 // New returns a new Subscriber.
-func New(sessions session.Sessions, logger *zerolog.Logger, config cfg.WebRTCConfigOptions) *Subscriber {
+func New(sessions *sync.Map, logger *zerolog.Logger, config cfg.WebRTCConfigOptions) *Subscriber {
 	l := logger.With().Str("component", "Subscriber").Logger()
 	return &Subscriber{
 		sessions: sessions,
@@ -58,13 +59,13 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 		logger := s.logger.With().Str("id", offer.Id).Int32("track_source", int32(offer.TrackSource)).Logger()
 		logger.Debug().Str("offer", offer.String()).Msg("received offer from subscriber")
 
-		ses, ok := s.sessions[session.MachineID(offer.Id)]
+		ses, ok := s.sessions.Load(session.MachineID(offer.Id))
 		if !ok {
 			logger.Warn().Msg("no machine in found in existing sessions")
 			http.Error(w, "wrong id", http.StatusBadRequest)
 			return
 		}
-		videoTrack, ok := ses[offer.TrackSource]
+		videoTrack, ok := ses.(session.Session)[offer.TrackSource]
 		if !ok {
 			logger.Warn().Msg("no track source found in existing sessions")
 			http.Error(w, "wrong track_source", http.StatusBadRequest)

From 24190b20337063d610f383d5f9694adb0b419227 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sat, 3 Apr 2021 23:24:44 +0800
Subject: [PATCH 12/52] Add UUID for track

---
 go.mod                              | 1 +
 internal/broadcast/webrtc/webrtc.go | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 8733cbd..5206035 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/SB-IM/mqtt-client v0.1.0
 	github.com/SB-IM/pb v0.1.3
 	github.com/eclipse/paho.mqtt.golang v1.3.3
+	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/pion/rtcp v1.2.6
 	github.com/pion/webrtc/v3 v3.0.20
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index edbea5e..a0fc73f 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	pb "github.com/SB-IM/pb/signal"
+	"github.com/google/uuid"
 	"github.com/pion/rtcp"
 	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
@@ -39,7 +40,8 @@ func New(videoTrack *webrtc.TrackLocalStaticRTP, config cfg.WebRTCConfigOptions,
 
 // CreateLocalTrack creates a TrackLocalStaticRTP and is only used by publisher.
 func CreateLocalTrack() (*webrtc.TrackLocalStaticRTP, error) {
-	return webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
+	id := uuid.New().String()
+	return webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video-"+id, "pion-"+id)
 }
 
 // CreatePublisher creates a webRTC publisher peer.

From 970b578ddb60dd521d929e7b0a46adc314850cbe Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sun, 4 Apr 2021 00:41:30 +0800
Subject: [PATCH 13/52] Simplify session id

---
 internal/broadcast/publisher/publisher.go   | 14 ++++---------
 internal/broadcast/session/session.go       | 12 -----------
 internal/broadcast/subscriber/subscriber.go | 22 +++++++++------------
 3 files changed, 13 insertions(+), 35 deletions(-)
 delete mode 100644 internal/broadcast/session/session.go

diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index 5d50a2a..cfe79d7 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -12,7 +12,6 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
-	"github.com/SB-IM/skywalker/internal/broadcast/session"
 	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
 )
 
@@ -101,7 +100,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 		logger.Debug().Str("answer_topic", answerTopic).Str("answer", answer.String()).Msg("sent answer to edge")
 
 		// Register session on signaling success.
-		p.registerSession(session.MachineID(offer.Id), offer.TrackSource, videoTrack)
+		p.registerSession(offer.Id, offer.TrackSource, videoTrack)
 	}
 }
 
@@ -130,15 +129,10 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 }
 
 func (p *Publisher) registerSession(
-	id session.MachineID,
+	id string,
 	trackSource pb.TrackSource,
 	videoTrack *webrtc.TrackLocalStaticRTP,
 ) {
-	if s, ok := p.sessions.Load(id); ok {
-		s.(session.Session)[trackSource] = videoTrack
-	} else {
-		s := make(session.Session)
-		s[trackSource] = videoTrack
-		p.sessions.Store(id, s)
-	}
+	sessionID := id + strconv.Itoa(int(trackSource))
+	p.sessions.Store(sessionID, videoTrack)
 }
diff --git a/internal/broadcast/session/session.go b/internal/broadcast/session/session.go
deleted file mode 100644
index e76582e..0000000
--- a/internal/broadcast/session/session.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package session
-
-import (
-	pb "github.com/SB-IM/pb/signal"
-	"github.com/pion/webrtc/v3"
-)
-
-// MachineID is unique id for edge device.
-type MachineID string
-
-// Session maps track source to local video track.
-type Session map[pb.TrackSource]*webrtc.TrackLocalStaticRTP
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 15370bc..0df9339 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -4,14 +4,15 @@ import (
 	"encoding/json"
 	"net/http"
 	"os"
+	"strconv"
 	"sync"
 
 	pb "github.com/SB-IM/pb/signal"
 	"github.com/gorilla/mux"
+	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
 
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
-	"github.com/SB-IM/skywalker/internal/broadcast/session"
 	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
 )
 
@@ -57,22 +58,17 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 			return
 		}
 		logger := s.logger.With().Str("id", offer.Id).Int32("track_source", int32(offer.TrackSource)).Logger()
-		logger.Debug().Str("offer", offer.String()).Msg("received offer from subscriber")
+		logger.Debug().Msg("received offer from subscriber")
 
-		ses, ok := s.sessions.Load(session.MachineID(offer.Id))
+		sessionID := offer.Id + strconv.Itoa(int(offer.TrackSource))
+		value, ok := s.sessions.Load(sessionID)
 		if !ok {
-			logger.Warn().Msg("no machine in found in existing sessions")
-			http.Error(w, "wrong id", http.StatusBadRequest)
-			return
-		}
-		videoTrack, ok := ses.(session.Session)[offer.TrackSource]
-		if !ok {
-			logger.Warn().Msg("no track source found in existing sessions")
-			http.Error(w, "wrong track_source", http.StatusBadRequest)
+			logger.Warn().Msg("no machine id or track source found in existing sessions")
+			http.Error(w, "wrong id or track source", http.StatusBadRequest)
 			return
 		}
 
-		wcx := webrtcx.New(videoTrack, s.config, &logger)
+		wcx := webrtcx.New(value.(*webrtc.TrackLocalStaticRTP), s.config, &logger)
 		// TODO: handle blocking case with timeout for channels.
 		wcx.OfferChan <- &offer
 		if err := wcx.CreateSubscriber(); err != nil {
@@ -89,6 +85,6 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
-		logger.Debug().Str("answer", answer.Sdp.String()).Msg("sent answer to subscriber")
+		logger.Debug().Msg("sent answer to subscriber")
 	}
 }

From 145a82c3853dd7d7559836001d77dedad550535d Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Sun, 4 Apr 2021 00:47:42 +0800
Subject: [PATCH 14/52] Minor change

---
 internal/broadcast/publisher/publisher.go   |  4 ++--
 internal/broadcast/subscriber/subscriber.go |  4 ++--
 internal/broadcast/webrtc/webrtc.go         | 12 +++++-------
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index cfe79d7..f7ce8f7 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -116,11 +116,11 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 	}
 	logger.Debug().Msg("created video track")
 
-	w := webrtcx.New(videoTrack, p.config.WebRTCConfigOptions, logger)
+	w := webrtcx.New(p.config.WebRTCConfigOptions, logger)
 
 	// TODO: handle blocking case with timeout for channels.
 	w.OfferChan <- offer
-	if err := w.CreatePublisher(); err != nil {
+	if err := w.CreatePublisher(videoTrack); err != nil {
 		return nil, nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
 	}
 	logger.Debug().Msg("created publisher")
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 0df9339..58653e6 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -68,10 +68,10 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 			return
 		}
 
-		wcx := webrtcx.New(value.(*webrtc.TrackLocalStaticRTP), s.config, &logger)
+		wcx := webrtcx.New(s.config, &logger)
 		// TODO: handle blocking case with timeout for channels.
 		wcx.OfferChan <- &offer
-		if err := wcx.CreateSubscriber(); err != nil {
+		if err := wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
 			logger.Err(err).Msg("failed to create subscriber")
 			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
 			return
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index a0fc73f..5a8b7c0 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -22,17 +22,15 @@ const (
 type WebRTC struct {
 	logger     zerolog.Logger
 	config     cfg.WebRTCConfigOptions
-	videoTrack *webrtc.TrackLocalStaticRTP
 	OfferChan  chan *pb.SessionDescription
 	AnswerChan chan *pb.SessionDescription
 }
 
 // New returns a new WebRTC.
-func New(videoTrack *webrtc.TrackLocalStaticRTP, config cfg.WebRTCConfigOptions, logger *zerolog.Logger) *WebRTC {
+func New(config cfg.WebRTCConfigOptions, logger *zerolog.Logger) *WebRTC {
 	return &WebRTC{
 		logger:     *logger,
 		config:     config,
-		videoTrack: videoTrack,
 		OfferChan:  make(chan *pb.SessionDescription, 1), // Make 1 buffer so offer sending never blocks
 		AnswerChan: make(chan *pb.SessionDescription, 1), // Make 1 buffer so answer sending never blocks
 	}
@@ -46,7 +44,7 @@ func CreateLocalTrack() (*webrtc.TrackLocalStaticRTP, error) {
 
 // CreatePublisher creates a webRTC publisher peer.
 // Caller must send offer first by OfferChan or this function blocks waiting for receiving offer forever.
-func (w *WebRTC) CreatePublisher() error {
+func (w *WebRTC) CreatePublisher(videoTrack *webrtc.TrackLocalStaticRTP) error {
 	peerConnection, err := w.newPeerConnection()
 	if err != nil {
 		return fmt.Errorf("could not create PeerConnection: %w", err)
@@ -68,7 +66,7 @@ func (w *WebRTC) CreatePublisher() error {
 				w.logger.Panic().Err(err).Msg("could not read buffer")
 			}
 			// ErrClosedPipe means we don't have any subscribers, this is ok if no peers have connected yet
-			if _, err = w.videoTrack.Write(rtpBuf[:i]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
+			if _, err = videoTrack.Write(rtpBuf[:i]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
 				w.logger.Panic().Err(err).Msg("could not write video track")
 			}
 		}
@@ -84,13 +82,13 @@ func (w *WebRTC) CreatePublisher() error {
 
 // CreateSubscriber creates a webRTC subscriber peer.
 // Caller must send offer first by OfferChan or this function blocks waiting for receiving offer forever.
-func (w *WebRTC) CreateSubscriber() error {
+func (w *WebRTC) CreateSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error {
 	peerConnection, err := w.newPeerConnection()
 	if err != nil {
 		return fmt.Errorf("could not create PeerConnection: %w", err)
 	}
 
-	rtpSender, err := peerConnection.AddTrack(w.videoTrack)
+	rtpSender, err := peerConnection.AddTrack(videoTrack)
 	if err != nil {
 		return fmt.Errorf("could not add track: %w", err)
 	}

From 94e1b31cfad4a1d76e8a146ce690aad764a103fa Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 5 Apr 2021 10:40:11 +0800
Subject: [PATCH 15/52] Update configs

---
 .gitignore                                  |  1 +
 cmd/broadcast/cmd.go                        | 59 ++++++++++++++++-----
 config/config.example.yaml                  | 10 ++--
 docker/docker-compose.dev.yaml              |  5 +-
 internal/broadcast/broadcast.go             |  6 +--
 internal/broadcast/cfg/cfg.go               | 17 +++---
 internal/broadcast/publisher/publisher.go   | 15 +++---
 internal/broadcast/subscriber/subscriber.go |  3 +-
 internal/broadcast/webrtc/webrtc.go         |  4 +-
 9 files changed, 84 insertions(+), 36 deletions(-)

diff --git a/.gitignore b/.gitignore
index 02ec59b..d98f1fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 .idea
 config/config.dev.yaml
 .env
+config/config.docker.yaml
diff --git a/cmd/broadcast/cmd.go b/cmd/broadcast/cmd.go
index 93b8f3b..d06893a 100644
--- a/cmd/broadcast/cmd.go
+++ b/cmd/broadcast/cmd.go
@@ -26,17 +26,17 @@ func Command() *cli.Command {
 
 		mc mqtt.Client
 
-		mqttConfigOptions   mqttclient.ConfigOptions
-		topicConfigOptions  cfg.TopicConfigOptions
-		webRTCConfigOptions cfg.WebRTCConfigOptions
-		serverConfigOptions cfg.ServerConfigOptions
+		mqttConfigOptions       mqttclient.ConfigOptions
+		mqttClientConfigOptions cfg.MQTTClientConfigOptions
+		webRTCConfigOptions     cfg.WebRTCConfigOptions
+		serverConfigOptions     cfg.ServerConfigOptions
 	)
 
 	flags := func() (flags []cli.Flag) {
 		for _, v := range [][]cli.Flag{
 			loadConfigFlag(),
 			mqttFlags(&mqttConfigOptions),
-			topicFlags(&topicConfigOptions),
+			mqttClientFlags(&mqttClientConfigOptions),
 			webRTCFlags(&webRTCConfigOptions),
 			serverFlags(&serverConfigOptions),
 		} {
@@ -73,9 +73,9 @@ func Command() *cli.Command {
 		},
 		Action: func(c *cli.Context) error {
 			svc := broadcast.New(ctx, &cfg.ConfigOptions{
-				WebRTCConfigOptions: webRTCConfigOptions,
-				TopicConfigOptions:  topicConfigOptions,
-				ServerConfigOptions: serverConfigOptions,
+				WebRTCConfigOptions:     webRTCConfigOptions,
+				MQTTClientConfigOptions: mqttClientConfigOptions,
+				ServerConfigOptions:     serverConfigOptions,
 			})
 			err := svc.Broadcast()
 			if err != nil {
@@ -136,7 +136,7 @@ func mqttFlags(mqttConfigOptions *mqttclient.ConfigOptions) []cli.Flag {
 	}
 }
 
-func topicFlags(topicConfigOptions *cfg.TopicConfigOptions) []cli.Flag {
+func mqttClientFlags(topicConfigOptions *cfg.MQTTClientConfigOptions) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "topic-offer",
@@ -146,11 +146,25 @@ func topicFlags(topicConfigOptions *cfg.TopicConfigOptions) []cli.Flag {
 			Destination: &topicConfigOptions.OfferTopic,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "topic-answer",
-			Usage:       "MQTT topic for WebRTC SDP answer signaling",
+			Name:        "topic-answer-prefix",
+			Usage:       "MQTT topic prefix for WebRTC SDP answer signaling",
 			Value:       "/edge/livestream/signal/answer",
 			DefaultText: "/edge/livestream/signal/answer",
-			Destination: &topicConfigOptions.AnswerTopic,
+			Destination: &topicConfigOptions.AnswerTopicPrefix,
+		}),
+		altsrc.NewUintFlag(&cli.UintFlag{
+			Name:        "qos",
+			Usage:       "MQTT client qos for WebRTC SDP signaling",
+			Value:       0,
+			DefaultText: "0",
+			Destination: &topicConfigOptions.Qos,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:        "retained",
+			Usage:       "MQTT client setting retainsion for WebRTC SDP signaling",
+			Value:       false,
+			DefaultText: "false",
+			Destination: &topicConfigOptions.Retained,
 		}),
 	}
 }
@@ -164,6 +178,27 @@ func webRTCFlags(webRTCConfigOptions *cfg.WebRTCConfigOptions) []cli.Flag {
 			DefaultText: "stun:stun.l.google.com:19302",
 			Destination: &webRTCConfigOptions.ICEServer,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "ice-server-username",
+			Usage:       "ICE server username",
+			Value:       "",
+			DefaultText: "",
+			Destination: &webRTCConfigOptions.Username,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "ice-server-credential",
+			Usage:       "ICE server credential",
+			Value:       "",
+			DefaultText: "",
+			Destination: &webRTCConfigOptions.Credential,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:        "enable-frontend",
+			Usage:       "Enable webRTC frontend server",
+			Value:       false,
+			DefaultText: "false",
+			Destination: &webRTCConfigOptions.EnableFrontend,
+		}),
 	}
 }
 
diff --git a/config/config.example.yaml b/config/config.example.yaml
index a2cfd14..80d6918 100644
--- a/config/config.example.yaml
+++ b/config/config.example.yaml
@@ -4,14 +4,18 @@ mqtt-clientID: mqtt_edge
 mqtt-username: user
 mqtt-password: password
 
-# MQTT topics  for webRTC signaling config
+# MQTT client config for webRTC signaling
 topic-offer: /edge/livestream/signal/offer
-topic-answer: /edge/livestream/signal/answer
+topic-answer-prefix: /edge/livestream/signal/answer
+qos: 0
+retained: false
 
 # WebRTC config
 ice-server: stun:stun.l.google.com:19302
+ice-server-username: ""
+ice-server-credential: ""
+enable-frontend: false
 
 # WebSocket config
 ws-host: 0.0.0.0
 ws-port: 8080
-ws-path: /ws/webrtc
diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 00fb512..f44a779 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -11,7 +11,7 @@ services:
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
     volumes:
-      - ../../sphinx/config/config.dev.yaml:/config/config.yaml
+      - ../../sphinx/config/config.docker.yaml:/config/config.yaml
       - /etc/machine-id:/etc/machine-id
     depends_on:
       - mosquitto
@@ -22,6 +22,7 @@ services:
     image: ghcr.io/sb-im/skywalker:broadcast-latest
     container_name: broadcast
     command:
+      - --debug
       - broadcast
       - -c
       - /config/config.yaml
@@ -31,7 +32,7 @@ services:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
       - DEBUG=true
     volumes:
-      - ../config/config.dev.yaml:/config/config.yaml
+      - ../config/config.docker.yaml:/config/config.yaml
     depends_on:
       - mosquitto
     restart: on-failure
diff --git a/internal/broadcast/broadcast.go b/internal/broadcast/broadcast.go
index ffd8a65..875053f 100644
--- a/internal/broadcast/broadcast.go
+++ b/internal/broadcast/broadcast.go
@@ -34,9 +34,9 @@ func New(ctx context.Context, config *cfg.ConfigOptions) *Service {
 }
 
 func (s *Service) Broadcast() error {
-	pub := publisher.New(s.client, &s.sessions, &s.logger, cfg.PublisherConfigOptions{
-		TopicConfigOptions:  s.config.TopicConfigOptions,
-		WebRTCConfigOptions: s.config.WebRTCConfigOptions,
+	pub := publisher.New(s.client, &s.sessions, &s.logger, &cfg.PublisherConfigOptions{
+		MQTTClientConfigOptions: s.config.MQTTClientConfigOptions,
+		WebRTCConfigOptions:     s.config.WebRTCConfigOptions,
 	})
 	pub.Signal()
 
diff --git a/internal/broadcast/cfg/cfg.go b/internal/broadcast/cfg/cfg.go
index 83e1c43..c8a26b7 100644
--- a/internal/broadcast/cfg/cfg.go
+++ b/internal/broadcast/cfg/cfg.go
@@ -2,22 +2,27 @@ package cfg
 
 type ConfigOptions struct {
 	WebRTCConfigOptions
-	TopicConfigOptions
+	MQTTClientConfigOptions
 	ServerConfigOptions
 }
 
 type PublisherConfigOptions struct {
-	TopicConfigOptions
+	MQTTClientConfigOptions
 	WebRTCConfigOptions
 }
 
 type WebRTCConfigOptions struct {
-	ICEServer string
+	ICEServer      string
+	Username       string
+	Credential     string
+	EnableFrontend bool // Enable static file server handler serving webRTC frontend, useful for debug
 }
 
-type TopicConfigOptions struct {
-	OfferTopic  string
-	AnswerTopic string
+type MQTTClientConfigOptions struct {
+	OfferTopic        string
+	AnswerTopicPrefix string
+	Qos               uint
+	Retained          bool
 }
 
 type ServerConfigOptions struct {
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index f7ce8f7..5c410db 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -19,7 +19,7 @@ import (
 type Publisher struct {
 	client mqtt.Client
 	logger zerolog.Logger
-	config cfg.PublisherConfigOptions
+	config *cfg.PublisherConfigOptions
 
 	// sessions must be created before used by publisher and is shared between publishers and subscribers.
 	// It's mainly written and maintained by publishers
@@ -31,7 +31,7 @@ func New(
 	client mqtt.Client,
 	sessions *sync.Map,
 	logger *zerolog.Logger,
-	config cfg.PublisherConfigOptions,
+	config *cfg.PublisherConfigOptions,
 ) *Publisher {
 	l := logger.With().Str("component", "Publisher").Logger()
 	return &Publisher{
@@ -47,7 +47,7 @@ func (p *Publisher) Signal() {
 	// The receiving topic is the same for each edge device, but message payload is different.
 	// The id and trackSource in payload determine the following publishing topic.
 	// Receive remote SDP with MQTT.
-	t := p.client.Subscribe(p.config.OfferTopic, 1, p.handleMessage())
+	t := p.client.Subscribe(p.config.OfferTopic, byte(p.config.Qos), p.handleMessage())
 	// the connection handler is called in a goroutine so blocking here would hot cause an issue. However as blocking
 	// in other handlers does cause problems its best to just assume we should not block
 	go func() {
@@ -74,7 +74,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 			Str("id", offer.Id).
 			Int32("track_source", int32(offer.TrackSource)).
 			Logger()
-		logger.Debug().Str("offer", offer.String()).Msg("received offer from edge")
+		logger.Debug().Msg("received offer from edge")
 
 		answer, videoTrack, err := p.signalPeerConnection(&offer, &logger)
 		if err != nil {
@@ -90,14 +90,14 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 		}
 
 		// The publishing topic is unique to each edge device and is determined by above receiving message payload.
-		answerTopic := p.config.AnswerTopic + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
-		t := c.Publish(answerTopic, 1, true, payload)
+		answerTopic := p.config.AnswerTopicPrefix + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
+		t := c.Publish(answerTopic, byte(p.config.Qos), p.config.Retained, payload)
 		<-t.Done()
 		if t.Error() != nil {
 			p.logger.Err(t.Error()).Msgf("could not publish to %s", answerTopic)
 			return
 		}
-		logger.Debug().Str("answer_topic", answerTopic).Str("answer", answer.String()).Msg("sent answer to edge")
+		logger.Debug().Str("answer_topic", answerTopic).Msg("sent answer to edge")
 
 		// Register session on signaling success.
 		p.registerSession(offer.Id, offer.TrackSource, videoTrack)
@@ -135,4 +135,5 @@ func (p *Publisher) registerSession(
 ) {
 	sessionID := id + strconv.Itoa(int(trackSource))
 	p.sessions.Store(sessionID, videoTrack)
+	p.logger.Debug().Str("key", sessionID).Int32("value", int32(trackSource)).Msg("registered session")
 }
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 58653e6..3a76218 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -3,7 +3,6 @@ package subscriber
 import (
 	"encoding/json"
 	"net/http"
-	"os"
 	"strconv"
 	"sync"
 
@@ -41,7 +40,7 @@ func (s *Subscriber) Signal() http.Handler {
 	r := mux.NewRouter()
 	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).Methods(http.MethodPost)
 
-	if os.Getenv("DEBUG") == "true" {
+	if s.config.EnableFrontend {
 		r.Handle("/", http.FileServer(http.Dir("e2e/broadcast/static")))
 	}
 	s.logger.Debug().Msg("registered signal HTTP handler")
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 5a8b7c0..8f93c8c 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -150,7 +150,9 @@ func (w *WebRTC) newPeerConnection() (*webrtc.PeerConnection, error) {
 	return webrtc.NewPeerConnection(webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
 			{
-				URLs: []string{w.config.ICEServer},
+				URLs:       []string{w.config.ICEServer},
+				Username:   w.config.Username,
+				Credential: w.config.Credential,
 			},
 		},
 	})

From fe06a9c59d2982f22402872db966d2383a7288f3 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 5 Apr 2021 15:54:35 +0800
Subject: [PATCH 16/52] Add README.md

---
 README.md                       | 30 ++++++++++++++++++++++++++++++
 e2e/broadcast/static/index.html |  8 +++++++-
 2 files changed, 37 insertions(+), 1 deletion(-)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..775abf8
--- /dev/null
+++ b/README.md
@@ -0,0 +1,30 @@
+# Skywalker
+
+Skywalker walks above the cloud.
+
+## What is Skywalker project?
+
+Skywalker is cloud service managing and controlling all edge devices, processing IOT data, etc. It also manages IOT users and provides friendly APIs to front-end.
+
+Currently, Skywalker includes:
+
+- `Broadcast`: forwards video streams from edge devices.
+
+## How to run?
+
+All sub-processes are executed through sub commands under `skywalker` command.
+
+Make sure you have the following tools installed:
+
+- `Go`
+- `GNU make`
+- `golangci-lint` (Optional)
+- `Docker`
+- `Docker-compose` (Optional)
+- `mosquitto` (Optional, can be on Docker)
+
+### Run `broadcast`
+
+```bash
+$ make
+```
diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index e3bac6b..58ae63e 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -15,7 +15,13 @@
 
 <script type="text/javascript">
     for (let i = 0; i < 2; i++) {
-        let pc = new RTCPeerConnection()
+        let pc = new RTCPeerConnection({
+            iceServers: [
+                {
+                    urls: "stun:stun.l.google.com:19302",
+                }
+            ]
+        })
         pc.ontrack = function (event) {
             var el = document.createElement(event.track.kind)
             el.srcObject = event.streams[0]

From b4fd93b0602a51195d257d803f36687030f69e8b Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 5 Apr 2021 18:09:15 +0800
Subject: [PATCH 17/52] Update action (#5)

Update action
---
 .../{build.yaml => build-broadcast.yaml}      | 35 ++++++++++++-------
 1 file changed, 23 insertions(+), 12 deletions(-)
 rename .github/workflows/{build.yaml => build-broadcast.yaml} (50%)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build-broadcast.yaml
similarity index 50%
rename from .github/workflows/build.yaml
rename to .github/workflows/build-broadcast.yaml
index 799d927..64898bb 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build-broadcast.yaml
@@ -1,4 +1,4 @@
-name: Build container image
+name: Build broadcast image
 on:
   workflow_dispatch:
   push:
@@ -8,18 +8,26 @@ on:
       - "v*"
 jobs:
   build:
-    name: Build container image
+    name: Build broadcast image
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Docker meta
-        id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        id: meta
+        uses: crazy-max/ghaction-docker-meta@v2
         with:
-          images: ${{ secrets.DOCKERHUB_USERNAME }}/orchid
-          tag-sha: true
-          tag-custom: latest
+          images: ghcr.io/sb-im/skywalker
+          tags: |
+            type=edge,prefix=broadcast-
+            type=ref,prefix=broadcast-,event=branch
+            type=ref,prefix=broadcast-,event=pr
+            type=semver,prefix=broadcast-,pattern={{version}}
+            type=semver,prefix=broadcast-,pattern={{major}}.{{minor}}
+            type=sha,prefix=broadcast-
+          flavor: |
+            latest=true
+            prefix=broadcast-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -35,14 +43,17 @@ jobs:
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
-          username: ${{ secrets.GITHUB_USERNAME }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.CONTAINER_USER }}
+          password: ${{ secrets.CONTAINER_TOKEN }}
+      - name: Prepare ssh key
+        run: echo "${{ secrets.SSH_KEY }}" > ssh-key
       - name: Build and push
         uses: docker/build-push-action@v2
         with:
-          platforms: linux/amd64,linux/arm64
+          ssh: github=ssh-key
+          file: docker/Dockerfile.broadcast
           push: true
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

From 27d5c5b5d0e21c51bcf603263139f7a71d78fdab Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 6 Apr 2021 19:58:59 +0800
Subject: [PATCH 18/52] Add ICECandidate trickle support(#6)

---
 Makefile                                      |  2 +-
 cmd/broadcast/cmd.go                          | 20 +++-
 config/config.example.yaml                    | 12 ++-
 go.mod                                        |  2 +-
 go.sum                                        |  2 +
 internal/broadcast/cfg/cfg.go                 | 10 +-
 internal/broadcast/publisher/publisher.go     | 97 ++++++++++++++++++-
 internal/broadcast/subscriber/subscriber.go   |  3 +-
 .../broadcast/subscriber/subscriber_test.go   |  4 +-
 internal/broadcast/webrtc/webrtc.go           | 85 +++++++++++++---
 10 files changed, 205 insertions(+), 32 deletions(-)

diff --git a/Makefile b/Makefile
index de04839..3a27e8f 100644
--- a/Makefile
+++ b/Makefile
@@ -29,7 +29,7 @@ up: down image
 
 .PHONY: down
 down:
-	@docker-compose -f docker/docker-compose.dev.yaml down
+	@docker-compose -f docker/docker-compose.dev.yaml down --remove-orphans
 
 .PHONY: broker
 broker:
diff --git a/cmd/broadcast/cmd.go b/cmd/broadcast/cmd.go
index d06893a..d890018 100644
--- a/cmd/broadcast/cmd.go
+++ b/cmd/broadcast/cmd.go
@@ -146,11 +146,25 @@ func mqttClientFlags(topicConfigOptions *cfg.MQTTClientConfigOptions) []cli.Flag
 			Destination: &topicConfigOptions.OfferTopic,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:        "topic-answer-prefix",
-			Usage:       "MQTT topic prefix for WebRTC SDP answer signaling",
+			Name:        "topic-answer-suffix",
+			Usage:       "MQTT topic suffix for WebRTC SDP answer signaling",
 			Value:       "/edge/livestream/signal/answer",
 			DefaultText: "/edge/livestream/signal/answer",
-			Destination: &topicConfigOptions.AnswerTopicPrefix,
+			Destination: &topicConfigOptions.AnswerTopicSuffix,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "topic-candidate-send-suffix",
+			Usage:       "MQTT topic suffix for WebRTC candidate sending, and the sending topic of edge is /edge/livestream/signal/candidate/send",
+			Value:       "/edge/livestream/signal/candidate/recv",
+			DefaultText: "/edge/livestream/signal/candidate/recv",
+			Destination: &topicConfigOptions.CandidateSendTopicSuffix,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:        "topic-candidate-recv-suffix",
+			Usage:       "MQTT topic suffix for WebRTC candidate receiving, and the receiving topic of edge is /edge/livestream/signal/candidate/recv",
+			Value:       "/edge/livestream/signal/candidate/send",
+			DefaultText: "/edge/livestream/signal/candidate/send",
+			Destination: &topicConfigOptions.CandidateRecvTopicSuffix,
 		}),
 		altsrc.NewUintFlag(&cli.UintFlag{
 			Name:        "qos",
diff --git a/config/config.example.yaml b/config/config.example.yaml
index 80d6918..64cdeed 100644
--- a/config/config.example.yaml
+++ b/config/config.example.yaml
@@ -6,7 +6,11 @@ mqtt-password: password
 
 # MQTT client config for webRTC signaling
 topic-offer: /edge/livestream/signal/offer
-topic-answer-prefix: /edge/livestream/signal/answer
+topic-answer-suffix: /edge/livestream/signal/answer
+# topic-candidate-send is opposite to edge's topic-candidate-recv-suffix topic
+topic-candidate-send-suffix: /edge/livestream/signal/candidate/recv
+# topic-candidate-recv-suffix is opposite to edge's topic-candidate-send-suffix topic
+topic-candidate-recv-suffix: /edge/livestream/signal/candidate/send
 qos: 0
 retained: false
 
@@ -16,6 +20,6 @@ ice-server-username: ""
 ice-server-credential: ""
 enable-frontend: false
 
-# WebSocket config
-ws-host: 0.0.0.0
-ws-port: 8080
+# Signal server config
+host: 0.0.0.0
+port: 8080
diff --git a/go.mod b/go.mod
index 5206035..b98d322 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.16
 require (
 	github.com/SB-IM/logging v0.2.2
 	github.com/SB-IM/mqtt-client v0.1.0
-	github.com/SB-IM/pb v0.1.3
+	github.com/SB-IM/pb v0.1.4
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
diff --git a/go.sum b/go.sum
index 07c810f..16ac8e3 100644
--- a/go.sum
+++ b/go.sum
@@ -7,6 +7,8 @@ github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXC
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
 github.com/SB-IM/pb v0.1.3 h1:Sq1wCcrcX5smwLvEtfw1cN04fHM2T4wiN5FggThLEU8=
 github.com/SB-IM/pb v0.1.3/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
+github.com/SB-IM/pb v0.1.4 h1:eO/S1xQC9Uh6RV7RDcBEPZthZ6uOjGhnlmqLXTKb/y0=
+github.com/SB-IM/pb v0.1.4/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
diff --git a/internal/broadcast/cfg/cfg.go b/internal/broadcast/cfg/cfg.go
index c8a26b7..d95382d 100644
--- a/internal/broadcast/cfg/cfg.go
+++ b/internal/broadcast/cfg/cfg.go
@@ -19,10 +19,12 @@ type WebRTCConfigOptions struct {
 }
 
 type MQTTClientConfigOptions struct {
-	OfferTopic        string
-	AnswerTopicPrefix string
-	Qos               uint
-	Retained          bool
+	OfferTopic               string
+	AnswerTopicSuffix        string
+	CandidateSendTopicSuffix string // Opposite to edge's CandidateRecvTopicSuffix topic
+	CandidateRecvTopicSuffix string // Opposite to edge's CandidateSendTopicSuffix topic.
+	Qos                      uint
+	Retained                 bool
 }
 
 type ServerConfigOptions struct {
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index 5c410db..bc2287c 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -60,6 +60,59 @@ func (p *Publisher) Signal() {
 	}()
 }
 
+// sendCandidate sends candidate to remote webRTC peer via MQTT.
+// The publish topic is unique to this edge device.
+func (p *Publisher) sendCandidate(id string, trackSource pb.TrackSource) webrtcx.SendCandidateFunc {
+	return func(candidate *webrtc.ICECandidate) error {
+		payload, err := encodeCandidate(candidate)
+		if err != nil {
+			return fmt.Errorf("could not encode candidate: %w", err)
+		}
+		topic := p.config.CandidateSendTopicSuffix + "/" + id + "/" + strconv.Itoa(int(trackSource))
+		t := p.client.Publish(topic, byte(p.config.Qos), p.config.Retained, payload)
+		// Handle the token in a go routine so this loop keeps sending messages regardless of delivery status
+		go func() {
+			<-t.Done()
+			if t.Error() != nil {
+				p.logger.Err(t.Error()).Msgf("could not publish to %s", p.config.CandidateSendTopicSuffix)
+			}
+		}()
+		return nil
+	}
+}
+
+// recvCandidate is not a one time subscriber.
+// The caller must check if result in channel is nil.
+// sendCandidate receive candidate from remote webRTC peer via MQTT.
+// The subscription topic is unique to this edge device.
+func (p *Publisher) recvCandidate(id string, trackSource pb.TrackSource) webrtcx.RecvCandidateFunc {
+	return func() <-chan *webrtc.ICECandidate {
+		// TODO: Figure how to properly close channel.
+		ch := make(chan *webrtc.ICECandidate, 2) // Make buffer 2 because we have at least 2 sendings.
+		topic := p.config.CandidateRecvTopicSuffix + "/" + id + "/" + strconv.Itoa(int(trackSource))
+		// Receive remote ICE candidate with MQTT.
+		t := p.client.Subscribe(topic, byte(p.config.Qos), func(c mqtt.Client, m mqtt.Message) {
+			candidate, err := decodeCandidate(m.Payload())
+			if err != nil {
+				p.logger.Err(err).Msg("could not decode candidate")
+				return
+			}
+			ch <- candidate
+		})
+		// the connection handler is called in a goroutine so blocking here would hot cause an issue. However as blocking
+		// in other handlers does cause problems its best to just assume we should not block
+		go func() {
+			<-t.Done()
+			if t.Error() != nil {
+				p.logger.Err(t.Error()).Msgf("could not subscribe to %s", topic)
+			} else {
+				p.logger.Info().Msgf("subscribed to %s", topic)
+			}
+		}()
+		return ch
+	}
+}
+
 // handleMessage handles MQTT subscription message.
 func (p *Publisher) handleMessage() mqtt.MessageHandler {
 	return func(c mqtt.Client, m mqtt.Message) {
@@ -90,7 +143,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 		}
 
 		// The publishing topic is unique to each edge device and is determined by above receiving message payload.
-		answerTopic := p.config.AnswerTopicPrefix + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
+		answerTopic := p.config.AnswerTopicSuffix + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
 		t := c.Publish(answerTopic, byte(p.config.Qos), p.config.Retained, payload)
 		<-t.Done()
 		if t.Error() != nil {
@@ -116,7 +169,12 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 	}
 	logger.Debug().Msg("created video track")
 
-	w := webrtcx.New(p.config.WebRTCConfigOptions, logger)
+	w := webrtcx.New(
+		p.config.WebRTCConfigOptions,
+		logger,
+		p.sendCandidate(offer.Id, offer.TrackSource),
+		p.recvCandidate(offer.Id, offer.TrackSource),
+	)
 
 	// TODO: handle blocking case with timeout for channels.
 	w.OfferChan <- offer
@@ -137,3 +195,38 @@ func (p *Publisher) registerSession(
 	p.sessions.Store(sessionID, videoTrack)
 	p.logger.Debug().Str("key", sessionID).Int32("value", int32(trackSource)).Msg("registered session")
 }
+
+func encodeCandidate(candidate *webrtc.ICECandidate) ([]byte, error) {
+	msg := pb.ICECandidate{
+		Foundation:     candidate.Foundation,
+		Priority:       candidate.Priority,
+		Address:        candidate.Address,
+		Protocol:       int32(candidate.Protocol),
+		Port:           uint32(candidate.Port),
+		Type:           int32(candidate.Typ),
+		Component:      uint32(candidate.Typ),
+		RelatedAddress: candidate.RelatedAddress,
+		RelatedPort:    uint32(candidate.RelatedPort),
+		TcpType:        candidate.TCPType,
+	}
+	return proto.Marshal(&msg)
+}
+
+func decodeCandidate(payload []byte) (*webrtc.ICECandidate, error) {
+	var candidate pb.ICECandidate
+	if err := proto.Unmarshal(payload, &candidate); err != nil {
+		return nil, err
+	}
+	return &webrtc.ICECandidate{
+		Foundation:     candidate.Foundation,
+		Priority:       candidate.Priority,
+		Address:        candidate.Address,
+		Protocol:       webrtc.ICEProtocol(candidate.Protocol),
+		Port:           uint16(candidate.Port),
+		Typ:            webrtc.ICECandidateType(candidate.Type),
+		Component:      uint16(candidate.Component),
+		RelatedAddress: candidate.RelatedAddress,
+		RelatedPort:    uint16(candidate.RelatedPort),
+		TCPType:        candidate.TcpType,
+	}, nil
+}
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 3a76218..56e5dfc 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -67,7 +67,7 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 			return
 		}
 
-		wcx := webrtcx.New(s.config, &logger)
+		wcx := webrtcx.New(s.config, &logger, nil, nil)
 		// TODO: handle blocking case with timeout for channels.
 		wcx.OfferChan <- &offer
 		if err := wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
@@ -77,6 +77,7 @@ func (s *Subscriber) handleSignal() http.HandlerFunc {
 		}
 		logger.Debug().Msg("successfully created subscriber")
 
+		// TODO: Timeout channel receiving to avoid blocking.
 		answer := <-wcx.AnswerChan
 		w.Header().Set("Content-Type", "application/json")
 		if err := json.NewEncoder(w).Encode(answer.Sdp); err != nil {
diff --git a/internal/broadcast/subscriber/subscriber_test.go b/internal/broadcast/subscriber/subscriber_test.go
index 3b14429..789e1c4 100644
--- a/internal/broadcast/subscriber/subscriber_test.go
+++ b/internal/broadcast/subscriber/subscriber_test.go
@@ -14,8 +14,8 @@ func TestSignalBadRequest(t *testing.T) {
 	logger := zerolog.Nop()
 	sub := New(nil, &logger, cfg.WebRTCConfigOptions{})
 
-	body := `{"id":"fa955cc6881b4b45b49ffbf2d81e7223","track_source":0,"sdp":{"type":"offer","sdp":"v=0\r\no=- 4497186609034332336 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS\r\nm=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 124 119 123 118 114 115 116\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:laPQ\r\na=ice-pwd:IEAqRBhJbIHYRq6fMDh7YZH9\r\na=ice-options:trickle\r\na=fingerprint:sha-256 28:5E:F4:E4:1F:A2:02:19:E3:5C:7C:B2:E2:8A:28:98:88:73:9F:0E:CD:17:E2:D0:98:99:6B:89:72:E8:58:5A\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 urn:3gpp:video-orientation\r\na=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\na=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\na=sendrecv\r\na=msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=rtcp-mux\r\na=rtcp-rsize\r\na=rtpmap:96 VP8/90000\r\na=rtcp-fb:96 goog-remb\r\na=rtcp-fb:96 transport-cc\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96 nack pli\r\na=rtpmap:97 rtx/90000\r\na=fmtp:97 apt=96\r\na=rtpmap:98 VP9/90000\r\na=rtcp-fb:98 goog-remb\r\na=rtcp-fb:98 transport-cc\r\na=rtcp-fb:98 ccm fir\r\na=rtcp-fb:98 nack\r\na=rtcp-fb:98 nack pli\r\na=fmtp:98 profile-id=0\r\na=rtpmap:99 rtx/90000\r\na=fmtp:99 apt=98\r\na=rtpmap:100 VP9/90000\r\na=rtcp-fb:100 goog-remb\r\na=rtcp-fb:100 transport-cc\r\na=rtcp-fb:100 ccm fir\r\na=rtcp-fb:100 nack\r\na=rtcp-fb:100 nack pli\r\na=fmtp:100 profile-id=2\r\na=rtpmap:101 rtx/90000\r\na=fmtp:101 apt=100\r\na=rtpmap:102 H264/90000\r\na=rtcp-fb:102 goog-remb\r\na=rtcp-fb:102 transport-cc\r\na=rtcp-fb:102 ccm fir\r\na=rtcp-fb:102 nack\r\na=rtcp-fb:102 nack pli\r\na=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r\na=rtpmap:121 rtx/90000\r\na=fmtp:121 apt=102\r\na=rtpmap:127 H264/90000\r\na=rtcp-fb:127 goog-remb\r\na=rtcp-fb:127 transport-cc\r\na=rtcp-fb:127 ccm fir\r\na=rtcp-fb:127 nack\r\na=rtcp-fb:127 nack pli\r\na=fmtp:127 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r\na=rtpmap:120 rtx/90000\r\na=fmtp:120 apt=127\r\na=rtpmap:125 H264/90000\r\na=rtcp-fb:125 goog-remb\r\na=rtcp-fb:125 transport-cc\r\na=rtcp-fb:125 ccm fir\r\na=rtcp-fb:125 nack\r\na=rtcp-fb:125 nack pli\r\na=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\na=rtpmap:107 rtx/90000\r\na=fmtp:107 apt=125\r\na=rtpmap:108 H264/90000\r\na=rtcp-fb:108 goog-remb\r\na=rtcp-fb:108 transport-cc\r\na=rtcp-fb:108 ccm fir\r\na=rtcp-fb:108 nack\r\na=rtcp-fb:108 nack pli\r\na=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\r\na=rtpmap:109 rtx/90000\r\na=fmtp:109 apt=108\r\na=rtpmap:124 H264/90000\r\na=rtcp-fb:124 goog-remb\r\na=rtcp-fb:124 transport-cc\r\na=rtcp-fb:124 ccm fir\r\na=rtcp-fb:124 nack\r\na=rtcp-fb:124 nack pli\r\na=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032\r\na=rtpmap:119 rtx/90000\r\na=fmtp:119 apt=124\r\na=rtpmap:123 H264/90000\r\na=rtcp-fb:123 goog-remb\r\na=rtcp-fb:123 transport-cc\r\na=rtcp-fb:123 ccm fir\r\na=rtcp-fb:123 nack\r\na=rtcp-fb:123 nack pli\r\na=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032\r\na=rtpmap:118 rtx/90000\r\na=fmtp:118 apt=123\r\na=rtpmap:114 red/90000\r\na=rtpmap:115 rtx/90000\r\na=fmtp:115 apt=114\r\na=rtpmap:116 ulpfec/90000\r\na=ssrc-group:FID 3334187606 1513158839\r\na=ssrc:3334187606 cname:2rzanSazYyXNIwZj\r\na=ssrc:3334187606 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:3334187606 mslabel:-\r\na=ssrc:3334187606 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 cname:2rzanSazYyXNIwZj\r\na=ssrc:1513158839 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 mslabel:-\r\na=ssrc:1513158839 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\n"}}`
-	req, err := http.NewRequest(http.MethodPost, "/v1/broadcast/signal", strings.NewReader(body))
+	body := `{"id":"fa955cc6881b4b45b49ffbf2d81e7223","track_source":0,"sdp":{"type":"offer","sdp":"v=0\r\no=- 4497186609034332336 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS\r\nm=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 124 119 123 118 114 115 116\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:laPQ\r\na=ice-pwd:IEAqRBhJbIHYRq6fMDh7YZH9\r\na=ice-options:trickle\r\na=fingerprint:sha-256 28:5E:F4:E4:1F:A2:02:19:E3:5C:7C:B2:E2:8A:28:98:88:73:9F:0E:CD:17:E2:D0:98:99:6B:89:72:E8:58:5A\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 urn:3gpp:video-orientation\r\na=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\na=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\na=sendrecv\r\na=msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=rtcp-mux\r\na=rtcp-rsize\r\na=rtpmap:96 VP8/90000\r\na=rtcp-fb:96 goog-remb\r\na=rtcp-fb:96 transport-cc\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96 nack pli\r\na=rtpmap:97 rtx/90000\r\na=fmtp:97 apt=96\r\na=rtpmap:98 VP9/90000\r\na=rtcp-fb:98 goog-remb\r\na=rtcp-fb:98 transport-cc\r\na=rtcp-fb:98 ccm fir\r\na=rtcp-fb:98 nack\r\na=rtcp-fb:98 nack pli\r\na=fmtp:98 profile-id=0\r\na=rtpmap:99 rtx/90000\r\na=fmtp:99 apt=98\r\na=rtpmap:100 VP9/90000\r\na=rtcp-fb:100 goog-remb\r\na=rtcp-fb:100 transport-cc\r\na=rtcp-fb:100 ccm fir\r\na=rtcp-fb:100 nack\r\na=rtcp-fb:100 nack pli\r\na=fmtp:100 profile-id=2\r\na=rtpmap:101 rtx/90000\r\na=fmtp:101 apt=100\r\na=rtpmap:102 H264/90000\r\na=rtcp-fb:102 goog-remb\r\na=rtcp-fb:102 transport-cc\r\na=rtcp-fb:102 ccm fir\r\na=rtcp-fb:102 nack\r\na=rtcp-fb:102 nack pli\r\na=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r\na=rtpmap:121 rtx/90000\r\na=fmtp:121 apt=102\r\na=rtpmap:127 H264/90000\r\na=rtcp-fb:127 goog-remb\r\na=rtcp-fb:127 transport-cc\r\na=rtcp-fb:127 ccm fir\r\na=rtcp-fb:127 nack\r\na=rtcp-fb:127 nack pli\r\na=fmtp:127 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r\na=rtpmap:120 rtx/90000\r\na=fmtp:120 apt=127\r\na=rtpmap:125 H264/90000\r\na=rtcp-fb:125 goog-remb\r\na=rtcp-fb:125 transport-cc\r\na=rtcp-fb:125 ccm fir\r\na=rtcp-fb:125 nack\r\na=rtcp-fb:125 nack pli\r\na=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\na=rtpmap:107 rtx/90000\r\na=fmtp:107 apt=125\r\na=rtpmap:108 H264/90000\r\na=rtcp-fb:108 goog-remb\r\na=rtcp-fb:108 transport-cc\r\na=rtcp-fb:108 ccm fir\r\na=rtcp-fb:108 nack\r\na=rtcp-fb:108 nack pli\r\na=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\r\na=rtpmap:109 rtx/90000\r\na=fmtp:109 apt=108\r\na=rtpmap:124 H264/90000\r\na=rtcp-fb:124 goog-remb\r\na=rtcp-fb:124 transport-cc\r\na=rtcp-fb:124 ccm fir\r\na=rtcp-fb:124 nack\r\na=rtcp-fb:124 nack pli\r\na=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032\r\na=rtpmap:119 rtx/90000\r\na=fmtp:119 apt=124\r\na=rtpmap:123 H264/90000\r\na=rtcp-fb:123 goog-remb\r\na=rtcp-fb:123 transport-cc\r\na=rtcp-fb:123 ccm fir\r\na=rtcp-fb:123 nack\r\na=rtcp-fb:123 nack pli\r\na=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032\r\na=rtpmap:118 rtx/90000\r\na=fmtp:118 apt=123\r\na=rtpmap:114 red/90000\r\na=rtpmap:115 rtx/90000\r\na=fmtp:115 apt=114\r\na=rtpmap:116 ulpfec/90000\r\na=ssrc-group:FID 3334187606 1513158839\r\na=ssrc:3334187606 cname:2rzanSazYyXNIwZj\r\na=ssrc:3334187606 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:3334187606 mslabel:-\r\na=ssrc:3334187606 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 cname:2rzanSazYyXNIwZj\r\na=ssrc:1513158839 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 mslabel:-\r\na=ssrc:1513158839 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\n"}}` //nolint:lll
+	req, err := http.NewRequest(http.MethodPost, "/v1/broadcast/signal", strings.NewReader(body)) //nolint:noctx
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 8f93c8c..255774a 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sync"
 	"time"
 
 	pb "github.com/SB-IM/pb/signal"
@@ -15,24 +16,39 @@ import (
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
 )
 
+// SendCandidateFunc sends a candidate to remote webRTC peer.
+type SendCandidateFunc func(candidate *webrtc.ICECandidate) error
+
+// SendCandidateFunc receives a candidate from remote webRTC peer.
+type RecvCandidateFunc func() <-chan *webrtc.ICECandidate
+
 const (
 	rtcpPLIInterval = time.Second * 3
 )
 
 type WebRTC struct {
-	logger     zerolog.Logger
-	config     cfg.WebRTCConfigOptions
+	logger zerolog.Logger
+	config cfg.WebRTCConfigOptions
+
 	OfferChan  chan *pb.SessionDescription
 	AnswerChan chan *pb.SessionDescription
+
+	pendingCandidates []*webrtc.ICECandidate
+	candidatesMux     sync.Mutex
+
+	sendCandidate SendCandidateFunc
+	recvCandidate RecvCandidateFunc
 }
 
 // New returns a new WebRTC.
-func New(config cfg.WebRTCConfigOptions, logger *zerolog.Logger) *WebRTC {
+func New(config cfg.WebRTCConfigOptions, logger *zerolog.Logger, sendCandidate SendCandidateFunc, recvCandidate RecvCandidateFunc) *WebRTC {
 	return &WebRTC{
-		logger:     *logger,
-		config:     config,
-		OfferChan:  make(chan *pb.SessionDescription, 1), // Make 1 buffer so offer sending never blocks
-		AnswerChan: make(chan *pb.SessionDescription, 1), // Make 1 buffer so answer sending never blocks
+		logger:        *logger,
+		config:        config,
+		OfferChan:     make(chan *pb.SessionDescription, 1), // Make 1 buffer so offer sending never blocks
+		AnswerChan:    make(chan *pb.SessionDescription, 1), // Make 1 buffer so answer sending never blocks
+		sendCandidate: sendCandidate,
+		recvCandidate: recvCandidate,
 	}
 }
 
@@ -105,6 +121,24 @@ func (w *WebRTC) CreateSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) error {
 	offer := <-w.OfferChan
 
+	peerConnection.OnICECandidate(func(c *webrtc.ICECandidate) {
+		if c == nil {
+			return
+		}
+		w.candidatesMux.Lock()
+		defer w.candidatesMux.Unlock()
+
+		desc := peerConnection.RemoteDescription()
+		if desc == nil {
+			w.pendingCandidates = append(w.pendingCandidates, c)
+			return
+		}
+		if err := w.sendCandidate(c); err != nil {
+			w.logger.Err(err).Msg("could not send candidate")
+		}
+		w.logger.Debug().Msg("sent an ICEcandidate")
+	})
+
 	// Set the handler for ICE connection state
 	// This will notify you when the peer has connected/disconnected
 	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
@@ -121,8 +155,8 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 		return fmt.Errorf("could not set remote description: %w", err)
 	}
 
-	// Create channel that is blocked until ICE Gathering is complete
-	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)
+	// Signal candidate after setting remote description.
+	go w.signalCandidate(peerConnection)
 
 	answer, err := peerConnection.CreateAnswer(nil)
 	if err != nil {
@@ -133,16 +167,22 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 		return fmt.Errorf("could not set local description: %w", err)
 	}
 
-	// Block until ICE Gathering is complete, disabling trickle ICE
-	// we do this because we only can exchange one signaling message
-	// in a production application you should exchange ICE Candidates via OnICECandidate
-	<-gatherComplete
-
 	// Send answer of local description.
 	// This is a universal answer for both publisher and subscriber in protobuf format.
 	sdp := webrtcSdp2pbSdp(peerConnection.LocalDescription())
 	w.AnswerChan <- sdp
 
+	// Signal candidate
+	w.candidatesMux.Lock()
+	defer w.candidatesMux.Unlock()
+
+	for _, c := range w.pendingCandidates {
+		if err := w.sendCandidate(c); err != nil {
+			return fmt.Errorf("could not send candidate: %w", err)
+		}
+		w.logger.Debug().Msg("sent an ICEcandidate")
+	}
+
 	return nil
 }
 
@@ -158,6 +198,23 @@ func (w *WebRTC) newPeerConnection() (*webrtc.PeerConnection, error) {
 	})
 }
 
+func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection) {
+	// TODO: Stop adding ICE candidate when after signaling succeeded, that is, to exit the loop.
+	// Just set a timer is not enough.
+	ch := w.recvCandidate()
+	for c := range ch {
+		if c == nil {
+			continue
+		}
+		if err := peerConnection.AddICECandidate(webrtc.ICECandidateInit{
+			Candidate: c.ToJSON().Candidate,
+		}); err != nil {
+			w.logger.Err(err).Msg("could not add ICE candidate")
+		}
+		w.logger.Debug().Msg("successfully added an ICE candidate")
+	}
+}
+
 // sendRTCP sends a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
 // This can be less wasteful by processing incoming RTCP events, then we would emit a NACK/PLI when a viewer requests it
 func (w *WebRTC) sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote) {

From 8d2f6b73f3ac16d7c4ab5279680287c803d67da9 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 8 Apr 2021 18:13:43 +0800
Subject: [PATCH 19/52] Full WebSocket SDP signal and candidate trickling
 support

---
 .dockerignore                               |   3 +
 cmd/broadcast/cmd.go                        |   2 +-
 e2e/broadcast/static/index.html             | 126 +++++++++----
 go.mod                                      |   3 +-
 go.sum                                      |  60 +++++-
 internal/broadcast/publisher/publisher.go   |  84 +++------
 internal/broadcast/subscriber/subscriber.go | 198 ++++++++++++++++----
 internal/broadcast/webrtc/webrtc.go         |  42 ++---
 8 files changed, 358 insertions(+), 160 deletions(-)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..ee26374
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+.github
+.idea
+config
diff --git a/cmd/broadcast/cmd.go b/cmd/broadcast/cmd.go
index d890018..96aa80d 100644
--- a/cmd/broadcast/cmd.go
+++ b/cmd/broadcast/cmd.go
@@ -161,7 +161,7 @@ func mqttClientFlags(topicConfigOptions *cfg.MQTTClientConfigOptions) []cli.Flag
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:        "topic-candidate-recv-suffix",
-			Usage:       "MQTT topic suffix for WebRTC candidate receiving, and the receiving topic of edge is /edge/livestream/signal/candidate/recv",
+			Usage:       "MQTT topic suffix for WebRTC candidate receiving, and the receiving topic of edge is /edge/livestream/signal/candidate/recv", //nolint:lll
 			Value:       "/edge/livestream/signal/candidate/send",
 			DefaultText: "/edge/livestream/signal/candidate/send",
 			Destination: &topicConfigOptions.CandidateRecvTopicSuffix,
diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index 58ae63e..aa91705 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -14,53 +14,101 @@
 </body>
 
 <script type="text/javascript">
-    for (let i = 0; i < 2; i++) {
-        let pc = new RTCPeerConnection({
-            iceServers: [
-                {
-                    urls: "stun:stun.l.google.com:19302",
-                }
-            ]
-        })
-        pc.ontrack = function (event) {
-            var el = document.createElement(event.track.kind)
-            el.srcObject = event.streams[0]
-            el.autoplay = true
-            el.controls = true
-
-            document.getElementById('remoteVideos').appendChild(el)
-        }
+    const conn = new WebSocket(`ws://${location.host}/v1/broadcast/signal`)
+
+    let pc = new RTCPeerConnection()
+    pc.ontrack = function (event) {
+        var el = document.createElement(event.track.kind)
+        el.srcObject = event.streams[0]
+        el.autoplay = true
+        el.controls = true
+
+        document.getElementById('remoteVideos').appendChild(el)
+    }
+
+    let log = (msg) => {
+        document.getElementById("log1").innerHTML += msg + "<br>";
+    };
 
-        let log = (msg) => {
-            document.getElementById(`log${i + 1}`).innerHTML += msg + "<br>";
-        };
+    pc.addTransceiver('video');
 
-        pc.addTransceiver('video');
+    pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
+
+    pc.onicecandidate = (e) => {
+        if (!e.candidate) {
+            return
+        }
+
+        let msg = {
+            event: "new-ice-candidate",
+            data: {
+                meta: {
+                    id: "fa955cc6881b4b45b49ffbf2d81e7223",
+                    track_source: 1,
+                },
+                candidate: JSON.stringify(e.candidate)
+            }
+        }
+        conn.send(JSON.stringify(msg))
+    }
 
-        pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
+    conn.addEventListener("close", ev => {
+        console.log(`WebSocket Disconnected code: ${ev.code}, reason: ${ev.reason}`)
+    })
+    conn.addEventListener("open", ev => {
+        console.info("websocket connected")
 
         pc.createOffer()
             .then(offer => {
-                pc.setLocalDescription(offer)
+                pc.setLocalDescription(offer).catch(console)
 
-                sdp = {
-                    id: "fa955cc6881b4b45b49ffbf2d81e7223",
-                    track_source: i,
-                    sdp: offer,
-                };
-                return fetch(`/v1/broadcast/signal`, {
-                    method: 'post',
-                    headers: {
-                        'Accept': 'application/json, text/plain, */*',
-                        'Content-Type': 'application/json'
-                    },
-                    body: JSON.stringify(sdp)
-                })
+                let msg = {
+                    event: "video-offer",
+                    data: {
+                        meta: {
+                            id: "fa955cc6881b4b45b49ffbf2d81e7223",
+                            track_source: 1,
+                        },
+                        sdp: JSON.stringify(offer),
+                    }
+                }
+                conn.send(JSON.stringify(msg))
             })
-            .then(res => res.json())
-            .then(res => pc.setRemoteDescription(res))
-            .catch(alert)
-    }
+            .catch(console);
+        console.log("sent offer")
+    })
+
+    conn.addEventListener("message", ev => {
+        console.log(`received message: ${ev.data}`)
+
+        let msg = JSON.parse(ev.data);
+        if (!msg) {
+            return console.log('failed to parse msg')
+        }
+        switch (msg.event) {
+            case "video-answer":
+                let answer = JSON.parse(msg.data.sdp)
+                if (!answer) {
+                    return console.log('failed to parse answer')
+                }
+                pc.setRemoteDescription(answer).catch(console)
+
+                console.log("set remote description")
+                break;
+            case "new-ice-candidate":
+                let candidate = JSON.parse(msg.data.candidate)
+                if (!candidate) {
+                    return console.log('failed to parse candidate')
+                }
+                pc.addIceCandidate(candidate).catch(console)
+
+                console.log("added a candidate")
+                break;
+            default:
+                break;
+        }
+    })
+
 </script>
 
 </html>
diff --git a/go.mod b/go.mod
index b98d322..210bed6 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.16
 require (
 	github.com/SB-IM/logging v0.2.2
 	github.com/SB-IM/mqtt-client v0.1.0
-	github.com/SB-IM/pb v0.1.4
+	github.com/SB-IM/pb v0.2.9
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
@@ -14,4 +14,5 @@ require (
 	github.com/rs/zerolog v1.21.0
 	github.com/urfave/cli/v2 v2.3.0
 	google.golang.org/protobuf v1.26.0
+	nhooyr.io/websocket v1.8.7
 )
diff --git a/go.sum b/go.sum
index 16ac8e3..47e3596 100644
--- a/go.sum
+++ b/go.sum
@@ -5,10 +5,16 @@ github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
 github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
 github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXCdI=
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
-github.com/SB-IM/pb v0.1.3 h1:Sq1wCcrcX5smwLvEtfw1cN04fHM2T4wiN5FggThLEU8=
-github.com/SB-IM/pb v0.1.3/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
-github.com/SB-IM/pb v0.1.4 h1:eO/S1xQC9Uh6RV7RDcBEPZthZ6uOjGhnlmqLXTKb/y0=
-github.com/SB-IM/pb v0.1.4/go.mod h1:RtzICl4Ha4uq6MGCy2mBu6rrYjVDa5P3GszGg8pjFo0=
+github.com/SB-IM/pb v0.2.4 h1:V1FqyLVTC3AAlJzUj5JkXsZ34/MAHc74N42W0qO9Y+c=
+github.com/SB-IM/pb v0.2.4/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
+github.com/SB-IM/pb v0.2.5 h1:cqtTTTnxnhENJLpK3OI6KVNze1giDzSfVeRhTFtzSCg=
+github.com/SB-IM/pb v0.2.5/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
+github.com/SB-IM/pb v0.2.7 h1:vnn3nyuxsyme/5F8J1y0xoLhcaNomVFwoFPripKcd2Y=
+github.com/SB-IM/pb v0.2.7/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
+github.com/SB-IM/pb v0.2.8 h1:qRwtdWBKgF4wObV2fJ9UCIi2mKstFDg3LS8BjCKwg2Q=
+github.com/SB-IM/pb v0.2.8/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
+github.com/SB-IM/pb v0.2.9 h1:vuLd9cj5+4mG5990qySIQXg4RMd4c/FWOfAEckVsmwM=
+github.com/SB-IM/pb v0.2.9/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -20,7 +26,26 @@ github.com/eclipse/paho.mqtt.golang v1.3.3 h1:Fh1zsLniMFJByLqKrSB9ZRjkbpU0k1Xne2
 github.com/eclipse/paho.mqtt.golang v1.3.3/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
+github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
+github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
+github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
+github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
+github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -35,18 +60,32 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8=
+github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
@@ -107,10 +146,16 @@ github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZ
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -143,6 +188,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -155,6 +201,7 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -181,7 +228,12 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
+nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
+nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
+nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index bc2287c..5ff2492 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -1,6 +1,7 @@
 package publisher
 
 import (
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"sync"
@@ -62,13 +63,13 @@ func (p *Publisher) Signal() {
 
 // sendCandidate sends candidate to remote webRTC peer via MQTT.
 // The publish topic is unique to this edge device.
-func (p *Publisher) sendCandidate(id string, trackSource pb.TrackSource) webrtcx.SendCandidateFunc {
+func (p *Publisher) sendCandidate(meta *pb.Meta) webrtcx.SendCandidateFunc {
 	return func(candidate *webrtc.ICECandidate) error {
-		payload, err := encodeCandidate(candidate)
+		payload, err := pb.EncodeCandidate(candidate)
 		if err != nil {
 			return fmt.Errorf("could not encode candidate: %w", err)
 		}
-		topic := p.config.CandidateSendTopicSuffix + "/" + id + "/" + strconv.Itoa(int(trackSource))
+		topic := p.config.CandidateSendTopicSuffix + "/" + meta.Id + "/" + strconv.Itoa(int(meta.TrackSource))
 		t := p.client.Publish(topic, byte(p.config.Qos), p.config.Retained, payload)
 		// Handle the token in a go routine so this loop keeps sending messages regardless of delivery status
 		go func() {
@@ -85,14 +86,14 @@ func (p *Publisher) sendCandidate(id string, trackSource pb.TrackSource) webrtcx
 // The caller must check if result in channel is nil.
 // sendCandidate receive candidate from remote webRTC peer via MQTT.
 // The subscription topic is unique to this edge device.
-func (p *Publisher) recvCandidate(id string, trackSource pb.TrackSource) webrtcx.RecvCandidateFunc {
-	return func() <-chan *webrtc.ICECandidate {
+func (p *Publisher) recvCandidate(meta *pb.Meta) webrtcx.RecvCandidateFunc {
+	return func() <-chan string {
 		// TODO: Figure how to properly close channel.
-		ch := make(chan *webrtc.ICECandidate, 2) // Make buffer 2 because we have at least 2 sendings.
-		topic := p.config.CandidateRecvTopicSuffix + "/" + id + "/" + strconv.Itoa(int(trackSource))
+		ch := make(chan string, 2) // Make buffer 2 because we have at least 2 sendings.
+		topic := p.config.CandidateRecvTopicSuffix + "/" + meta.Id + "/" + strconv.Itoa(int(meta.TrackSource))
 		// Receive remote ICE candidate with MQTT.
 		t := p.client.Subscribe(topic, byte(p.config.Qos), func(c mqtt.Client, m mqtt.Message) {
-			candidate, err := decodeCandidate(m.Payload())
+			candidate, err := pb.DecodeCandidate(m.Payload())
 			if err != nil {
 				p.logger.Err(err).Msg("could not decode candidate")
 				return
@@ -124,8 +125,8 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 
 		logger := p.logger.With().
 			Str("offer_topic", p.config.OfferTopic).
-			Str("id", offer.Id).
-			Int32("track_source", int32(offer.TrackSource)).
+			Str("id", offer.Meta.Id).
+			Int32("track_source", int32(offer.Meta.TrackSource)).
 			Logger()
 		logger.Debug().Msg("received offer from edge")
 
@@ -136,14 +137,14 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 		}
 		logger.Debug().Msg("Successfully signaled peer connection")
 
-		payload, err := proto.Marshal(answer)
+		payload, err := pb.EncodeSDP(answer, nil)
 		if err != nil {
 			logger.Err(err).Msg("could not encode sdp")
 			return
 		}
 
 		// The publishing topic is unique to each edge device and is determined by above receiving message payload.
-		answerTopic := p.config.AnswerTopicSuffix + "/" + offer.Id + "/" + strconv.Itoa(int(offer.TrackSource))
+		answerTopic := p.config.AnswerTopicSuffix + "/" + offer.Meta.Id + "/" + strconv.Itoa(int(offer.Meta.TrackSource))
 		t := c.Publish(answerTopic, byte(p.config.Qos), p.config.Retained, payload)
 		<-t.Done()
 		if t.Error() != nil {
@@ -153,16 +154,21 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 		logger.Debug().Str("answer_topic", answerTopic).Msg("sent answer to edge")
 
 		// Register session on signaling success.
-		p.registerSession(offer.Id, offer.TrackSource, videoTrack)
+		p.registerSession(offer.Meta, videoTrack)
 	}
 }
 
 // signalPeerConnection creates video track and performs webRTC signaling.
 func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *zerolog.Logger) (
-	*pb.SessionDescription,
+	*webrtc.SessionDescription,
 	*webrtc.TrackLocalStaticRTP,
 	error,
 ) {
+	var sdp webrtc.SessionDescription
+	if err := json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
+		return nil, nil, err
+	}
+
 	videoTrack, err := webrtcx.CreateLocalTrack()
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create webRTC local video track: %w", err)
@@ -172,61 +178,25 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 	w := webrtcx.New(
 		p.config.WebRTCConfigOptions,
 		logger,
-		p.sendCandidate(offer.Id, offer.TrackSource),
-		p.recvCandidate(offer.Id, offer.TrackSource),
+		p.sendCandidate(offer.Meta),
+		p.recvCandidate(offer.Meta),
 	)
 
 	// TODO: handle blocking case with timeout for channels.
-	w.OfferChan <- offer
+	w.SignalChan <- &sdp
 	if err := w.CreatePublisher(videoTrack); err != nil {
 		return nil, nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
 	}
 	logger.Debug().Msg("created publisher")
 
-	return <-w.AnswerChan, videoTrack, nil
+	return <-w.SignalChan, videoTrack, nil
 }
 
 func (p *Publisher) registerSession(
-	id string,
-	trackSource pb.TrackSource,
+	meta *pb.Meta,
 	videoTrack *webrtc.TrackLocalStaticRTP,
 ) {
-	sessionID := id + strconv.Itoa(int(trackSource))
+	sessionID := meta.Id + strconv.Itoa(int(meta.TrackSource))
 	p.sessions.Store(sessionID, videoTrack)
-	p.logger.Debug().Str("key", sessionID).Int32("value", int32(trackSource)).Msg("registered session")
-}
-
-func encodeCandidate(candidate *webrtc.ICECandidate) ([]byte, error) {
-	msg := pb.ICECandidate{
-		Foundation:     candidate.Foundation,
-		Priority:       candidate.Priority,
-		Address:        candidate.Address,
-		Protocol:       int32(candidate.Protocol),
-		Port:           uint32(candidate.Port),
-		Type:           int32(candidate.Typ),
-		Component:      uint32(candidate.Typ),
-		RelatedAddress: candidate.RelatedAddress,
-		RelatedPort:    uint32(candidate.RelatedPort),
-		TcpType:        candidate.TCPType,
-	}
-	return proto.Marshal(&msg)
-}
-
-func decodeCandidate(payload []byte) (*webrtc.ICECandidate, error) {
-	var candidate pb.ICECandidate
-	if err := proto.Unmarshal(payload, &candidate); err != nil {
-		return nil, err
-	}
-	return &webrtc.ICECandidate{
-		Foundation:     candidate.Foundation,
-		Priority:       candidate.Priority,
-		Address:        candidate.Address,
-		Protocol:       webrtc.ICEProtocol(candidate.Protocol),
-		Port:           uint16(candidate.Port),
-		Typ:            webrtc.ICECandidateType(candidate.Type),
-		Component:      uint16(candidate.Component),
-		RelatedAddress: candidate.RelatedAddress,
-		RelatedPort:    uint16(candidate.RelatedPort),
-		TCPType:        candidate.TcpType,
-	}, nil
+	p.logger.Debug().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("registered session")
 }
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 56e5dfc..333021b 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -1,6 +1,7 @@
 package subscriber
 
 import (
+	"context"
 	"encoding/json"
 	"net/http"
 	"strconv"
@@ -10,6 +11,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
+	"nhooyr.io/websocket"
+	"nhooyr.io/websocket/wsjson"
 
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
 	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
@@ -25,6 +28,18 @@ type Subscriber struct {
 	sessions *sync.Map
 }
 
+// incomingMessage is a generic WebSocket incoming message.
+type incomingMessage struct {
+	Event string          `json:"event"`
+	Data  json.RawMessage `json:"data"`
+}
+
+// outgoingMessage is a generic WebSocket outgoing message.
+type outgoingMessage struct {
+	Event string      `json:"event"`
+	Data  interface{} `json:"data"`
+}
+
 // New returns a new Subscriber.
 func New(sessions *sync.Map, logger *zerolog.Logger, config cfg.WebRTCConfigOptions) *Subscriber {
 	l := logger.With().Str("component", "Subscriber").Logger()
@@ -38,53 +53,172 @@ func New(sessions *sync.Map, logger *zerolog.Logger, config cfg.WebRTCConfigOpti
 // Signal performs webRTC signaling for all subscriber peers.
 func (s *Subscriber) Signal() http.Handler {
 	r := mux.NewRouter()
-	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()).Methods(http.MethodPost)
+	r.HandleFunc("/v1/broadcast/signal", s.handleSignal()) // WebRTC SDP signaling. candidates trickling
+	s.logger.Debug().Msg("registered signal HTTP handler")
 
 	if s.config.EnableFrontend {
-		r.Handle("/", http.FileServer(http.Dir("e2e/broadcast/static")))
+		r.Handle("/", http.FileServer(http.Dir("e2e/broadcast/static"))) // E2e static file server for debuging
+		s.logger.Debug().Msg("registered broadcast e2e static file server handler")
 	}
-	s.logger.Debug().Msg("registered signal HTTP handler")
 	return r
 }
 
-// handleSignal handles HTTP subscriber.
+// handleSignal handles subscriber with webSocket api.
+// Has candidate trickle suppoort.
 func (s *Subscriber) handleSignal() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		var offer pb.SessionDescription
-		if err := json.NewDecoder(r.Body).Decode(&offer); err != nil {
-			s.logger.Err(err).Msg("could not decode request json body")
-			http.Error(w, err.Error(), http.StatusBadRequest)
+		c, err := websocket.Accept(w, r, nil)
+		if err != nil {
+			s.logger.Err(err).Msg("could not upgrade to webSocket connection")
 			return
 		}
-		logger := s.logger.With().Str("id", offer.Id).Int32("track_source", int32(offer.TrackSource)).Logger()
-		logger.Debug().Msg("received offer from subscriber")
-
-		sessionID := offer.Id + strconv.Itoa(int(offer.TrackSource))
-		value, ok := s.sessions.Load(sessionID)
-		if !ok {
-			logger.Warn().Msg("no machine id or track source found in existing sessions")
-			http.Error(w, "wrong id or track source", http.StatusBadRequest)
-			return
+		defer c.Close(websocket.StatusNormalClosure, "")
+
+		ctx, cancel := context.WithCancel(r.Context())
+		defer cancel()
+
+		s.processMessage(ctx, c)
+	}
+}
+
+func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
+	candidateChan := map[pb.TrackSource]chan string{
+		pb.TrackSource_DRONE:   make(chan string, 2), // make buffer 2 because we send candidate at least twice.
+		pb.TrackSource_MONITOR: make(chan string, 2), // make buffer 2 because we send candidate at least twice.
+	}
+	defer func() {
+		for _, ch := range candidateChan {
+			close(ch)
 		}
+	}()
 
-		wcx := webrtcx.New(s.config, &logger, nil, nil)
-		// TODO: handle blocking case with timeout for channels.
-		wcx.OfferChan <- &offer
-		if err := wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
-			logger.Err(err).Msg("failed to create subscriber")
-			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+	for {
+		msgType, raw, err := c.Read(ctx)
+		if err != nil {
+			s.logger.Err(err).Msg("could not read message")
+			return
+		}
+		if msgType != websocket.MessageText {
+			s.logger.Warn().Msg("message type is not text")
 			return
 		}
-		logger.Debug().Msg("successfully created subscriber")
-
-		// TODO: Timeout channel receiving to avoid blocking.
-		answer := <-wcx.AnswerChan
-		w.Header().Set("Content-Type", "application/json")
-		if err := json.NewEncoder(w).Encode(answer.Sdp); err != nil {
-			logger.Err(err).Msg("could not encode json response body")
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+
+		var msg incomingMessage
+		if err := json.Unmarshal(raw, &msg); err != nil {
+			s.logger.Err(err).Msg("could not unmarshal JSON data")
 			return
 		}
-		logger.Debug().Msg("sent answer to subscriber")
+
+		switch msg.Event {
+		case "video-offer":
+			var offer pb.SessionDescription
+			if err := json.Unmarshal(msg.Data, &offer); err != nil {
+				s.logger.Err(err).Msg("could not unmarshal JSON data")
+				return
+			}
+			if offer.Meta == nil || offer.Meta.Id == "" {
+				s.logger.Error().Msg("incorrect metadata")
+				return
+			}
+			logger := s.logger.With().Str("id", offer.Meta.Id).Int32("track_source", int32(offer.Meta.TrackSource)).Logger()
+			logger.Debug().Msg("received offer from subscriber")
+
+			sessionID := offer.Meta.Id + strconv.Itoa(int(offer.Meta.TrackSource))
+			value, ok := s.sessions.Load(sessionID)
+			if !ok {
+				logger.Error().Msg("no machine id or track source found in existing sessions")
+				return
+			}
+
+			wcx := webrtcx.New(s.config, &logger, sendCandidate(ctx, c, offer.Meta), recvCandidate(candidateChan[offer.Meta.TrackSource]))
+
+			var sdp webrtc.SessionDescription
+			if err = json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
+				s.logger.Err(err).Msg("could not unmarshal sdp")
+				return
+			}
+			// TODO: handle blocking case with timeout for channels.
+			wcx.SignalChan <- &sdp
+			if err = wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
+				logger.Err(err).Msg("failed to create subscriber")
+				return
+			}
+			logger.Debug().Msg("successfully created subscriber")
+
+			// TODO: Timeout channel receiving to avoid blocking.
+			answer := <-wcx.SignalChan
+			b, err := json.Marshal(answer)
+			if err != nil {
+				s.logger.Err(err).Msg("could not unmarshal answer to JSON")
+				return
+			}
+			if err := wsjson.Write(ctx, c, &outgoingMessage{
+				Event: "video-answer",
+				Data: &pb.SessionDescription{
+					Meta: offer.Meta,
+					Sdp:  string(b),
+				},
+			}); err != nil {
+				s.logger.Err(err).Msg("could not write answer JSON")
+				return
+			}
+			logger.Debug().Msg("sent answer to subscriber")
+		case "new-ice-candidate":
+			var candidate pb.ICECandidate
+			if err := json.Unmarshal(msg.Data, &candidate); err != nil {
+				s.logger.Err(err).Msg("could not unmarshal JSON data")
+				return
+			}
+			if candidate.Meta == nil || candidate.Meta.Id == "" {
+				s.logger.Error().Msg("incorrect metadata")
+				return
+			}
+			sessionID := candidate.Meta.Id + strconv.Itoa(int(candidate.Meta.TrackSource))
+			_, ok := s.sessions.Load(sessionID)
+			if !ok {
+				s.logger.Error().Msg("no machine id or track source found in existing sessions")
+				return
+			}
+
+			var candidateInit webrtc.ICECandidateInit
+			if err := json.Unmarshal([]byte(candidate.Candidate), &candidateInit); err != nil {
+				s.logger.Err(err).Msg("could not unmarshal JSON candidate")
+				return
+			}
+			if candidate.Meta.TrackSource == pb.TrackSource_DRONE {
+				candidateChan[pb.TrackSource_DRONE] <- candidateInit.Candidate
+			} else {
+				candidateChan[pb.TrackSource_MONITOR] <- candidateInit.Candidate
+			}
+		default:
+			s.logger.Warn().Str("event", msg.Event).Msg("unknown event")
+		}
+	}
+}
+
+// sendCandidate sends an ice candidate through webSocket.
+// It can be called multiple time to send multiple ice candidates.
+func sendCandidate(ctx context.Context, c *websocket.Conn, meta *pb.Meta) webrtcx.SendCandidateFunc {
+	return func(candidate *webrtc.ICECandidate) error {
+		// See: https://github.com/pion/example-webrtc-applications/blob/166d375aa9f8725e968758747e0d5bcf66d5b8dc/sfu-ws/main.go#L269-L269
+		candidateJSON, err := json.Marshal(candidate.ToJSON())
+		if err != nil {
+			return err
+		}
+		return wsjson.Write(ctx, c, outgoingMessage{
+			Event: "new-ice-candidate",
+			Data: &pb.ICECandidate{
+				Meta:      meta,
+				Candidate: string(candidateJSON),
+			},
+		})
+	}
+}
+
+// recvCandidate sends an ice candidate through webSocket.
+// It continually reads from established webSocket connection getting ice candidates.
+func recvCandidate(candidateChan <-chan string) webrtcx.RecvCandidateFunc {
+	return func() <-chan string {
+		return candidateChan
 	}
 }
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 255774a..2d2a1f0 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -7,7 +7,6 @@ import (
 	"sync"
 	"time"
 
-	pb "github.com/SB-IM/pb/signal"
 	"github.com/google/uuid"
 	"github.com/pion/rtcp"
 	"github.com/pion/webrtc/v3"
@@ -20,7 +19,7 @@ import (
 type SendCandidateFunc func(candidate *webrtc.ICECandidate) error
 
 // SendCandidateFunc receives a candidate from remote webRTC peer.
-type RecvCandidateFunc func() <-chan *webrtc.ICECandidate
+type RecvCandidateFunc func() <-chan string
 
 const (
 	rtcpPLIInterval = time.Second * 3
@@ -30,8 +29,8 @@ type WebRTC struct {
 	logger zerolog.Logger
 	config cfg.WebRTCConfigOptions
 
-	OfferChan  chan *pb.SessionDescription
-	AnswerChan chan *pb.SessionDescription
+	// SignalChan is a biderection channel.
+	SignalChan chan *webrtc.SessionDescription
 
 	pendingCandidates []*webrtc.ICECandidate
 	candidatesMux     sync.Mutex
@@ -45,8 +44,7 @@ func New(config cfg.WebRTCConfigOptions, logger *zerolog.Logger, sendCandidate S
 	return &WebRTC{
 		logger:        *logger,
 		config:        config,
-		OfferChan:     make(chan *pb.SessionDescription, 1), // Make 1 buffer so offer sending never blocks
-		AnswerChan:    make(chan *pb.SessionDescription, 1), // Make 1 buffer so answer sending never blocks
+		SignalChan:    make(chan *webrtc.SessionDescription, 1), // Make 1 buffer so SDP signaling never blocks
 		sendCandidate: sendCandidate,
 		recvCandidate: recvCandidate,
 	}
@@ -119,7 +117,7 @@ func (w *WebRTC) CreateSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 }
 
 func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) error {
-	offer := <-w.OfferChan
+	offer := <-w.SignalChan
 
 	peerConnection.OnICECandidate(func(c *webrtc.ICECandidate) {
 		if c == nil {
@@ -151,7 +149,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 		}
 	})
 
-	if err := peerConnection.SetRemoteDescription(pbSdp2webrtcSdp(offer)); err != nil {
+	if err := peerConnection.SetRemoteDescription(*offer); err != nil {
 		return fmt.Errorf("could not set remote description: %w", err)
 	}
 
@@ -169,8 +167,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 
 	// Send answer of local description.
 	// This is a universal answer for both publisher and subscriber in protobuf format.
-	sdp := webrtcSdp2pbSdp(peerConnection.LocalDescription())
-	w.AnswerChan <- sdp
+	w.SignalChan <- peerConnection.LocalDescription()
 
 	// Signal candidate
 	w.candidatesMux.Lock()
@@ -203,11 +200,8 @@ func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection) {
 	// Just set a timer is not enough.
 	ch := w.recvCandidate()
 	for c := range ch {
-		if c == nil {
-			continue
-		}
 		if err := peerConnection.AddICECandidate(webrtc.ICECandidateInit{
-			Candidate: c.ToJSON().Candidate,
+			Candidate: c,
 		}); err != nil {
 			w.logger.Err(err).Msg("could not add ICE candidate")
 		}
@@ -243,18 +237,14 @@ func (w *WebRTC) processRTCP(rtpSender *webrtc.RTPSender) {
 	}
 }
 
-func pbSdp2webrtcSdp(sdp *pb.SessionDescription) webrtc.SessionDescription {
-	return webrtc.SessionDescription{
-		Type: webrtc.NewSDPType(sdp.Sdp.Type),
-		SDP:  sdp.Sdp.Sdp,
-	}
+// NoopSendCandidateFunc does nothing.
+func NoopSendCandidateFunc(candidate *webrtc.ICECandidate) error {
+	return nil
 }
 
-func webrtcSdp2pbSdp(sdp *webrtc.SessionDescription) *pb.SessionDescription {
-	return &pb.SessionDescription{
-		Sdp: &pb.SDP{
-			Type: sdp.Type.String(),
-			Sdp:  sdp.SDP,
-		},
-	}
+// NoopRecvCandidateFunc does nothing.
+func NoopRecvCandidateFunc() <-chan string {
+	ch := make(chan string)
+	close(ch)
+	return ch
 }

From 703dd1d7ea89e14e2a84f8b1688506dda56db9ff Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 10:38:03 +0800
Subject: [PATCH 20/52] Remove old HTTP test

---
 .../broadcast/subscriber/subscriber_test.go   | 32 -------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 internal/broadcast/subscriber/subscriber_test.go

diff --git a/internal/broadcast/subscriber/subscriber_test.go b/internal/broadcast/subscriber/subscriber_test.go
deleted file mode 100644
index 789e1c4..0000000
--- a/internal/broadcast/subscriber/subscriber_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package subscriber
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
-	"github.com/rs/zerolog"
-)
-
-func TestSignalBadRequest(t *testing.T) {
-	logger := zerolog.Nop()
-	sub := New(nil, &logger, cfg.WebRTCConfigOptions{})
-
-	body := `{"id":"fa955cc6881b4b45b49ffbf2d81e7223","track_source":0,"sdp":{"type":"offer","sdp":"v=0\r\no=- 4497186609034332336 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS\r\nm=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 102 121 127 120 125 107 108 109 124 119 123 118 114 115 116\r\nc=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:laPQ\r\na=ice-pwd:IEAqRBhJbIHYRq6fMDh7YZH9\r\na=ice-options:trickle\r\na=fingerprint:sha-256 28:5E:F4:E4:1F:A2:02:19:E3:5C:7C:B2:E2:8A:28:98:88:73:9F:0E:CD:17:E2:D0:98:99:6B:89:72:E8:58:5A\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:toffset\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 urn:3gpp:video-orientation\r\na=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\na=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\na=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\na=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\na=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\na=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\na=sendrecv\r\na=msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=rtcp-mux\r\na=rtcp-rsize\r\na=rtpmap:96 VP8/90000\r\na=rtcp-fb:96 goog-remb\r\na=rtcp-fb:96 transport-cc\r\na=rtcp-fb:96 ccm fir\r\na=rtcp-fb:96 nack\r\na=rtcp-fb:96 nack pli\r\na=rtpmap:97 rtx/90000\r\na=fmtp:97 apt=96\r\na=rtpmap:98 VP9/90000\r\na=rtcp-fb:98 goog-remb\r\na=rtcp-fb:98 transport-cc\r\na=rtcp-fb:98 ccm fir\r\na=rtcp-fb:98 nack\r\na=rtcp-fb:98 nack pli\r\na=fmtp:98 profile-id=0\r\na=rtpmap:99 rtx/90000\r\na=fmtp:99 apt=98\r\na=rtpmap:100 VP9/90000\r\na=rtcp-fb:100 goog-remb\r\na=rtcp-fb:100 transport-cc\r\na=rtcp-fb:100 ccm fir\r\na=rtcp-fb:100 nack\r\na=rtcp-fb:100 nack pli\r\na=fmtp:100 profile-id=2\r\na=rtpmap:101 rtx/90000\r\na=fmtp:101 apt=100\r\na=rtpmap:102 H264/90000\r\na=rtcp-fb:102 goog-remb\r\na=rtcp-fb:102 transport-cc\r\na=rtcp-fb:102 ccm fir\r\na=rtcp-fb:102 nack\r\na=rtcp-fb:102 nack pli\r\na=fmtp:102 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f\r\na=rtpmap:121 rtx/90000\r\na=fmtp:121 apt=102\r\na=rtpmap:127 H264/90000\r\na=rtcp-fb:127 goog-remb\r\na=rtcp-fb:127 transport-cc\r\na=rtcp-fb:127 ccm fir\r\na=rtcp-fb:127 nack\r\na=rtcp-fb:127 nack pli\r\na=fmtp:127 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f\r\na=rtpmap:120 rtx/90000\r\na=fmtp:120 apt=127\r\na=rtpmap:125 H264/90000\r\na=rtcp-fb:125 goog-remb\r\na=rtcp-fb:125 transport-cc\r\na=rtcp-fb:125 ccm fir\r\na=rtcp-fb:125 nack\r\na=rtcp-fb:125 nack pli\r\na=fmtp:125 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f\r\na=rtpmap:107 rtx/90000\r\na=fmtp:107 apt=125\r\na=rtpmap:108 H264/90000\r\na=rtcp-fb:108 goog-remb\r\na=rtcp-fb:108 transport-cc\r\na=rtcp-fb:108 ccm fir\r\na=rtcp-fb:108 nack\r\na=rtcp-fb:108 nack pli\r\na=fmtp:108 level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f\r\na=rtpmap:109 rtx/90000\r\na=fmtp:109 apt=108\r\na=rtpmap:124 H264/90000\r\na=rtcp-fb:124 goog-remb\r\na=rtcp-fb:124 transport-cc\r\na=rtcp-fb:124 ccm fir\r\na=rtcp-fb:124 nack\r\na=rtcp-fb:124 nack pli\r\na=fmtp:124 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d0032\r\na=rtpmap:119 rtx/90000\r\na=fmtp:119 apt=124\r\na=rtpmap:123 H264/90000\r\na=rtcp-fb:123 goog-remb\r\na=rtcp-fb:123 transport-cc\r\na=rtcp-fb:123 ccm fir\r\na=rtcp-fb:123 nack\r\na=rtcp-fb:123 nack pli\r\na=fmtp:123 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032\r\na=rtpmap:118 rtx/90000\r\na=fmtp:118 apt=123\r\na=rtpmap:114 red/90000\r\na=rtpmap:115 rtx/90000\r\na=fmtp:115 apt=114\r\na=rtpmap:116 ulpfec/90000\r\na=ssrc-group:FID 3334187606 1513158839\r\na=ssrc:3334187606 cname:2rzanSazYyXNIwZj\r\na=ssrc:3334187606 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:3334187606 mslabel:-\r\na=ssrc:3334187606 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 cname:2rzanSazYyXNIwZj\r\na=ssrc:1513158839 msid:- 0e5d83df-122c-442b-b987-6f0071ebb905\r\na=ssrc:1513158839 mslabel:-\r\na=ssrc:1513158839 label:0e5d83df-122c-442b-b987-6f0071ebb905\r\n"}}` //nolint:lll
-	req, err := http.NewRequest(http.MethodPost, "/v1/broadcast/signal", strings.NewReader(body)) //nolint:noctx
-	if err != nil {
-		t.Fatal(err)
-	}
-	req.Header.Add("Content-Type", "application/json")
-
-	rr := httptest.NewRecorder()
-
-	r := sub.Signal()
-	r.ServeHTTP(rr, req)
-
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("handler returned wrong status code, got: %v want: %v", rr.Code, http.StatusBadRequest)
-	}
-}

From eea57920610a7df424cae54f79b163c0f8b3c9a3 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 14:57:51 +0800
Subject: [PATCH 21/52] Add WebSocket errors api

---
 internal/broadcast/httpx/errors.go          | 24 +++++++++
 internal/broadcast/subscriber/subscriber.go | 56 +++++++++++++++------
 internal/broadcast/webrtc/webrtc.go         |  6 +--
 3 files changed, 67 insertions(+), 19 deletions(-)
 create mode 100644 internal/broadcast/httpx/errors.go

diff --git a/internal/broadcast/httpx/errors.go b/internal/broadcast/httpx/errors.go
new file mode 100644
index 0000000..181c0fc
--- /dev/null
+++ b/internal/broadcast/httpx/errors.go
@@ -0,0 +1,24 @@
+package httpx
+
+// Code is an error code.
+type Code int
+
+const (
+	// Code specifically for broadcast service.
+	ErrReadMessage Code = iota + 10000
+	ErrIncorrectMetadata
+	ErrMetadataNotMatched
+	ErrFailedToCreateSubscriber
+
+	// Code for Common errors.
+	ErrUnmarshalJSON
+)
+
+// Errors maps error code to error message.
+var Errors = map[Code]string{
+	ErrReadMessage:              "Could not read message",
+	ErrIncorrectMetadata:        "Incorrect edge device metadata",
+	ErrMetadataNotMatched:       "Metadata not matched with any existing session",
+	ErrFailedToCreateSubscriber: "Failed to create subscriber for user",
+	ErrUnmarshalJSON:            "Could not unmarshal JSON data",
+}
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 333021b..b117f69 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -15,6 +15,7 @@ import (
 	"nhooyr.io/websocket/wsjson"
 
 	"github.com/SB-IM/skywalker/internal/broadcast/cfg"
+	"github.com/SB-IM/skywalker/internal/broadcast/httpx"
 	webrtcx "github.com/SB-IM/skywalker/internal/broadcast/webrtc"
 )
 
@@ -31,12 +32,14 @@ type Subscriber struct {
 // incomingMessage is a generic WebSocket incoming message.
 type incomingMessage struct {
 	Event string          `json:"event"`
+	ID    string          `json:"id"`
 	Data  json.RawMessage `json:"data"`
 }
 
 // outgoingMessage is a generic WebSocket outgoing message.
 type outgoingMessage struct {
 	Event string      `json:"event"`
+	ID    string      `json:"id"`
 	Data  interface{} `json:"data"`
 }
 
@@ -64,10 +67,12 @@ func (s *Subscriber) Signal() http.Handler {
 }
 
 // handleSignal handles subscriber with webSocket api.
-// Has candidate trickle suppoort.
+// Has candidate trickle support.
 func (s *Subscriber) handleSignal() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		c, err := websocket.Accept(w, r, nil)
+		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
+			OriginPatterns: []string{"InsecureSkipVerify"},
+		})
 		if err != nil {
 			s.logger.Err(err).Msg("could not upgrade to webSocket connection")
 			return
@@ -93,19 +98,10 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 	}()
 
 	for {
-		msgType, raw, err := c.Read(ctx)
-		if err != nil {
-			s.logger.Err(err).Msg("could not read message")
-			return
-		}
-		if msgType != websocket.MessageText {
-			s.logger.Warn().Msg("message type is not text")
-			return
-		}
-
 		var msg incomingMessage
-		if err := json.Unmarshal(raw, &msg); err != nil {
-			s.logger.Err(err).Msg("could not unmarshal JSON data")
+		if err := wsjson.Read(ctx, c, &msg); err != nil {
+			s.logger.Err(err).Msg("could not read message")
+			_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrReadMessage)
 			return
 		}
 
@@ -114,10 +110,12 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 			var offer pb.SessionDescription
 			if err := json.Unmarshal(msg.Data, &offer); err != nil {
 				s.logger.Err(err).Msg("could not unmarshal JSON data")
+				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrUnmarshalJSON)
 				return
 			}
 			if offer.Meta == nil || offer.Meta.Id == "" {
 				s.logger.Error().Msg("incorrect metadata")
+				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrIncorrectMetadata)
 				return
 			}
 			logger := s.logger.With().Str("id", offer.Meta.Id).Int32("track_source", int32(offer.Meta.TrackSource)).Logger()
@@ -127,20 +125,23 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 			value, ok := s.sessions.Load(sessionID)
 			if !ok {
 				logger.Error().Msg("no machine id or track source found in existing sessions")
+				_ = replyErr(ctx, c, msg.ID, offer.Meta, httpx.ErrMetadataNotMatched)
 				return
 			}
 
 			wcx := webrtcx.New(s.config, &logger, sendCandidate(ctx, c, offer.Meta), recvCandidate(candidateChan[offer.Meta.TrackSource]))
 
 			var sdp webrtc.SessionDescription
-			if err = json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
+			if err := json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
 				s.logger.Err(err).Msg("could not unmarshal sdp")
+				_ = replyErr(ctx, c, msg.ID, offer.Meta, httpx.ErrUnmarshalJSON)
 				return
 			}
 			// TODO: handle blocking case with timeout for channels.
 			wcx.SignalChan <- &sdp
-			if err = wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
+			if err := wcx.CreateSubscriber(value.(*webrtc.TrackLocalStaticRTP)); err != nil {
 				logger.Err(err).Msg("failed to create subscriber")
+				_ = replyErr(ctx, c, msg.ID, offer.Meta, httpx.ErrFailedToCreateSubscriber)
 				return
 			}
 			logger.Debug().Msg("successfully created subscriber")
@@ -150,6 +151,7 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 			b, err := json.Marshal(answer)
 			if err != nil {
 				s.logger.Err(err).Msg("could not unmarshal answer to JSON")
+				_ = replyErr(ctx, c, msg.ID, offer.Meta, httpx.ErrUnmarshalJSON)
 				return
 			}
 			if err := wsjson.Write(ctx, c, &outgoingMessage{
@@ -167,22 +169,26 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 			var candidate pb.ICECandidate
 			if err := json.Unmarshal(msg.Data, &candidate); err != nil {
 				s.logger.Err(err).Msg("could not unmarshal JSON data")
+				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrUnmarshalJSON)
 				return
 			}
 			if candidate.Meta == nil || candidate.Meta.Id == "" {
 				s.logger.Error().Msg("incorrect metadata")
+				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrIncorrectMetadata)
 				return
 			}
 			sessionID := candidate.Meta.Id + strconv.Itoa(int(candidate.Meta.TrackSource))
 			_, ok := s.sessions.Load(sessionID)
 			if !ok {
 				s.logger.Error().Msg("no machine id or track source found in existing sessions")
+				_ = replyErr(ctx, c, msg.ID, candidate.Meta, httpx.ErrMetadataNotMatched)
 				return
 			}
 
 			var candidateInit webrtc.ICECandidateInit
 			if err := json.Unmarshal([]byte(candidate.Candidate), &candidateInit); err != nil {
 				s.logger.Err(err).Msg("could not unmarshal JSON candidate")
+				_ = replyErr(ctx, c, msg.ID, candidate.Meta, httpx.ErrUnmarshalJSON)
 				return
 			}
 			if candidate.Meta.TrackSource == pb.TrackSource_DRONE {
@@ -222,3 +228,21 @@ func recvCandidate(candidateChan <-chan string) webrtcx.RecvCandidateFunc {
 		return candidateChan
 	}
 }
+
+// replyErr is an uniform error event reply to WebSocket client.
+func replyErr(ctx context.Context, c *websocket.Conn, id string, meta *pb.Meta, code httpx.Code) error {
+	type data struct {
+		Meta *pb.Meta   `json:"meta,omitempty"`
+		Code httpx.Code `json:"code"`
+		Msg  string     `json:"message"`
+	}
+	return wsjson.Write(ctx, c, outgoingMessage{
+		Event: "error",
+		ID:    id,
+		Data: data{
+			Meta: meta,
+			Code: code,
+			Msg:  httpx.Errors[code],
+		},
+	})
+}
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 2d2a1f0..61efb15 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -29,7 +29,7 @@ type WebRTC struct {
 	logger zerolog.Logger
 	config cfg.WebRTCConfigOptions
 
-	// SignalChan is a biderection channel.
+	// SignalChan is a bi-direction channel.
 	SignalChan chan *webrtc.SessionDescription
 
 	pendingCandidates []*webrtc.ICECandidate
@@ -134,7 +134,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 		if err := w.sendCandidate(c); err != nil {
 			w.logger.Err(err).Msg("could not send candidate")
 		}
-		w.logger.Debug().Msg("sent an ICEcandidate")
+		w.logger.Debug().Msg("sent an ICE candidate")
 	})
 
 	// Set the handler for ICE connection state
@@ -238,7 +238,7 @@ func (w *WebRTC) processRTCP(rtpSender *webrtc.RTPSender) {
 }
 
 // NoopSendCandidateFunc does nothing.
-func NoopSendCandidateFunc(candidate *webrtc.ICECandidate) error {
+func NoopSendCandidateFunc(_ *webrtc.ICECandidate) error {
 	return nil
 }
 

From 34cdf4af760a4c3f0de13cca43e376a2905798de Mon Sep 17 00:00:00 2001
From: Rocka <i@rocka.me>
Date: Mon, 12 Apr 2021 17:00:20 +0800
Subject: [PATCH 22/52] fix(e2e): ensure add candidates after remote sdp

---
 e2e/broadcast/static/index.html | 44 ++++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index aa91705..8cc5270 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -16,6 +16,8 @@
 <script type="text/javascript">
     const conn = new WebSocket(`ws://${location.host}/v1/broadcast/signal`)
 
+    let answered = false
+    let candidates = []
     let pc = new RTCPeerConnection()
     pc.ontrack = function (event) {
         var el = document.createElement(event.track.kind)
@@ -78,6 +80,21 @@
         console.log("sent offer")
     })
 
+    function addCandidate(str) {
+        let candidate
+        try {
+            candidate = JSON.parse(str)
+        } catch (e) {
+            return console.log('failed to parse candidate')
+        }
+        if (!candidate) {
+            return console.log('empty candidate')
+        }
+        pc.addIceCandidate(candidate)
+            .then(() => console.log("added a candidate"))
+            .catch(e => console.error(e))
+    }
+
     conn.addEventListener("message", ev => {
         console.log(`received message: ${ev.data}`)
 
@@ -87,22 +104,27 @@
         }
         switch (msg.event) {
             case "video-answer":
-                let answer = JSON.parse(msg.data.sdp)
-                if (!answer) {
+                let answer
+                try {
+                    answer = JSON.parse(msg.data.sdp)
+                } catch (e) {
                     return console.log('failed to parse answer')
                 }
-                pc.setRemoteDescription(answer).catch(console)
-
-                console.log("set remote description")
+                pc.setRemoteDescription(answer)
+                    .then(() => {
+                        answered = true
+                        console.log("set remote description")
+                        candidates.forEach(c => addCandidate(c))
+                        candidates = []
+                    })
+                    .catch(e => console.error(e))
                 break;
             case "new-ice-candidate":
-                let candidate = JSON.parse(msg.data.candidate)
-                if (!candidate) {
-                    return console.log('failed to parse candidate')
+                if (!answered) {
+                    candidates.push(msg.data.candidate)
+                    return
                 }
-                pc.addIceCandidate(candidate).catch(console)
-
-                console.log("added a candidate")
+                addCandidate(msg.data.candidate)
                 break;
             default:
                 break;

From 0cff40d7abba82dc12f03b5d6b6e318dc0177082 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 18:32:30 +0800
Subject: [PATCH 23/52] Add event id

---
 e2e/broadcast/static/index.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index 8cc5270..ef37d92 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -43,6 +43,7 @@
 
         let msg = {
             event: "new-ice-candidate",
+            id: Date.now().toString(),
             data: {
                 meta: {
                     id: "fa955cc6881b4b45b49ffbf2d81e7223",
@@ -66,6 +67,7 @@
 
                 let msg = {
                     event: "video-offer",
+                    id: Date.now().toString(),
                     data: {
                         meta: {
                             id: "fa955cc6881b4b45b49ffbf2d81e7223",

From 1bf0524a4706f8840d8bd63e31f992314c365857 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 18:32:41 +0800
Subject: [PATCH 24/52] Close WebSocket connection when client disconnects

---
 internal/broadcast/subscriber/subscriber.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index b117f69..90ad91c 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -11,6 +11,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
 	"nhooyr.io/websocket"
 	"nhooyr.io/websocket/wsjson"
 
@@ -118,7 +119,7 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrIncorrectMetadata)
 				return
 			}
-			logger := s.logger.With().Str("id", offer.Meta.Id).Int32("track_source", int32(offer.Meta.TrackSource)).Logger()
+			logger := s.logger.With().Str("event_id", msg.ID).Str("id", offer.Meta.Id).Int32("track_source", int32(offer.Meta.TrackSource)).Logger()
 			logger.Debug().Msg("received offer from subscriber")
 
 			sessionID := offer.Meta.Id + strconv.Itoa(int(offer.Meta.TrackSource))
@@ -199,6 +200,12 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 		default:
 			s.logger.Warn().Str("event", msg.Event).Msg("unknown event")
 		}
+
+		select {
+		case <-ctx.Done():
+			log.Debug().Msgf("context is done: %v", ctx.Err())
+			return
+		}
 	}
 }
 

From 9d0e71f946451069cfa2ba068ca79587548aba5e Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 18:51:00 +0800
Subject: [PATCH 25/52] Close WebSocket connection when client disconnects

---
 internal/broadcast/subscriber/subscriber.go | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 90ad91c..3422840 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -11,7 +11,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pion/webrtc/v3"
 	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
 	"nhooyr.io/websocket"
 	"nhooyr.io/websocket/wsjson"
 
@@ -101,8 +100,12 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 	for {
 		var msg incomingMessage
 		if err := wsjson.Read(ctx, c, &msg); err != nil {
-			s.logger.Err(err).Msg("could not read message")
-			_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrReadMessage)
+			if websocket.CloseStatus(err) == websocket.StatusGoingAway {
+				s.logger.Debug().Msg("client closed connection")
+			} else {
+				s.logger.Err(err).Msg("could not read message")
+				_ = replyErr(ctx, c, msg.ID, nil, httpx.ErrReadMessage)
+			}
 			return
 		}
 
@@ -200,12 +203,6 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 		default:
 			s.logger.Warn().Str("event", msg.Event).Msg("unknown event")
 		}
-
-		select {
-		case <-ctx.Done():
-			log.Debug().Msgf("context is done: %v", ctx.Err())
-			return
-		}
 	}
 }
 

From f755a1439d09e3aeccc514beedad54f86dfe7a41 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 12 Apr 2021 19:24:10 +0800
Subject: [PATCH 26/52] Tidy RTPSender after disconnecting(#8)

---
 internal/broadcast/webrtc/webrtc.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 61efb15..2b2e5f5 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -231,7 +231,11 @@ func (w *WebRTC) processRTCP(rtpSender *webrtc.RTPSender) {
 	rtcpBuf := make([]byte, 1500)
 	for {
 		if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
-			w.logger.Err(rtcpErr).Send()
+			if errors.Is(rtcpErr, io.EOF) || errors.Is(rtcpErr, io.ErrClosedPipe) {
+				_ = rtpSender.Stop()
+			} else {
+				w.logger.Err(rtcpErr).Send()
+			}
 			return
 		}
 	}

From 6d09a50110963272e3728fdcfc75cae418e3bed9 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 13 Apr 2021 11:33:36 +0800
Subject: [PATCH 27/52] Deregister session after publisher disconnected(#8)

---
 internal/broadcast/publisher/publisher.go   |  9 +++++
 internal/broadcast/subscriber/subscriber.go |  8 +++-
 internal/broadcast/webrtc/webrtc.go         | 43 +++++++++++++++------
 3 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index 5ff2492..ec7b245 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -180,6 +180,7 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 		logger,
 		p.sendCandidate(offer.Meta),
 		p.recvCandidate(offer.Meta),
+		p.deregisterSession(offer.Meta),
 	)
 
 	// TODO: handle blocking case with timeout for channels.
@@ -200,3 +201,11 @@ func (p *Publisher) registerSession(
 	p.sessions.Store(sessionID, videoTrack)
 	p.logger.Debug().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("registered session")
 }
+
+func (p *Publisher) deregisterSession(meta *pb.Meta) webrtcx.DeregisterSessionFunc {
+	return func() {
+		sessionID := meta.Id + strconv.Itoa(int(meta.TrackSource))
+		p.sessions.Delete(sessionID)
+		p.logger.Debug().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("deregistered session")
+	}
+}
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 3422840..9f47fb2 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -133,7 +133,13 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 				return
 			}
 
-			wcx := webrtcx.New(s.config, &logger, sendCandidate(ctx, c, offer.Meta), recvCandidate(candidateChan[offer.Meta.TrackSource]))
+			wcx := webrtcx.New(
+				s.config,
+				&logger,
+				sendCandidate(ctx, c, offer.Meta),
+				recvCandidate(candidateChan[offer.Meta.TrackSource]),
+				webrtcx.NoopDeregisterSessionFunc,
+			)
 
 			var sdp webrtc.SessionDescription
 			if err := json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 2b2e5f5..27ed785 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -21,6 +21,10 @@ type SendCandidateFunc func(candidate *webrtc.ICECandidate) error
 // SendCandidateFunc receives a candidate from remote webRTC peer.
 type RecvCandidateFunc func() <-chan string
 
+// DeregisterSessionFunc deregisters a edge WebRTC session. Only used for publisher.
+// For subscriber, it should use NoopDeregisterSessionFunc instead.
+type DeregisterSessionFunc func()
+
 const (
 	rtcpPLIInterval = time.Second * 3
 )
@@ -35,18 +39,26 @@ type WebRTC struct {
 	pendingCandidates []*webrtc.ICECandidate
 	candidatesMux     sync.Mutex
 
-	sendCandidate SendCandidateFunc
-	recvCandidate RecvCandidateFunc
+	sendCandidate     SendCandidateFunc
+	recvCandidate     RecvCandidateFunc
+	deregisterSession DeregisterSessionFunc
 }
 
 // New returns a new WebRTC.
-func New(config cfg.WebRTCConfigOptions, logger *zerolog.Logger, sendCandidate SendCandidateFunc, recvCandidate RecvCandidateFunc) *WebRTC {
+func New(
+	config cfg.WebRTCConfigOptions,
+	logger *zerolog.Logger,
+	sendCandidate SendCandidateFunc,
+	recvCandidate RecvCandidateFunc,
+	deregisterSessionFunc DeregisterSessionFunc,
+) *WebRTC {
 	return &WebRTC{
-		logger:        *logger,
-		config:        config,
-		SignalChan:    make(chan *webrtc.SessionDescription, 1), // Make 1 buffer so SDP signaling never blocks
-		sendCandidate: sendCandidate,
-		recvCandidate: recvCandidate,
+		logger:            *logger,
+		config:            config,
+		SignalChan:        make(chan *webrtc.SessionDescription, 1), // Make 1 buffer so SDP signaling never blocks
+		sendCandidate:     sendCandidate,
+		recvCandidate:     recvCandidate,
+		deregisterSession: deregisterSessionFunc,
 	}
 }
 
@@ -77,11 +89,13 @@ func (w *WebRTC) CreatePublisher(videoTrack *webrtc.TrackLocalStaticRTP) error {
 		for {
 			i, _, readErr := t.Read(rtpBuf)
 			if readErr != nil {
-				w.logger.Panic().Err(err).Msg("could not read buffer")
+				w.logger.Err(err).Msg("could not read buffer")
+				return
 			}
 			// ErrClosedPipe means we don't have any subscribers, this is ok if no peers have connected yet
 			if _, err = videoTrack.Write(rtpBuf[:i]); err != nil && !errors.Is(err, io.ErrClosedPipe) {
-				w.logger.Panic().Err(err).Msg("could not write video track")
+				w.logger.Err(err).Msg("could not write video track")
+				return
 			}
 		}
 	})
@@ -146,6 +160,9 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 				w.logger.Panic().Err(err).Msg("could not close peer connection")
 			}
 			w.logger.Debug().Msg("peer connection has been closed")
+
+			// Deregister session after peer connection closed.
+			w.deregisterSession()
 		}
 	})
 
@@ -219,7 +236,8 @@ func (w *WebRTC) sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *we
 				MediaSSRC: uint32(remoteTrack.SSRC()),
 			},
 		}); rtcpSendErr != nil {
-			w.logger.Panic().Err(rtcpSendErr).Send()
+			w.logger.Err(rtcpSendErr).Send()
+			return
 		}
 	}
 }
@@ -252,3 +270,6 @@ func NoopRecvCandidateFunc() <-chan string {
 	close(ch)
 	return ch
 }
+
+// NoopDeregisterSessionFunc does nothing.
+func NoopDeregisterSessionFunc() {}

From 786a0bfef60a424309114f70289eb9660091af82 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 13 Apr 2021 15:16:22 +0800
Subject: [PATCH 28/52] Add loki grafana support

---
 docker/docker-compose.dev.yaml | 39 ++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index f44a779..9b0dce3 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -17,6 +17,12 @@ services:
       - mosquitto
       - broadcast
     restart: on-failure
+    logging:
+      driver: loki
+      options:
+        loki-url: "http://localhost:3100/api/prom/push"
+        loki-retries: "5"
+        loki-batch-size: "400"
 
   broadcast:
     image: ghcr.io/sb-im/skywalker:broadcast-latest
@@ -36,6 +42,12 @@ services:
     depends_on:
       - mosquitto
     restart: on-failure
+    logging:
+      driver: loki
+      options:
+        loki-url: "http://localhost:3100/api/prom/push"
+        loki-retries: "5"
+        loki-batch-size: "400"
 
   mosquitto:
     image: eclipse-mosquitto:2
@@ -43,3 +55,30 @@ services:
     volumes:
       - ../config/mosquitto.conf:/mosquitto/config/mosquitto.conf
     restart: on-failure
+    logging:
+      driver: loki
+      options:
+        loki-url: "http://localhost:3100/api/prom/push"
+        loki-retries: "5"
+        loki-batch-size: "400"
+
+  loki:
+    image: grafana/loki:latest
+    command: -config.file=/etc/loki/local-config.yaml
+    logging:
+      driver: loki
+      options:
+        loki-url: "http://localhost:3100/api/prom/push"
+        loki-retries: "5"
+        loki-batch-size: "400"
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    logging:
+      driver: loki
+      options:
+        loki-url: "http://localhost:3100/api/prom/push"
+        loki-retries: "5"
+        loki-batch-size: "400"

From 93b1980a1ec932027529994fc90a68102fe55cde Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 13 Apr 2021 18:52:25 +0800
Subject: [PATCH 29/52] Add push target for pushing docker image to github
 container registry

---
 Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Makefile b/Makefile
index 3a27e8f..985db2e 100644
--- a/Makefile
+++ b/Makefile
@@ -22,6 +22,10 @@ image:
 	-t $(IMAGE_BROADCAST) \
 	-f docker/Dockerfile.broadcast.dev .
 
+.PHONY: push
+push:
+	@docker push ${IMAGE_BROADCAST}
+
 # Note: '--env-file' value is relative to '-f' value's directory.
 .PHONY: up
 up: down image

From 81eb114aa896df990d1399dea87aec7876c44461 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 13 Apr 2021 23:50:17 +0800
Subject: [PATCH 30/52] Use one Dockerfile for all commands

---
 docker/{Dockerfile.broadcast => Dockerfile}         | 8 +++++---
 docker/{Dockerfile.broadcast.dev => Dockerfile.dev} | 4 ++--
 2 files changed, 7 insertions(+), 5 deletions(-)
 rename docker/{Dockerfile.broadcast => Dockerfile} (89%)
 rename docker/{Dockerfile.broadcast.dev => Dockerfile.dev} (89%)

diff --git a/docker/Dockerfile.broadcast b/docker/Dockerfile
similarity index 89%
rename from docker/Dockerfile.broadcast
rename to docker/Dockerfile
index 2f70195..c9699c4 100644
--- a/docker/Dockerfile.broadcast
+++ b/docker/Dockerfile
@@ -23,19 +23,21 @@ RUN --mount=type=ssh,id=github git config --global url."git@github.com:".instead
 
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=linux go build -tags timetzdata -ldflags="-w -s" -a -installsuffix cgo -o broadcast ./cmd
+RUN CGO_ENABLED=0 GOOS=linux go build -tags timetzdata -ldflags="-w -s" -a -installsuffix cgo -o skywalker ./cmd
 
 FROM scratch
 
 COPY --from=0 /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=0 /src/broadcast /
+COPY --from=0 /src/skywalker /
 
 ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
+VOLUME [ "/src/e2e" ]
+
 EXPOSE 8080
 
-ENTRYPOINT [ "/broadcast" ]
+ENTRYPOINT [ "/skywalker" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]
diff --git a/docker/Dockerfile.broadcast.dev b/docker/Dockerfile.dev
similarity index 89%
rename from docker/Dockerfile.broadcast.dev
rename to docker/Dockerfile.dev
index 83faa98..0cdab2c 100644
--- a/docker/Dockerfile.broadcast.dev
+++ b/docker/Dockerfile.dev
@@ -17,12 +17,12 @@ RUN --mount=type=ssh,id=github git config --global url."git@github.com:".instead
 
 COPY . .
 
-RUN GOOS=linux go build -race -tags timetzdata -o broadcast ./cmd
+RUN GOOS=linux go build -race -tags timetzdata -o skywalker ./cmd
 
 ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
-ENTRYPOINT [ "/src/broadcast" ]
+ENTRYPOINT [ "/src/skywalker" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]

From f04e89456e491a775dc2990f9d38afd0aabbb3aa Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 13 Apr 2021 23:54:53 +0800
Subject: [PATCH 31/52] Update Makefile

---
 .gitignore |  1 +
 Makefile   | 15 +++++++++++----
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index d98f1fa..6f28c4d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 config/config.dev.yaml
 .env
 config/config.docker.yaml
+skywalker
diff --git a/Makefile b/Makefile
index 985db2e..e678e53 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,6 @@
+# Project name.
+PROJECT_NAME:=skywalker
+
 # SSH private key set up.
 CURRENT_USER?=william
 PRIVATE_KEY_FILE?=id_ed25519
@@ -5,12 +8,16 @@ PRIVATE_KEY_PATH?=github=/home/$(CURRENT_USER)/.ssh/$(PRIVATE_KEY_FILE)
 PROJECT_DIR?=/home/${CURRENT_USER}/go/src/github.com/SB-IM/skywalker
 
 # Project image repo.
-IMAGE_BROADCAST?=ghcr.io/sb-im/skywalker:broadcast-latest
+IMAGE?=ghcr.io/sb-im/skywalker:latest
 
 .PHONY: run
 run:
 	@DEBUG_MQTT_CLIENT=false go run -race ./cmd --debug broadcast -c config/config.dev.yaml
 
+.PHONY: build
+build:
+	@go build -race -o ${PROJECT_NAME} ./cmd
+
 .PHONY: lint
 lint:
 	@golangci-lint run ./...
@@ -19,12 +26,12 @@ lint:
 image:
 	@docker build \
 	--ssh $(PRIVATE_KEY_PATH) \
-	-t $(IMAGE_BROADCAST) \
-	-f docker/Dockerfile.broadcast.dev .
+	-t $(IMAGE) \
+	-f docker/Dockerfile.dev .
 
 .PHONY: push
 push:
-	@docker push ${IMAGE_BROADCAST}
+	@docker push ${IMAGE}
 
 # Note: '--env-file' value is relative to '-f' value's directory.
 .PHONY: up

From 97678d07533d3306e882715fbc2a80af170922a9 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 00:00:13 +0800
Subject: [PATCH 32/52] Use one Dockerfile for all commands

---
 docker/docker-compose.dev.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 9b0dce3..e0d6585 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   livestream:
-    image: ghcr.io/sb-im/sphinx:livestream-amd64-latest
+    image: ghcr.io/sb-im/sphinx:amd64-latest
     container_name: livestream
     command:
       - --debug
@@ -25,7 +25,7 @@ services:
         loki-batch-size: "400"
 
   broadcast:
-    image: ghcr.io/sb-im/skywalker:broadcast-latest
+    image: ghcr.io/sb-im/skywalker:latest
     container_name: broadcast
     command:
       - --debug

From 83b2e982d0a2dc511fe2b58977f8943950bee573 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 5 Apr 2021 18:03:43 +0800
Subject: [PATCH 33/52] Update action

---
 .github/workflows/build-broadcast.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-broadcast.yaml b/.github/workflows/build-broadcast.yaml
index 64898bb..0b17966 100644
--- a/.github/workflows/build-broadcast.yaml
+++ b/.github/workflows/build-broadcast.yaml
@@ -17,7 +17,7 @@ jobs:
         id: meta
         uses: crazy-max/ghaction-docker-meta@v2
         with:
-          images: ghcr.io/sb-im/skywalker
+          images: ghcr.io/sb-im/sphinx
           tags: |
             type=edge,prefix=broadcast-
             type=ref,prefix=broadcast-,event=branch

From 32f212d0a3a05ac72ab53f0b129e867ad55b0a5e Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 5 Apr 2021 18:05:48 +0800
Subject: [PATCH 34/52] Update action

---
 .github/workflows/build-broadcast.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-broadcast.yaml b/.github/workflows/build-broadcast.yaml
index 0b17966..64898bb 100644
--- a/.github/workflows/build-broadcast.yaml
+++ b/.github/workflows/build-broadcast.yaml
@@ -17,7 +17,7 @@ jobs:
         id: meta
         uses: crazy-max/ghaction-docker-meta@v2
         with:
-          images: ghcr.io/sb-im/sphinx
+          images: ghcr.io/sb-im/skywalker
           tags: |
             type=edge,prefix=broadcast-
             type=ref,prefix=broadcast-,event=branch

From b8ac7e731d6856d8981323fd8fe9b7edd15b960e Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 10:39:12 +0800
Subject: [PATCH 35/52] Use one Dockerfile for all commands

---
 .../{build-broadcast.yaml => build.yaml}       | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)
 rename .github/workflows/{build-broadcast.yaml => build.yaml} (76%)

diff --git a/.github/workflows/build-broadcast.yaml b/.github/workflows/build.yaml
similarity index 76%
rename from .github/workflows/build-broadcast.yaml
rename to .github/workflows/build.yaml
index 64898bb..c08e787 100644
--- a/.github/workflows/build-broadcast.yaml
+++ b/.github/workflows/build.yaml
@@ -1,4 +1,4 @@
-name: Build broadcast image
+name: Build container image
 on:
   workflow_dispatch:
   push:
@@ -8,7 +8,7 @@ on:
       - "v*"
 jobs:
   build:
-    name: Build broadcast image
+    name: Build container image
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -19,15 +19,13 @@ jobs:
         with:
           images: ghcr.io/sb-im/skywalker
           tags: |
-            type=edge,prefix=broadcast-
-            type=ref,prefix=broadcast-,event=branch
-            type=ref,prefix=broadcast-,event=pr
-            type=semver,prefix=broadcast-,pattern={{version}}
-            type=semver,prefix=broadcast-,pattern={{major}}.{{minor}}
-            type=sha,prefix=broadcast-
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha
           flavor: |
             latest=true
-            prefix=broadcast-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
@@ -51,7 +49,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           ssh: github=ssh-key
-          file: docker/Dockerfile.broadcast
+          file: docker/Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

From 0ce551d85f6685251d1f7bb7657068feb0c2e00a Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 10:58:03 +0800
Subject: [PATCH 36/52] docker: minor update

---
 docker/Dockerfile     | 2 --
 docker/Dockerfile.dev | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/Dockerfile b/docker/Dockerfile
index c9699c4..77af1ef 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -34,8 +34,6 @@ ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
-VOLUME [ "/src/e2e" ]
-
 EXPOSE 8080
 
 ENTRYPOINT [ "/skywalker" ]
diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev
index 0cdab2c..1b61cce 100644
--- a/docker/Dockerfile.dev
+++ b/docker/Dockerfile.dev
@@ -23,6 +23,8 @@ ENV DEBUG_MQTT_CLIENT=false
 
 VOLUME [ "/config" ]
 
+VOLUME [ "/src/e2e" ]
+
 ENTRYPOINT [ "/src/skywalker" ]
 
 CMD [ "--debug", "broadcast", "-c", "/config/config.yaml" ]

From 9c23cea97c3a66303c3f1e5fe930e696030db0f9 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 12:16:08 +0800
Subject: [PATCH 37/52] docker: minor update

---
 docker/docker-compose.dev.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index e0d6585..a89519d 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   livestream:
-    image: ghcr.io/sb-im/sphinx:amd64-latest
+    image: ghcr.io/sb-im/sphinx:latest-amd64
     container_name: livestream
     command:
       - --debug
@@ -11,8 +11,8 @@ services:
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
     volumes:
-      - ../../sphinx/config/config.docker.yaml:/config/config.yaml
-      - /etc/machine-id:/etc/machine-id
+      - ../../sphinx/config/config.docker.yaml:/config/config.yaml:ro
+      - /etc/machine-id:/etc/machine-id:ro
     depends_on:
       - mosquitto
       - broadcast
@@ -38,7 +38,7 @@ services:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
       - DEBUG=true
     volumes:
-      - ../config/config.docker.yaml:/config/config.yaml
+      - ../config/config.docker.yaml:/config/config.yaml:ro
     depends_on:
       - mosquitto
     restart: on-failure
@@ -53,7 +53,7 @@ services:
     image: eclipse-mosquitto:2
     container_name: mosquitto
     volumes:
-      - ../config/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - ../config/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
     restart: on-failure
     logging:
       driver: loki

From ac7287827266650970b8ba7949ab3f61aa042101 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 13:02:52 +0800
Subject: [PATCH 38/52] docker: fix loki port

---
 docker/docker-compose.dev.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index a89519d..a9683e1 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -64,6 +64,8 @@ services:
 
   loki:
     image: grafana/loki:latest
+    ports:
+      - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
     logging:
       driver: loki

From cbe98cf8c5895e3ecd3c5fd192fce5a11c30f049 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 13:07:43 +0800
Subject: [PATCH 39/52] docker: reuse logging block

---
 docker/docker-compose.dev.yaml | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index a9683e1..01bc7b8 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -17,7 +17,7 @@ services:
       - mosquitto
       - broadcast
     restart: on-failure
-    logging:
+    logging: &loki
       driver: loki
       options:
         loki-url: "http://localhost:3100/api/prom/push"
@@ -42,12 +42,7 @@ services:
     depends_on:
       - mosquitto
     restart: on-failure
-    logging:
-      driver: loki
-      options:
-        loki-url: "http://localhost:3100/api/prom/push"
-        loki-retries: "5"
-        loki-batch-size: "400"
+    logging: *loki
 
   mosquitto:
     image: eclipse-mosquitto:2
@@ -55,32 +50,17 @@ services:
     volumes:
       - ../config/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro
     restart: on-failure
-    logging:
-      driver: loki
-      options:
-        loki-url: "http://localhost:3100/api/prom/push"
-        loki-retries: "5"
-        loki-batch-size: "400"
+    logging: *loki
 
   loki:
     image: grafana/loki:latest
     ports:
       - "3100:3100"
     command: -config.file=/etc/loki/local-config.yaml
-    logging:
-      driver: loki
-      options:
-        loki-url: "http://localhost:3100/api/prom/push"
-        loki-retries: "5"
-        loki-batch-size: "400"
+    logging: *loki
 
   grafana:
     image: grafana/grafana:latest
     ports:
       - "3000:3000"
-    logging:
-      driver: loki
-      options:
-        loki-url: "http://localhost:3100/api/prom/push"
-        loki-retries: "5"
-        loki-batch-size: "400"
+    logging: *loki

From 69d90b56b7519fc1d659d4d9ef99cfc53962cc2c Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 16:53:08 +0800
Subject: [PATCH 40/52] chore: Update dependencies

---
 go.mod |  4 ++--
 go.sum | 15 +++++++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 210bed6..7f16804 100644
--- a/go.mod
+++ b/go.mod
@@ -4,13 +4,13 @@ go 1.16
 
 require (
 	github.com/SB-IM/logging v0.2.2
-	github.com/SB-IM/mqtt-client v0.1.0
+	github.com/SB-IM/mqtt-client v0.1.1
 	github.com/SB-IM/pb v0.2.9
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/pion/rtcp v1.2.6
-	github.com/pion/webrtc/v3 v3.0.20
+	github.com/pion/webrtc/v3 v3.0.21
 	github.com/rs/zerolog v1.21.0
 	github.com/urfave/cli/v2 v2.3.0
 	google.golang.org/protobuf v1.26.0
diff --git a/go.sum b/go.sum
index 47e3596..389f297 100644
--- a/go.sum
+++ b/go.sum
@@ -5,6 +5,8 @@ github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
 github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
 github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXCdI=
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
+github.com/SB-IM/mqtt-client v0.1.1 h1:2GHv07Pfi90M1zGH8ZHu8/wfnntWC3mnKxCetrjqq9Y=
+github.com/SB-IM/mqtt-client v0.1.1/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
 github.com/SB-IM/pb v0.2.4 h1:V1FqyLVTC3AAlJzUj5JkXsZ34/MAHc74N42W0qO9Y+c=
 github.com/SB-IM/pb v0.2.4/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
 github.com/SB-IM/pb v0.2.5 h1:cqtTTTnxnhENJLpK3OI6KVNze1giDzSfVeRhTFtzSCg=
@@ -97,6 +99,8 @@ github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXm
 github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
 github.com/pion/dtls/v2 v2.0.8 h1:reGe8rNIMfO/UAeFLqO61tl64t154Qfkr4U3Gzu1tsg=
 github.com/pion/dtls/v2 v2.0.8/go.mod h1:QuDII+8FVvk9Dp5t5vYIMTo7hh7uBkra+8QIm7QGm10=
+github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8=
+github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
 github.com/pion/ice/v2 v2.0.16 h1:K6bzD8ef9vMKbGMTHaUweHXEyuNGnvr2zdqKoLKZPn0=
 github.com/pion/ice/v2 v2.0.16/go.mod h1:SJNJzC27gDZoOW0UoxIoC8Hf2PDxG28hQyNdSexDu38=
 github.com/pion/interceptor v0.0.12 h1:eC1iVneBIAQJEfaNAfDqAncJWhMDAnaXPRCJsltdokE=
@@ -129,8 +133,12 @@ github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
 github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
 github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
 github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
+github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
+github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
 github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw=
 github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M=
+github.com/pion/webrtc/v3 v3.0.21 h1:AbWcIdpMjk95qnVmWzgj561yfGm1LHSvJdfUvvuJdMM=
+github.com/pion/webrtc/v3 v3.0.21/go.mod h1:hYG8kKExnrs8KJHWE6ljDUD8v8zXdfIdOtnJzJXJsrg=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -164,6 +172,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -176,8 +186,11 @@ golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c h1:KHUzaHIpjWVlVVNh65G3hhuj3KB1HnjY6Cq5cTvRQT8=
+golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -196,6 +209,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

From 84cbcaef5bbafa8b41da351cff02121b0b274683 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Wed, 14 Apr 2021 17:40:49 +0800
Subject: [PATCH 41/52] Remove unneeded environment variable

---
 docker/docker-compose.dev.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 01bc7b8..385d16d 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -36,7 +36,6 @@ services:
       - "8080:8080"
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
-      - DEBUG=true
     volumes:
       - ../config/config.docker.yaml:/config/config.yaml:ro
     depends_on:

From 77dff853279980674b3c4aa99275a20862d821bf Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 00:00:09 +0800
Subject: [PATCH 42/52] docker: change image tag

---
 Makefile                       | 10 +++++-----
 docker/docker-compose.dev.yaml |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index e678e53..52b9614 100644
--- a/Makefile
+++ b/Makefile
@@ -5,10 +5,10 @@ PROJECT_NAME:=skywalker
 CURRENT_USER?=william
 PRIVATE_KEY_FILE?=id_ed25519
 PRIVATE_KEY_PATH?=github=/home/$(CURRENT_USER)/.ssh/$(PRIVATE_KEY_FILE)
-PROJECT_DIR?=/home/${CURRENT_USER}/go/src/github.com/SB-IM/skywalker
+PROJECT_DIR?=/home/$(CURRENT_USER)/go/src/github.com/SB-IM/skywalker
 
 # Project image repo.
-IMAGE?=ghcr.io/sb-im/skywalker:latest
+IMAGE?=ghcr.io/sb-im/skywalker:latest-dev
 
 .PHONY: run
 run:
@@ -16,7 +16,7 @@ run:
 
 .PHONY: build
 build:
-	@go build -race -o ${PROJECT_NAME} ./cmd
+	@go build -race -o $(PROJECT_NAME) ./cmd
 
 .PHONY: lint
 lint:
@@ -31,7 +31,7 @@ image:
 
 .PHONY: push
 push:
-	@docker push ${IMAGE}
+	@docker push $(IMAGE)
 
 # Note: '--env-file' value is relative to '-f' value's directory.
 .PHONY: up
@@ -44,7 +44,7 @@ down:
 
 .PHONY: broker
 broker:
-	@docker run -d --rm --name mosquitto -p 1883:1883 -p 9001:9001 -v ${PROJECT_DIR}/config/mosquitto.conf:/mosquitto/config/mosquitto.conf eclipse-mosquitto:2
+	@docker run -d --rm --name mosquitto -p 1883:1883 -p 9001:9001 -v $(PROJECT_DIR)/config/mosquitto.conf:/mosquitto/config/mosquitto.conf eclipse-mosquitto:2
 
 .PHONY: stop-broker
 stop-broker:
diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 385d16d..7969f81 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   livestream:
-    image: ghcr.io/sb-im/sphinx:latest-amd64
+    image: ghcr.io/sb-im/sphinx:latest-dev-amd64
     container_name: livestream
     command:
       - --debug
@@ -25,7 +25,7 @@ services:
         loki-batch-size: "400"
 
   broadcast:
-    image: ghcr.io/sb-im/skywalker:latest
+    image: ghcr.io/sb-im/skywalker:latest-dev
     container_name: broadcast
     command:
       - --debug

From d79cf3295e231a91ef83b06a28fa4488722702bd Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 14:23:43 +0800
Subject: [PATCH 43/52] chore: Update dependencies

---
 e2e/broadcast/static/index.html | 10 +++++++++-
 go.mod                          |  2 +-
 go.sum                          |  2 ++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/e2e/broadcast/static/index.html b/e2e/broadcast/static/index.html
index ef37d92..3c42cab 100644
--- a/e2e/broadcast/static/index.html
+++ b/e2e/broadcast/static/index.html
@@ -18,7 +18,15 @@
 
     let answered = false
     let candidates = []
-    let pc = new RTCPeerConnection()
+    let pc = new RTCPeerConnection({
+        iceServers: [
+            {
+                urls: "turn:sbnet.xyz:4478",
+                username: "test",
+                credential: "test"
+            }
+        ]
+    })
     pc.ontrack = function (event) {
         var el = document.createElement(event.track.kind)
         el.srcObject = event.streams[0]
diff --git a/go.mod b/go.mod
index 7f16804..9dac9cc 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/SB-IM/logging v0.2.2
-	github.com/SB-IM/mqtt-client v0.1.1
+	github.com/SB-IM/mqtt-client v0.1.2
 	github.com/SB-IM/pb v0.2.9
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
diff --git a/go.sum b/go.sum
index 389f297..d9b7a6a 100644
--- a/go.sum
+++ b/go.sum
@@ -7,6 +7,8 @@ github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXC
 github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
 github.com/SB-IM/mqtt-client v0.1.1 h1:2GHv07Pfi90M1zGH8ZHu8/wfnntWC3mnKxCetrjqq9Y=
 github.com/SB-IM/mqtt-client v0.1.1/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
+github.com/SB-IM/mqtt-client v0.1.2 h1:mdrcmuzmcLJXLl9gi3bmNzGJ+KHfp2zqWKhaqJeN//8=
+github.com/SB-IM/mqtt-client v0.1.2/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
 github.com/SB-IM/pb v0.2.4 h1:V1FqyLVTC3AAlJzUj5JkXsZ34/MAHc74N42W0qO9Y+c=
 github.com/SB-IM/pb v0.2.4/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
 github.com/SB-IM/pb v0.2.5 h1:cqtTTTnxnhENJLpK3OI6KVNze1giDzSfVeRhTFtzSCg=

From 1d56a3082f73649ff6f3459274004f832b35ca2c Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 14:37:59 +0800
Subject: [PATCH 44/52] Subscribe candidate topics before consumption

---
 internal/broadcast/webrtc/webrtc.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 27ed785..607310b 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -132,6 +132,7 @@ func (w *WebRTC) CreateSubscriber(videoTrack *webrtc.TrackLocalStaticRTP) error
 
 func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) error {
 	offer := <-w.SignalChan
+	candidateChan := w.recvCandidate()
 
 	peerConnection.OnICECandidate(func(c *webrtc.ICECandidate) {
 		if c == nil {
@@ -171,7 +172,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 	}
 
 	// Signal candidate after setting remote description.
-	go w.signalCandidate(peerConnection)
+	go w.signalCandidate(peerConnection, candidateChan)
 
 	answer, err := peerConnection.CreateAnswer(nil)
 	if err != nil {
@@ -212,10 +213,9 @@ func (w *WebRTC) newPeerConnection() (*webrtc.PeerConnection, error) {
 	})
 }
 
-func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection) {
+func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection, ch <-chan string) {
 	// TODO: Stop adding ICE candidate when after signaling succeeded, that is, to exit the loop.
 	// Just set a timer is not enough.
-	ch := w.recvCandidate()
 	for c := range ch {
 		if err := peerConnection.AddICECandidate(webrtc.ICECandidateInit{
 			Candidate: c,

From 9581dd05f7cfada8c5f1f7c03b33c5bc6559e3bb Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 16:50:30 +0800
Subject: [PATCH 45/52] Remove deregister session(#8)

---
 internal/broadcast/publisher/publisher.go   | 33 +++++++---------
 internal/broadcast/subscriber/subscriber.go |  2 +-
 internal/broadcast/webrtc/webrtc.go         | 42 +++++++++++----------
 3 files changed, 38 insertions(+), 39 deletions(-)

diff --git a/internal/broadcast/publisher/publisher.go b/internal/broadcast/publisher/publisher.go
index ec7b245..836dc4c 100644
--- a/internal/broadcast/publisher/publisher.go
+++ b/internal/broadcast/publisher/publisher.go
@@ -130,7 +130,7 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 			Logger()
 		logger.Debug().Msg("received offer from edge")
 
-		answer, videoTrack, err := p.signalPeerConnection(&offer, &logger)
+		answer, err := p.signalPeerConnection(&offer, &logger)
 		if err != nil {
 			logger.Err(err).Msg("failed to signal peer connection")
 			return
@@ -152,26 +152,22 @@ func (p *Publisher) handleMessage() mqtt.MessageHandler {
 			return
 		}
 		logger.Debug().Str("answer_topic", answerTopic).Msg("sent answer to edge")
-
-		// Register session on signaling success.
-		p.registerSession(offer.Meta, videoTrack)
 	}
 }
 
 // signalPeerConnection creates video track and performs webRTC signaling.
 func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *zerolog.Logger) (
 	*webrtc.SessionDescription,
-	*webrtc.TrackLocalStaticRTP,
 	error,
 ) {
 	var sdp webrtc.SessionDescription
 	if err := json.Unmarshal([]byte(offer.Sdp), &sdp); err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	videoTrack, err := webrtcx.CreateLocalTrack()
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not create webRTC local video track: %w", err)
+		return nil, fmt.Errorf("could not create webRTC local video track: %w", err)
 	}
 	logger.Debug().Msg("created video track")
 
@@ -180,32 +176,31 @@ func (p *Publisher) signalPeerConnection(offer *pb.SessionDescription, logger *z
 		logger,
 		p.sendCandidate(offer.Meta),
 		p.recvCandidate(offer.Meta),
-		p.deregisterSession(offer.Meta),
+		p.registerSession(offer.Meta, videoTrack),
 	)
 
 	// TODO: handle blocking case with timeout for channels.
 	w.SignalChan <- &sdp
 	if err := w.CreatePublisher(videoTrack); err != nil {
-		return nil, nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
+		return nil, fmt.Errorf("failed to create webRTC publisher: %w", err)
 	}
 	logger.Debug().Msg("created publisher")
 
-	return <-w.SignalChan, videoTrack, nil
+	return <-w.SignalChan, nil
 }
 
 func (p *Publisher) registerSession(
 	meta *pb.Meta,
 	videoTrack *webrtc.TrackLocalStaticRTP,
-) {
-	sessionID := meta.Id + strconv.Itoa(int(meta.TrackSource))
-	p.sessions.Store(sessionID, videoTrack)
-	p.logger.Debug().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("registered session")
-}
-
-func (p *Publisher) deregisterSession(meta *pb.Meta) webrtcx.DeregisterSessionFunc {
+) webrtcx.RegisterSessionFunc {
 	return func() {
 		sessionID := meta.Id + strconv.Itoa(int(meta.TrackSource))
-		p.sessions.Delete(sessionID)
-		p.logger.Debug().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("deregistered session")
+		_, ok := p.sessions.Load(sessionID)
+		p.sessions.Store(sessionID, videoTrack)
+		if ok {
+			p.logger.Info().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("re-registered old session")
+		} else {
+			p.logger.Info().Str("key", sessionID).Int32("value", int32(meta.TrackSource)).Msg("registered session")
+		}
 	}
 }
diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 9f47fb2..72de495 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -138,7 +138,7 @@ func (s *Subscriber) processMessage(ctx context.Context, c *websocket.Conn) {
 				&logger,
 				sendCandidate(ctx, c, offer.Meta),
 				recvCandidate(candidateChan[offer.Meta.TrackSource]),
-				webrtcx.NoopDeregisterSessionFunc,
+				webrtcx.NoopRegisterSessionFunc,
 			)
 
 			var sdp webrtc.SessionDescription
diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 607310b..a22897e 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -21,9 +21,9 @@ type SendCandidateFunc func(candidate *webrtc.ICECandidate) error
 // SendCandidateFunc receives a candidate from remote webRTC peer.
 type RecvCandidateFunc func() <-chan string
 
-// DeregisterSessionFunc deregisters a edge WebRTC session. Only used for publisher.
-// For subscriber, it should use NoopDeregisterSessionFunc instead.
-type DeregisterSessionFunc func()
+// RegisterSessionFunc registers a edge WebRTC session. Only used for publisher.
+// For subscriber, it should use NoopRegisterSessionFunc instead.
+type RegisterSessionFunc func()
 
 const (
 	rtcpPLIInterval = time.Second * 3
@@ -39,9 +39,10 @@ type WebRTC struct {
 	pendingCandidates []*webrtc.ICECandidate
 	candidatesMux     sync.Mutex
 
-	sendCandidate     SendCandidateFunc
-	recvCandidate     RecvCandidateFunc
-	deregisterSession DeregisterSessionFunc
+	sendCandidate SendCandidateFunc
+	recvCandidate RecvCandidateFunc
+
+	registerSession RegisterSessionFunc
 }
 
 // New returns a new WebRTC.
@@ -50,15 +51,15 @@ func New(
 	logger *zerolog.Logger,
 	sendCandidate SendCandidateFunc,
 	recvCandidate RecvCandidateFunc,
-	deregisterSessionFunc DeregisterSessionFunc,
+	registerSession RegisterSessionFunc,
 ) *WebRTC {
 	return &WebRTC{
-		logger:            *logger,
-		config:            config,
-		SignalChan:        make(chan *webrtc.SessionDescription, 1), // Make 1 buffer so SDP signaling never blocks
-		sendCandidate:     sendCandidate,
-		recvCandidate:     recvCandidate,
-		deregisterSession: deregisterSessionFunc,
+		logger:          *logger,
+		config:          config,
+		SignalChan:      make(chan *webrtc.SessionDescription, 1), // Make 1 buffer so SDP signaling never blocks
+		sendCandidate:   sendCandidate,
+		recvCandidate:   recvCandidate,
+		registerSession: registerSession,
 	}
 }
 
@@ -156,14 +157,17 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 	// This will notify you when the peer has connected/disconnected
 	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
 		w.logger.Debug().Str("state", connectionState.String()).Msg("ICE connection state has changed")
-		if connectionState == webrtc.ICEConnectionStateFailed {
+
+		switch connectionState {
+		case webrtc.ICEConnectionStateFailed:
 			if err := peerConnection.Close(); err != nil {
 				w.logger.Panic().Err(err).Msg("could not close peer connection")
 			}
 			w.logger.Debug().Msg("peer connection has been closed")
-
-			// Deregister session after peer connection closed.
-			w.deregisterSession()
+		case webrtc.ICEConnectionStateConnected:
+			// Register session after ICE state is connected.
+			w.registerSession()
+		default:
 		}
 	})
 
@@ -271,5 +275,5 @@ func NoopRecvCandidateFunc() <-chan string {
 	return ch
 }
 
-// NoopDeregisterSessionFunc does nothing.
-func NoopDeregisterSessionFunc() {}
+// NoopRegisterSessionFunc does nothing.
+func NoopRegisterSessionFunc() {}

From 6497b1280aa6c4b1bfe0df8f5d364af15c78f749 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 16:54:29 +0800
Subject: [PATCH 46/52] Expose livestream RTP port

---
 docker/docker-compose.dev.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/docker-compose.dev.yaml b/docker/docker-compose.dev.yaml
index 7969f81..67eab2e 100644
--- a/docker/docker-compose.dev.yaml
+++ b/docker/docker-compose.dev.yaml
@@ -8,6 +8,8 @@ services:
       - livestream
       - -c
       - /config/config.yaml
+    ports:
+      - "5004:5004/udp"
     environment:
       - DEBUG_MQTT_CLIENT="${DEBUG_MQTT_CLIENT:-true}"
     volumes:

From 361c7a5b7c9479eeef6954baffe4340069f84dbd Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 15 Apr 2021 17:58:01 +0800
Subject: [PATCH 47/52] WebSocket CORS enabled

---
 internal/broadcast/subscriber/subscriber.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/broadcast/subscriber/subscriber.go b/internal/broadcast/subscriber/subscriber.go
index 72de495..3b3f024 100644
--- a/internal/broadcast/subscriber/subscriber.go
+++ b/internal/broadcast/subscriber/subscriber.go
@@ -71,7 +71,7 @@ func (s *Subscriber) Signal() http.Handler {
 func (s *Subscriber) handleSignal() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		c, err := websocket.Accept(w, r, &websocket.AcceptOptions{
-			OriginPatterns: []string{"InsecureSkipVerify"},
+			OriginPatterns: []string{"*"}, // TODO: Must remove this option on production environment.
 		})
 		if err != nil {
 			s.logger.Err(err).Msg("could not upgrade to webSocket connection")

From 3f8c9daf494938aa336d750255194aa1a278e4ab Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Fri, 16 Apr 2021 16:25:34 +0800
Subject: [PATCH 48/52] Fix some typo

---
 internal/broadcast/webrtc/webrtc.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index a22897e..9ff2066 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -18,7 +18,7 @@ import (
 // SendCandidateFunc sends a candidate to remote webRTC peer.
 type SendCandidateFunc func(candidate *webrtc.ICECandidate) error
 
-// SendCandidateFunc receives a candidate from remote webRTC peer.
+// RecvCandidateFunc receives a candidate from remote webRTC peer.
 type RecvCandidateFunc func() <-chan string
 
 // RegisterSessionFunc registers a edge WebRTC session. Only used for publisher.
@@ -199,7 +199,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 		if err := w.sendCandidate(c); err != nil {
 			return fmt.Errorf("could not send candidate: %w", err)
 		}
-		w.logger.Debug().Msg("sent an ICEcandidate")
+		w.logger.Debug().Msg("sent an ICE candidate")
 	}
 
 	return nil

From a384a676e808059cc4b5f4e8ce3bea5d49634a12 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 19 Apr 2021 11:41:42 +0800
Subject: [PATCH 49/52] Add candidate debug log

---
 internal/broadcast/webrtc/webrtc.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 9ff2066..6e404f3 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -226,7 +226,7 @@ func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection, ch <-cha
 		}); err != nil {
 			w.logger.Err(err).Msg("could not add ICE candidate")
 		}
-		w.logger.Debug().Msg("successfully added an ICE candidate")
+		w.logger.Debug().Str("candidate", c).Msg("successfully added an ICE candidate")
 	}
 }
 

From efd06fe092ddb47463495bff9952a3ac3e466517 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Mon, 19 Apr 2021 16:49:23 +0800
Subject: [PATCH 50/52] chore: Update dependencies

---
 go.mod |  4 ++--
 go.sum | 67 +++++++++++++++++++---------------------------------------
 2 files changed, 24 insertions(+), 47 deletions(-)

diff --git a/go.mod b/go.mod
index 9dac9cc..1bb537d 100644
--- a/go.mod
+++ b/go.mod
@@ -5,12 +5,12 @@ go 1.16
 require (
 	github.com/SB-IM/logging v0.2.2
 	github.com/SB-IM/mqtt-client v0.1.2
-	github.com/SB-IM/pb v0.2.9
+	github.com/SB-IM/pb v0.3.1
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
 	github.com/gorilla/mux v1.8.0
 	github.com/pion/rtcp v1.2.6
-	github.com/pion/webrtc/v3 v3.0.21
+	github.com/pion/webrtc/v3 v3.0.22
 	github.com/rs/zerolog v1.21.0
 	github.com/urfave/cli/v2 v2.3.0
 	google.golang.org/protobuf v1.26.0
diff --git a/go.sum b/go.sum
index d9b7a6a..7154552 100644
--- a/go.sum
+++ b/go.sum
@@ -3,22 +3,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/SB-IM/logging v0.1.0/go.mod h1:VtWxAA+Qto3I28BcaS8eRgdxkOjT9ROwBqlSwL9n6Fw=
 github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
 github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
-github.com/SB-IM/mqtt-client v0.1.0 h1:3IAAD2G+ty4B8VrTckLOBLBx0Q405H/EOH83wvSXCdI=
-github.com/SB-IM/mqtt-client v0.1.0/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
-github.com/SB-IM/mqtt-client v0.1.1 h1:2GHv07Pfi90M1zGH8ZHu8/wfnntWC3mnKxCetrjqq9Y=
-github.com/SB-IM/mqtt-client v0.1.1/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
 github.com/SB-IM/mqtt-client v0.1.2 h1:mdrcmuzmcLJXLl9gi3bmNzGJ+KHfp2zqWKhaqJeN//8=
 github.com/SB-IM/mqtt-client v0.1.2/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
-github.com/SB-IM/pb v0.2.4 h1:V1FqyLVTC3AAlJzUj5JkXsZ34/MAHc74N42W0qO9Y+c=
-github.com/SB-IM/pb v0.2.4/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
-github.com/SB-IM/pb v0.2.5 h1:cqtTTTnxnhENJLpK3OI6KVNze1giDzSfVeRhTFtzSCg=
-github.com/SB-IM/pb v0.2.5/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
-github.com/SB-IM/pb v0.2.7 h1:vnn3nyuxsyme/5F8J1y0xoLhcaNomVFwoFPripKcd2Y=
-github.com/SB-IM/pb v0.2.7/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
-github.com/SB-IM/pb v0.2.8 h1:qRwtdWBKgF4wObV2fJ9UCIi2mKstFDg3LS8BjCKwg2Q=
-github.com/SB-IM/pb v0.2.8/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
-github.com/SB-IM/pb v0.2.9 h1:vuLd9cj5+4mG5990qySIQXg4RMd4c/FWOfAEckVsmwM=
-github.com/SB-IM/pb v0.2.9/go.mod h1:m33Pz96XbuiWx5h/59WRJ52FjzgOofCCIkVP5OFZri0=
+github.com/SB-IM/pb v0.3.1 h1:CgSdrOi8kiX15J7Wx+n/jcnyi2gu9fx7ySWt8QrQ1G4=
+github.com/SB-IM/pb v0.3.1/go.mod h1:/XrV0hXCaaOQSDhP6msTH8MwvhdC1RKQSsRKDkeB2FE=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -41,6 +29,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
 github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
 github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
@@ -56,9 +45,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -91,20 +80,19 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/pion/datachannel v1.4.21 h1:3ZvhNyfmxsAqltQrApLPQMhSFNA+aT87RqyCq4OXmf0=
 github.com/pion/datachannel v1.4.21/go.mod h1:oiNyP4gHx2DIwRzX/MFyH0Rz/Gz05OgBlayAI2hAWjg=
-github.com/pion/dtls/v2 v2.0.8 h1:reGe8rNIMfO/UAeFLqO61tl64t154Qfkr4U3Gzu1tsg=
-github.com/pion/dtls/v2 v2.0.8/go.mod h1:QuDII+8FVvk9Dp5t5vYIMTo7hh7uBkra+8QIm7QGm10=
 github.com/pion/dtls/v2 v2.0.9 h1:7Ow+V++YSZQMYzggI0P9vLJz/hUFcffsfGMfT/Qy+u8=
 github.com/pion/dtls/v2 v2.0.9/go.mod h1:O0Wr7si/Zj5/EBFlDzDd6UtVxx25CE1r7XM7BQKYQho=
-github.com/pion/ice/v2 v2.0.16 h1:K6bzD8ef9vMKbGMTHaUweHXEyuNGnvr2zdqKoLKZPn0=
-github.com/pion/ice/v2 v2.0.16/go.mod h1:SJNJzC27gDZoOW0UoxIoC8Hf2PDxG28hQyNdSexDu38=
+github.com/pion/ice/v2 v2.1.2 h1:hnZrg7jZyclNdfrIOyU/MGmIftXxqEP0hCrRXt6r6lY=
+github.com/pion/ice/v2 v2.1.2/go.mod h1:kV4EODVD5ux2z8XncbLHIOtcXKtYXVgLVCeVqnpoeP0=
 github.com/pion/interceptor v0.0.12 h1:eC1iVneBIAQJEfaNAfDqAncJWhMDAnaXPRCJsltdokE=
 github.com/pion/interceptor v0.0.12/go.mod h1:qzeuWuD/ZXvPqOnxNcnhWfkCZ2e1kwwslicyyPnhoK4=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
@@ -126,21 +114,16 @@ github.com/pion/srtp/v2 v2.0.2 h1:664iGzVmaY7KYS5M0gleY0DscRo9ReDfTxQrq4UgGoU=
 github.com/pion/srtp/v2 v2.0.2/go.mod h1:VEyLv4CuxrwGY8cxM+Ng3bmVy8ckz/1t6A0q/msKOw0=
 github.com/pion/stun v0.3.5 h1:uLUCBCkQby4S1cf6CGuR9QrVOKcvUwFeemaC865QHDg=
 github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA=
-github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE=
 github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A=
 github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q=
 github.com/pion/transport v0.12.3 h1:vdBfvfU/0Wq8kd2yhUMSDB/x+O4Z9MYVl2fJ5BT4JZw=
 github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A=
 github.com/pion/turn/v2 v2.0.5 h1:iwMHqDfPEDEOFzwWKT56eFmh6DYC6o/+xnLAEzgISbA=
 github.com/pion/turn/v2 v2.0.5/go.mod h1:APg43CFyt/14Uy7heYUOGWdkem/Wu4PhCO/bjyrTqMw=
-github.com/pion/udp v0.1.0 h1:uGxQsNyrqG3GLINv36Ff60covYmfrLoxzwnCsIYspXI=
-github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths=
 github.com/pion/udp v0.1.1 h1:8UAPvyqmsxK8oOjloDk4wUt63TzFe9WEJkg5lChlj7o=
 github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M=
-github.com/pion/webrtc/v3 v3.0.20 h1:Jj0sk45MqQdkR24E1wbFRmOzb1Lv258ot9zd2fYB/Pw=
-github.com/pion/webrtc/v3 v3.0.20/go.mod h1:0eJnCpQrUMpRnvyonw4ZiWClToerpixrZ2KcoTxvX9M=
-github.com/pion/webrtc/v3 v3.0.21 h1:AbWcIdpMjk95qnVmWzgj561yfGm1LHSvJdfUvvuJdMM=
-github.com/pion/webrtc/v3 v3.0.21/go.mod h1:hYG8kKExnrs8KJHWE6ljDUD8v8zXdfIdOtnJzJXJsrg=
+github.com/pion/webrtc/v3 v3.0.22 h1:FSpPotB8BApCzChpM8jBpOFgX3WI17iUoNAy1/VYhyk=
+github.com/pion/webrtc/v3 v3.0.22/go.mod h1:uy3JEfbQwohVUHwFCRV9DN3oTwnJrPnbUpP2BqMt65g=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -172,27 +155,23 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
-golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4 h1:b0LrWgu8+q7z4J+0Y3Umo5q1dL7NXBkKBWkaVkAq17E=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c h1:KHUzaHIpjWVlVVNh65G3hhuj3KB1HnjY6Cq5cTvRQT8=
 golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210414194228-064579744ee0 h1:iqW3Mjl/6IP9cHJC/wdiIu3lyBDMUfDElRMyFlqbtiQ=
+golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -201,27 +180,26 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005 h1:pDMpM2zh2MT0kHy037cKlSby2nEhD50SYqwQk76Nm40=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 h1:Bli41pIlzTzf3KEY06n+xnzK/BESIg2ze4Pgfh/aI8c=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0 h1:g9s1Ppvvun/fI+BptTMj909BBIcGrzQ32k9FNlcevOE=
+golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -246,11 +224,10 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
-nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=

From e8e0d9e72e7e969c566179e9918928c8bc905f20 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Tue, 20 Apr 2021 20:06:24 +0800
Subject: [PATCH 51/52] Update dependencies

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 1bb537d..3fb28c9 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.16
 
 require (
 	github.com/SB-IM/logging v0.2.2
-	github.com/SB-IM/mqtt-client v0.1.2
+	github.com/SB-IM/mqtt-client v0.1.3
 	github.com/SB-IM/pb v0.3.1
 	github.com/eclipse/paho.mqtt.golang v1.3.3
 	github.com/google/uuid v1.2.0
diff --git a/go.sum b/go.sum
index 7154552..cf63e4c 100644
--- a/go.sum
+++ b/go.sum
@@ -5,6 +5,8 @@ github.com/SB-IM/logging v0.2.2 h1:YiG/bWJfc2L6Sx3/2U4MIWmjRiTOGKCUt2subs0cIuE=
 github.com/SB-IM/logging v0.2.2/go.mod h1:P3uXFI5pK3K/zvuOgBVwEjCB+NhW5DB0pv96JXez3lA=
 github.com/SB-IM/mqtt-client v0.1.2 h1:mdrcmuzmcLJXLl9gi3bmNzGJ+KHfp2zqWKhaqJeN//8=
 github.com/SB-IM/mqtt-client v0.1.2/go.mod h1:cNLK452FV+CbQl/xslDvkiJwQcj2oq7PfbwRMdoJHcg=
+github.com/SB-IM/mqtt-client v0.1.3 h1:b2qL+/5xlp8ZatqqjDbP/RqDPbYZ74N33jzs8NmrUCE=
+github.com/SB-IM/mqtt-client v0.1.3/go.mod h1:R6no5syEw8Vk2tLXgFY3Aizgfo4k+Dfa9N1ymE59uQQ=
 github.com/SB-IM/pb v0.3.1 h1:CgSdrOi8kiX15J7Wx+n/jcnyi2gu9fx7ySWt8QrQ1G4=
 github.com/SB-IM/pb v0.3.1/go.mod h1:/XrV0hXCaaOQSDhP6msTH8MwvhdC1RKQSsRKDkeB2FE=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=

From be212478d1a74467d6feb0aae991511c40001567 Mon Sep 17 00:00:00 2001
From: William <gopherx@protonmail.ch>
Date: Thu, 22 Apr 2021 14:01:53 +0800
Subject: [PATCH 52/52] Tidy track and RTP sender attached to failed peer
 connection(#12)

---
 internal/broadcast/webrtc/webrtc.go | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/internal/broadcast/webrtc/webrtc.go b/internal/broadcast/webrtc/webrtc.go
index 6e404f3..62ec20d 100644
--- a/internal/broadcast/webrtc/webrtc.go
+++ b/internal/broadcast/webrtc/webrtc.go
@@ -160,7 +160,7 @@ func (w *WebRTC) signalPeerConnection(peerConnection *webrtc.PeerConnection) err
 
 		switch connectionState {
 		case webrtc.ICEConnectionStateFailed:
-			if err := peerConnection.Close(); err != nil {
+			if err := closePeerConnection(peerConnection); err != nil {
 				w.logger.Panic().Err(err).Msg("could not close peer connection")
 			}
 			w.logger.Debug().Msg("peer connection has been closed")
@@ -230,6 +230,24 @@ func (w *WebRTC) signalCandidate(peerConnection *webrtc.PeerConnection, ch <-cha
 	}
 }
 
+// closePeerConnection tidies RTPSender and remvoes track from peer connection.
+// It's used after a subscriber peer connection fails.
+// A publisher calls this has no effect.
+func closePeerConnection(peerConnection *webrtc.PeerConnection) error {
+	if peerConnection == nil {
+		return nil
+	}
+	for _, sender := range peerConnection.GetSenders() {
+		if err := sender.Stop(); err != nil {
+			return fmt.Errorf("could not stop RTP sender: %w", err)
+		}
+		if err := peerConnection.RemoveTrack(sender); err != nil {
+			return fmt.Errorf("could not remove track: %w", err)
+		}
+	}
+	return peerConnection.Close()
+}
+
 // sendRTCP sends a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
 // This can be less wasteful by processing incoming RTCP events, then we would emit a NACK/PLI when a viewer requests it
 func (w *WebRTC) sendRTCP(peerConnection *webrtc.PeerConnection, remoteTrack *webrtc.TrackRemote) {