Skip to content

Commit

Permalink
Graceful restart for l3afd (#432)
Browse files Browse the repository at this point in the history 
Implementing the graceful restart for l3afd and ensuring an uninterrupted data plane.

-------
Signed-off-by: Atul-source <atulprajapati6031@gmail.com>
  • Loading branch information
@Atul-source
Atul-source authored Sep 26, 2024
1 parent 5234bef commit 552225f
Show file tree
Hide file tree
Showing 40 changed files with 2,282 additions and 483 deletions.
27 changes: 17 additions & 10 deletions apis/configwatch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"net"
"net/http"
"os"
Expand All @@ -26,6 +27,7 @@ import (

"github.com/l3af-project/l3afd/v2/bpfprogs"
"github.com/l3af-project/l3afd/v2/config"
"github.com/l3af-project/l3afd/v2/models"
"github.com/l3af-project/l3afd/v2/routes"
"github.com/l3af-project/l3afd/v2/signals"

Expand Down Expand Up @@ -58,7 +60,17 @@ func StartConfigWatcher(ctx context.Context, hostname, daemonName string, conf *
},
SANMatchRules: conf.MTLSSANMatchRules,
}

if _, ok := models.AllNetListeners.Load("main_http"); !ok {
tcpAddr, err := net.ResolveTCPAddr("tcp", conf.L3afConfigsRestAPIAddr)
if err != nil {
return fmt.Errorf("error resolving TCP address:%w", err)
}
listener, err := net.ListenTCP("tcp", tcpAddr)
if err != nil {
return fmt.Errorf("creating tcp listner failed with %w", err)
}
models.AllNetListeners.Store("main_http", listener)
}
term := make(chan os.Signal, 1)
signal.Notify(term, signals.ShutdownSignals...)
go func() {
Expand All @@ -73,14 +85,14 @@ func StartConfigWatcher(ctx context.Context, hostname, daemonName string, conf *
if conf.SwaggerApiEnabled {
r.Mount("/swagger", httpSwagger.WrapHandler)
}

s.l3afdServer.Handler = r

// As per design discussion when mTLS flag is not set and not listening on loopback or localhost
if !conf.MTLSEnabled && !isLoopback(conf.L3afConfigsRestAPIAddr) && conf.Environment == config.ENV_PROD {
conf.MTLSEnabled = true
}

val, _ := models.AllNetListeners.Load("main_http")
l, _ := val.(*net.TCPListener)
if conf.MTLSEnabled {
log.Info().Msgf("l3afd server listening with mTLS - %s ", conf.L3afConfigsRestAPIAddr)
// Create a CA certificate pool and add client ca's to it
Expand Down Expand Up @@ -137,25 +149,21 @@ func StartConfigWatcher(ctx context.Context, hostname, daemonName string, conf *
}
}
}()

if err := s.l3afdServer.ListenAndServeTLS(serverCertFile, serverKeyFile); err != nil {
if err := s.l3afdServer.ServeTLS(l, serverCertFile, serverKeyFile); !errors.Is(err, http.ErrServerClosed) {
log.Fatal().Err(err).Msgf("failed to start L3AFD server with mTLS enabled")
}
} else {
log.Info().Msgf("l3afd server listening - %s ", conf.L3afConfigsRestAPIAddr)

if err := s.l3afdServer.ListenAndServe(); err != nil {
if err := s.l3afdServer.Serve(l); !errors.Is(err, http.ErrServerClosed) {
log.Fatal().Err(err).Msgf("failed to start L3AFD server")
}
}
}()

return nil
}

func (s *Server) GracefulStop(shutdownTimeout time.Duration) error {
log.Info().Msg("L3afd graceful stop initiated")

exitCode := 0
if len(s.BPFRTConfigs.IngressXDPBpfs) > 0 || len(s.BPFRTConfigs.IngressTCBpfs) > 0 || len(s.BPFRTConfigs.EgressTCBpfs) > 0 || s.BPFRTConfigs.ProbesBpfs.Len() > 0 {
ctx, cancelfunc := context.WithTimeout(context.Background(), shutdownTimeout)
Expand All @@ -165,7 +173,6 @@ func (s *Server) GracefulStop(shutdownTimeout time.Duration) error {
exitCode = 1
}
}

os.Exit(exitCode)
return nil
}
Expand Down
19 changes: 18 additions & 1 deletion apis/handlers/addprog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,13 @@ func AddEbpfPrograms(ctx context.Context, bpfcfg *bpfprogs.NFConfigs) http.Handl
log.Warn().Msgf("Failed to write response bytes: %v", err)
}
}(&mesg, &statusCode)

if models.IsReadOnly {
log.Warn().Msgf("We are in between restart please try after some time")
mesg = "We are currently in the middle of a restart. Please attempt again after a while."
return
}
defer DecWriteReq()
IncWriteReq()
if r.Body == nil {
log.Warn().Msgf("Empty request body")
return
Expand Down Expand Up @@ -70,3 +76,14 @@ func AddEbpfPrograms(ctx context.Context, bpfcfg *bpfprogs.NFConfigs) http.Handl
}
}
}

func IncWriteReq() {
models.StateLock.Lock()
models.CurrentWriteReq++
models.StateLock.Unlock()
}
func DecWriteReq() {
models.StateLock.Lock()
models.CurrentWriteReq--
models.StateLock.Unlock()
}
43 changes: 31 additions & 12 deletions apis/handlers/addprog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/l3af-project/l3afd/v2/bpfprogs"
"github.com/l3af-project/l3afd/v2/config"
"github.com/l3af-project/l3afd/v2/models"
)

const dummypayload string = `[
Expand All @@ -31,27 +32,30 @@ const dummypayload string = `[
func Test_addprog(t *testing.T) {

tests := []struct {
name string
Body *strings.Reader
header map[string]string
status int
cfg *bpfprogs.NFConfigs
name string
Body *strings.Reader
header map[string]string
status int
cfg *bpfprogs.NFConfigs
isreadonly bool
}{
{
name: "NilBody",
Body: nil,
status: http.StatusOK,
name: "NilBody",
Body: nil,
status: http.StatusOK,
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
},
},
},
{
name: "FailedToUnmarshal",
Body: strings.NewReader("Something"),
status: http.StatusInternalServerError,
header: map[string]string{},
name: "FailedToUnmarshal",
Body: strings.NewReader("Something"),
status: http.StatusInternalServerError,
header: map[string]string{},
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
Expand All @@ -64,6 +68,7 @@ func Test_addprog(t *testing.T) {
header: map[string]string{
"Content-Type": "application/json",
},
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
Expand All @@ -82,6 +87,17 @@ func Test_addprog(t *testing.T) {
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
},
},
isreadonly: false,
},
{
name: "InReadonly",
Body: nil,
header: map[string]string{
"Content-Type": "application/json",
},
isreadonly: true,
cfg: nil,
status: http.StatusOK,
},
}
for _, tt := range tests {
Expand All @@ -94,11 +110,14 @@ func Test_addprog(t *testing.T) {
for key, val := range tt.header {
req.Header.Set(key, val)
}
models.IsReadOnly = tt.isreadonly
rr := httptest.NewRecorder()
handler := AddEbpfPrograms(context.Background(), tt.cfg)
handler.ServeHTTP(rr, req)
if rr.Code != tt.status {
models.IsReadOnly = false
t.Error("AddEbpfPrograms Failed")
}
models.IsReadOnly = false
}
}
8 changes: 7 additions & 1 deletion apis/handlers/deleteprog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,13 @@ func DeleteEbpfPrograms(ctx context.Context, bpfcfg *bpfprogs.NFConfigs) http.Ha
log.Warn().Msgf("Failed to write response bytes: %v", err)
}
}(&mesg, &statusCode)

if models.IsReadOnly {
log.Warn().Msgf("We are in between restart please try after some time")
mesg = "We are currently in the middle of a restart. Please attempt again after a while."
return
}
defer DecWriteReq()
IncWriteReq()
if r.Body == nil {
log.Warn().Msgf("Empty request body")
return
Expand Down
51 changes: 35 additions & 16 deletions apis/handlers/deleteprog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/l3af-project/l3afd/v2/bpfprogs"
"github.com/l3af-project/l3afd/v2/config"
"github.com/l3af-project/l3afd/v2/models"
)

const payloadfordelete string = `[
Expand All @@ -29,27 +30,30 @@ const payloadfordelete string = `[
func Test_DeleteEbpfPrograms(t *testing.T) {

tests := []struct {
name string
Body *strings.Reader
header map[string]string
status int
cfg *bpfprogs.NFConfigs
name string
Body *strings.Reader
header map[string]string
status int
cfg *bpfprogs.NFConfigs
isreadonly bool
}{
{
name: "NilBody",
Body: nil,
status: http.StatusOK,
name: "NilBody",
Body: nil,
status: http.StatusOK,
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
},
},
},
{
name: "FailedToUnmarshal",
Body: strings.NewReader("Something"),
status: http.StatusInternalServerError,
header: map[string]string{},
name: "FailedToUnmarshal",
Body: strings.NewReader("Something"),
status: http.StatusInternalServerError,
header: map[string]string{},
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
Expand All @@ -62,6 +66,7 @@ func Test_DeleteEbpfPrograms(t *testing.T) {
header: map[string]string{
"Content-Type": "application/json",
},
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
Expand All @@ -70,17 +75,28 @@ func Test_DeleteEbpfPrograms(t *testing.T) {
status: http.StatusOK,
},
{
name: "UnknownHostName",
Body: strings.NewReader(payloadfordelete),
status: http.StatusInternalServerError,
header: map[string]string{},
name: "UnknownHostName",
Body: strings.NewReader(payloadfordelete),
status: http.StatusInternalServerError,
header: map[string]string{},
isreadonly: false,
cfg: &bpfprogs.NFConfigs{
HostName: "dummy",
HostConfig: &config.Config{
L3afConfigStoreFileName: filepath.FromSlash("../../testdata/Test_l3af-config.json"),
},
},
},
{
name: "InReadonly",
Body: nil,
status: http.StatusOK,
header: map[string]string{
"Content-Type": "application/json",
},
isreadonly: true,
cfg: nil,
},
}
for _, tt := range tests {
var req *http.Request
Expand All @@ -92,11 +108,14 @@ func Test_DeleteEbpfPrograms(t *testing.T) {
for key, val := range tt.header {
req.Header.Set(key, val)
}
models.IsReadOnly = tt.isreadonly
rr := httptest.NewRecorder()
handler := DeleteEbpfPrograms(context.Background(), tt.cfg)
handler.ServeHTTP(rr, req)
if rr.Code != tt.status {
models.IsReadOnly = false
t.Error("DeleteEbpfPrograms Failed")
}
models.IsReadOnly = false
}
}
Loading

0 comments on commit 552225f

Please sign in to comment.