From 04bead7c788653b29aa180bdfb43ab96295fc87b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 20:37:36 +0200 Subject: [PATCH 01/39] Godeps updates --- Godeps/Godeps.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a207807..2f817bb 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -4,7 +4,7 @@ "Deps": [ { "ImportPath": "github.com/dawanda/go-mesos/marathon", - "Rev": "76abdfc1b8876b39505c5a1e63583bba16f3243a" + "Rev": "41be4aa9c9f5170cb339043d8f21eb66d8a07807" }, { "ImportPath": "github.com/gorilla/context", From 126a9444b3aa3b9cd3ebb488ec43110f9085ef8e Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 20:38:16 +0200 Subject: [PATCH 02/39] adds some helper more functions --- helper.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/helper.go b/helper.go index 77a95b0..bef9b77 100644 --- a/helper.go +++ b/helper.go @@ -21,6 +21,13 @@ func PrettifyAppId(name string, portIndex int, servicePort uint) (appID string) return } +func PrettifyAppId2(name string, portIndex int) (appID string) { + appID = strings.Replace(name[1:], "/", ".", -1) + appID = fmt.Sprintf("%v-%v", appID, portIndex) + + return +} + func PrettifyDnsName(dns string) string { return strings.SplitN(dns, ".", 1)[0] } @@ -165,6 +172,16 @@ func GetHealthCheckProtocol(app *marathon.App, portIndex int) string { return "" } +func FindHealthCheckForPortIndex(healthChecks []marathon.HealthCheck, portIndex int) *marathon.HealthCheck { + for _, hs := range healthChecks { + if hs.PortIndex == portIndex { + return &hs + } + } + + return nil +} + func GetHealthCheckForPortIndex(healthChecks []marathon.HealthCheck, portIndex int) marathon.HealthCheck { for _, hs := range healthChecks { if hs.PortIndex == portIndex { From a92774d2a6a5334d932180b68d4b3f759a30d18d Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 20:39:06 +0200 Subject: [PATCH 03/39] initial and incomplete refactor code with more abstract (easy) module API --- app.go | 32 ++++++++++++ event_listener.go | 10 ++++ event_logger.go | 42 +++++++++++++++ main.go | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 app.go create mode 100644 event_listener.go create mode 100644 event_logger.go diff --git a/app.go b/app.go new file mode 100644 index 0000000..136f082 --- /dev/null +++ b/app.go @@ -0,0 +1,32 @@ +package main + +// Marathon independant application definitions + +type AppCluster struct { + Name string + Id string + ServicePort uint + Protocol string + PortName string + Labels map[string]string + HealthCheck *AppHealthCheck + Backends []AppBackend +} + +type AppHealthCheck struct { + Protocol string + Path string + Command *string + GracePeriodSeconds uint + IntervalSeconds uint + TimeoutSeconds uint + MaxConsecutiveFailures uint + IgnoreHttp1xx bool +} + +type AppBackend struct { + Id string + Host string + Port uint + State string +} diff --git a/event_listener.go b/event_listener.go new file mode 100644 index 0000000..3111b6f --- /dev/null +++ b/event_listener.go @@ -0,0 +1,10 @@ +package main + +type EventListener interface { + Startup() + Shutdown() + + Apply(apps []AppCluster) + AddTask(task AppBackend, app AppCluster) + RemoveTask(task AppBackend, app AppCluster) +} diff --git a/event_logger.go b/event_logger.go new file mode 100644 index 0000000..83b3100 --- /dev/null +++ b/event_logger.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" +) + +type EventLogger struct { + Verbose bool +} + +func (logger *EventLogger) Startup() { + log.Printf("Initialize\n") +} + +func (logger *EventLogger) Shutdown() { + log.Printf("Shutdown\n") +} + +func (logger *EventLogger) Apply(apps []AppCluster) { + if logger.Verbose { + out, err := json.MarshalIndent(apps, "", " ") + if err != nil { + log.Printf("Marshal failed. %v\n", err) + } else { + fmt.Printf("%v\n", string(out)) + } + } + + for _, app := range apps { + log.Printf("Apply: %v\n", app.Id) + } +} + +func (logger *EventLogger) AddTask(task AppBackend, app AppCluster) { + log.Printf("Task Add: %v: %v %v\n", task.State, app.Id, task.Host) +} + +func (logger *EventLogger) RemoveTask(task AppBackend, app AppCluster) { + log.Printf("Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) +} diff --git a/main.go b/main.go index 062d699..4c42f43 100644 --- a/main.go +++ b/main.go @@ -56,6 +56,7 @@ type mmsdService struct { HttpApiPort uint Verbose bool Handlers []mmsdHandler + Listeners []EventListener quitChannel chan bool RunStateDir string FilterGroups string @@ -98,6 +99,9 @@ type mmsdService struct { DnsBaseName string DnsTTL time.Duration DnsPushSRV bool + + // runtime state + apps []AppCluster } func (mmsd *mmsdService) setupHttpService() { @@ -123,6 +127,55 @@ func (mmsd *mmsdService) v0Version(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "mmsd %v\n", appVersion) } +func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { + var apps []AppCluster + for _, mApp := range mApps { + for portIndex, _ := range mApp.PortDefinitions { + var healthCheck *AppHealthCheck + if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { + var mCommand *string + if mHealthCheck.Command != nil { + mCommand = new(string) + *mCommand = mHealthCheck.Command.Value + } + healthCheck = &AppHealthCheck{ + Protocol: mHealthCheck.Protocol, + Path: mHealthCheck.Path, + Command: mCommand, + GracePeriodSeconds: mHealthCheck.GracePeriodSeconds, + IntervalSeconds: mHealthCheck.IntervalSeconds, + TimeoutSeconds: mHealthCheck.TimeoutSeconds, + MaxConsecutiveFailures: mHealthCheck.MaxConsecutiveFailures, + IgnoreHttp1xx: mHealthCheck.IgnoreHttp1xx, + } + } + + var backends []AppBackend + for _, mTask := range mApp.Tasks { + backends = append(backends, AppBackend{ + Id: mTask.Id, + Host: mTask.Host, + Port: mTask.Ports[portIndex], + State: string(*mTask.State), + }) + } + + app := AppCluster{ + Name: mApp.Id, + Id: PrettifyAppId2(mApp.Id, portIndex), + ServicePort: mApp.PortDefinitions[portIndex].Port, + Protocol: mApp.PortDefinitions[portIndex].Protocol, + PortName: mApp.PortDefinitions[portIndex].Name, + Labels: mApp.PortDefinitions[portIndex].Labels, + HealthCheck: healthCheck, + Backends: backends, + } + apps = append(apps, app) + } + } + return apps +} + func (mmsd *mmsdService) v1Apps(w http.ResponseWriter, r *http.Request) { m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) @@ -290,6 +343,8 @@ func (mmsd *mmsdService) setupEventBusListener() { sse.OnOpen = func(event, data string) { log.Printf("Listening for events from Marathon on %v\n", url) + + mmsd.applyApps(mmsd.convertMarathonApps(mmsd.getMarathonApps())) } sse.OnError = func(event, data string) { @@ -320,6 +375,16 @@ func (mmsd *mmsdService) setupEventBusListener() { go sse.RunForever() } +func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []AppCluster { + var apps []AppCluster + for _, app := range mmsd.apps { + if app.Name == mAppId { + apps = append(apps, app) + } + } + return apps +} + func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { switch event.TaskStatus { case marathon.TaskRunning: @@ -334,10 +399,30 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { // XXX Only update propagate no health checks have been configured. // So we consider thie TASK_RUNNING state as healthy-notice. if len(app.HealthChecks) == 0 { + for _, app := range mmsd.findAppsByMarathonId(event.AppId) { + for _, task := range app.Backends { + if task.Id == event.TaskId { + for _, listener := range mmsd.Listeners { + listener.AddTask(task, app) + } + } + } + } + //. mmsd.Update(event.AppId, event.TaskId, true) } case marathon.TaskFinished, marathon.TaskFailed, marathon.TaskKilling, marathon.TaskKilled, marathon.TaskLost: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) + for _, app := range mmsd.findAppsByMarathonId(event.AppId) { + for _, task := range app.Backends { + if task.Id == event.TaskId { + for _, listener := range mmsd.Listeners { + listener.RemoveTask(task, app) + } + } + } + } + //. app, err := mmsd.getMarathonApp(event.AppId) if err != nil { log.Printf("Failed to fetch Marathon app. %+v. %v\n", event, err) @@ -348,6 +433,21 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { } func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusChangedEvent) { + for _, app := range mmsd.findAppsByMarathonId(event.AppId) { + for _, task := range app.Backends { + if task.Id == event.TaskId { + for _, listener := range mmsd.Listeners { + if event.Alive { + listener.AddTask(task, app) + } else { + listener.RemoveTask(task, app) + } + } + } + } + } + // EOF + app, err := mmsd.getMarathonApp(event.AppId) if err != nil { log.Printf("Failed to fetch Marathon app. %+v. %v\n", event, err) @@ -376,6 +476,22 @@ func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusCh mmsd.Update(event.AppId, event.TaskId, event.Alive) } +func (mmsd *mmsdService) getMarathonApps() []marathon.App { + m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) + if err != nil { + log.Printf("Failed to get marathon.Service: %v\n", err) + return nil + } + + mApps, err := m.GetApps() + var mApps2 []marathon.App + for _, mApp := range mApps { + mApps2 = append(mApps2, *mApp) + } + + return mApps2 +} + func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) if err != nil { @@ -498,7 +614,19 @@ func (mmsd *mmsdService) Run() { <-mmsd.quitChannel } +func (mmsd *mmsdService) applyApps(apps []AppCluster) { + mmsd.apps = apps + + for _, listener := range mmsd.Listeners { + listener.Apply(apps) + } +} + func (mmsd *mmsdService) setupHandlers() { + mmsd.Listeners = append(mmsd.Listeners, &EventLogger{ + Verbose: true, + }) + if mmsd.DnsEnabled { mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ Verbose: mmsd.Verbose, From 3cd00d7640c8f7394727ae184031fd9ebd569777 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 22:05:03 +0200 Subject: [PATCH 04/39] adds shutdown code paths --- event_logger.go | 2 ++ main.go | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/event_logger.go b/event_logger.go index 83b3100..c5ff4d7 100644 --- a/event_logger.go +++ b/event_logger.go @@ -6,6 +6,8 @@ import ( "log" ) +/* EventLogger adds simple event logging to the logger. + */ type EventLogger struct { Verbose bool } diff --git a/main.go b/main.go index 4c42f43..aa77a73 100644 --- a/main.go +++ b/main.go @@ -612,6 +612,8 @@ func (mmsd *mmsdService) Run() { mmsd.setupHttpService() <-mmsd.quitChannel + + mmsd.shutdown() } func (mmsd *mmsdService) applyApps(apps []AppCluster) { @@ -688,6 +690,16 @@ func (mmsd *mmsdService) setupHandlers() { if err != nil { log.Printf("Could not force task state reset. %v\n", err) } + + for _, listener := range mmsd.Listeners { + listener.Startup() + } +} + +func (mmsd *mmsdService) shutdown() { + for _, listener := range mmsd.Listeners { + listener.Shutdown() + } } func locateExe(name string) string { From f719a65f8c72305c6658d5e03b2fa892fa7f5848 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 22:25:46 +0200 Subject: [PATCH 05/39] partial move to new module interface --- dns_manager.go | 4 +- files_manager.go | 13 +- haproxy_manager.go | 39 ++--- helper.go | 62 +++++++ main.go | 415 ++++++++++++++------------------------------- udp_manager.go | 16 +- 6 files changed, 207 insertions(+), 342 deletions(-) diff --git a/dns_manager.go b/dns_manager.go index e0c0f1d..d122179 100644 --- a/dns_manager.go +++ b/dns_manager.go @@ -44,7 +44,7 @@ type appEntry struct { app *marathon.App } -func (manager *DnsManager) Setup() error { +func (manager *DnsManager) Startup() { dns.HandleFunc(manager.BaseName, manager.dnsHandler) go func() { @@ -70,8 +70,6 @@ func (manager *DnsManager) Setup() error { log.Fatal(err) } }() - - return nil } func (manager *DnsManager) Shutdown() { diff --git a/files_manager.go b/files_manager.go index cab6fc8..a47a749 100644 --- a/files_manager.go +++ b/files_manager.go @@ -12,7 +12,6 @@ import ( ) type FilesManager struct { - Enabled bool Verbose bool BasePath string } @@ -23,18 +22,10 @@ func (upstream *FilesManager) Log(msg string) { } } -func (manager *FilesManager) Setup() error { - return nil -} - -func (manager *FilesManager) IsEnabled() bool { - return manager.Enabled +func (manager *FilesManager) Startup() { } -func (manager *FilesManager) SetEnabled(value bool) { - if value != manager.Enabled { - manager.Enabled = value - } +func (manager *FilesManager) Shutdown() { } func (upstream *FilesManager) Remove(appID string, taskID string, app *marathon.App) error { diff --git a/haproxy_manager.go b/haproxy_manager.go index 24db654..f2f03df 100644 --- a/haproxy_manager.go +++ b/haproxy_manager.go @@ -23,7 +23,6 @@ import ( ) type HaproxyMgr struct { - Enabled bool Verbose bool LocalHealthChecks bool FilterGroups []string @@ -141,21 +140,22 @@ func makeStringArray(s string) []string { } } -func (manager *HaproxyMgr) Setup() error { - return nil +func (manager *HaproxyMgr) Startup() { } -func (manager *HaproxyMgr) IsEnabled() bool { - return manager.Enabled +func (manager *HaproxyMgr) Shutdown() { } -func (manager *HaproxyMgr) SetEnabled(value bool) { - if value != manager.Enabled { - manager.Enabled = value - } +func (manager *HaproxyMgr) Apply(apps []AppCluster) { +} + +func (manager *HaproxyMgr) AddTask(task AppBackend, app AppCluster) { +} + +func (manager *HaproxyMgr) RemoveTask(task AppBackend, app AppCluster) { } -func (manager *HaproxyMgr) Apply(apps []*marathon.App, force bool) error { +func (manager *HaproxyMgr) OLD_Apply(apps []*marathon.App, force bool) error { manager.appConfigFragments = make(map[string]string) manager.clearAppStateCache() @@ -184,7 +184,7 @@ func (manager *HaproxyMgr) Apply(apps []*marathon.App, force bool) error { return manager.reloadConfig(false) } -func (manager *HaproxyMgr) Remove(appID string, taskID string, app *marathon.App) error { +func (manager *HaproxyMgr) OLD_Remove(appID string, taskID string, app *marathon.App) error { if app != nil { // make sure we *remove* the task from the cluster config, err := manager.makeConfig(app) @@ -213,23 +213,10 @@ func (manager *HaproxyMgr) Remove(appID string, taskID string, app *marathon.App } func isAppJustSpawned(app *marathon.App) bool { - // find out if an app has just been spawned by checking - // if it ever failed already. - - if len(app.Tasks) == 0 { - return false - } - - for _, hsr := range app.Tasks[0].HealthCheckResults { - if hsr.LastFailure != nil { - return false - } - } - - return true + return false // XXX removed } -func (manager *HaproxyMgr) Update(app *marathon.App, taskID string) error { +func (manager *HaproxyMgr) OLD_Update(app *marathon.App, taskID string) error { // collect list of task labels as we formatted them in haproxy.cfg. var instanceNames []string for portIndex, servicePort := range app.Ports { diff --git a/helper.go b/helper.go index bef9b77..b1a733f 100644 --- a/helper.go +++ b/helper.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "errors" "fmt" "hash/fnv" "io" @@ -14,6 +15,10 @@ import ( "github.com/dawanda/go-mesos/marathon" ) +var ( + ErrInvalidPortRange = errors.New("Invalid port range") +) + func PrettifyAppId(name string, portIndex int, servicePort uint) (appID string) { appID = strings.Replace(name[1:], "/", ".", -1) appID = fmt.Sprintf("%v-%v-%v", appID, portIndex, servicePort) @@ -197,3 +202,60 @@ func Hash(s string) uint32 { h.Write([]byte(s)) return h.Sum32() } + +func resolveIPAddr(dns string, skip bool) string { + if skip { + return dns + } else { + ip, err := net.ResolveIPAddr("ip", dns) + if err != nil { + return dns + } else { + return ip.String() + } + } +} + +func parseRange(input string) (int, int, error) { + if len(input) == 0 { + return 0, 0, nil + } + + vals := strings.Split(input, ":") + log.Printf("vals: %+q\n", vals) + + if len(vals) == 1 { + i, err := strconv.Atoi(input) + return i, i, err + } + + if len(vals) > 2 { + return 0, 0, ErrInvalidPortRange + } + + var ( + begin int + end int + err error + ) + + // parse begin + if vals[0] != "" { + begin, err = strconv.Atoi(vals[0]) + if err != nil { + return begin, end, err + } + } + + // parse end + if vals[1] != "" { + end, err = strconv.Atoi(vals[1]) + if begin > end { + return begin, end, ErrInvalidPortRange + } + } else { + end = -1 // XXX that is: until the end + } + + return begin, end, err +} diff --git a/main.go b/main.go index aa77a73..7e41ce8 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,6 @@ XXX Changes: import ( "encoding/json" - "errors" "fmt" "log" "net" @@ -36,7 +35,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" "time" @@ -55,8 +53,7 @@ type mmsdHandler interface { type mmsdService struct { HttpApiPort uint Verbose bool - Handlers []mmsdHandler - Listeners []EventListener + Handlers []EventListener quitChannel chan bool RunStateDir string FilterGroups string @@ -104,6 +101,7 @@ type mmsdService struct { apps []AppCluster } +// {{{ HTTP endpoint func (mmsd *mmsdService) setupHttpService() { router := mux.NewRouter() router.HandleFunc("/ping", mmsd.v0Ping) @@ -127,55 +125,6 @@ func (mmsd *mmsdService) v0Version(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "mmsd %v\n", appVersion) } -func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { - var apps []AppCluster - for _, mApp := range mApps { - for portIndex, _ := range mApp.PortDefinitions { - var healthCheck *AppHealthCheck - if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { - var mCommand *string - if mHealthCheck.Command != nil { - mCommand = new(string) - *mCommand = mHealthCheck.Command.Value - } - healthCheck = &AppHealthCheck{ - Protocol: mHealthCheck.Protocol, - Path: mHealthCheck.Path, - Command: mCommand, - GracePeriodSeconds: mHealthCheck.GracePeriodSeconds, - IntervalSeconds: mHealthCheck.IntervalSeconds, - TimeoutSeconds: mHealthCheck.TimeoutSeconds, - MaxConsecutiveFailures: mHealthCheck.MaxConsecutiveFailures, - IgnoreHttp1xx: mHealthCheck.IgnoreHttp1xx, - } - } - - var backends []AppBackend - for _, mTask := range mApp.Tasks { - backends = append(backends, AppBackend{ - Id: mTask.Id, - Host: mTask.Host, - Port: mTask.Ports[portIndex], - State: string(*mTask.State), - }) - } - - app := AppCluster{ - Name: mApp.Id, - Id: PrettifyAppId2(mApp.Id, portIndex), - ServicePort: mApp.PortDefinitions[portIndex].Port, - Protocol: mApp.PortDefinitions[portIndex].Protocol, - PortName: mApp.PortDefinitions[portIndex].Name, - Labels: mApp.PortDefinitions[portIndex].Labels, - HealthCheck: healthCheck, - Backends: backends, - } - apps = append(apps, app) - } - } - return apps -} - func (mmsd *mmsdService) v1Apps(w http.ResponseWriter, r *http.Request) { m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) @@ -201,52 +150,6 @@ func (mmsd *mmsdService) v1Apps(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "%s\n", strings.Join(appList, "\n")) } -var ErrInvalidPortRange = errors.New("Invalid port range") - -func parseRange(input string) (int, int, error) { - if len(input) == 0 { - return 0, 0, nil - } - - vals := strings.Split(input, ":") - log.Printf("vals: %+q\n", vals) - - if len(vals) == 1 { - i, err := strconv.Atoi(input) - return i, i, err - } - - if len(vals) > 2 { - return 0, 0, ErrInvalidPortRange - } - - var ( - begin int - end int - err error - ) - - // parse begin - if vals[0] != "" { - begin, err = strconv.Atoi(vals[0]) - if err != nil { - return begin, end, err - } - } - - // parse end - if vals[1] != "" { - end, err = strconv.Atoi(vals[1]) - if begin > end { - return begin, end, ErrInvalidPortRange - } - } else { - end = -1 // XXX that is: until the end - } - - return begin, end, err -} - func (mmsd *mmsdService) v1Instances(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) appID := vars["app_id"] @@ -322,17 +225,95 @@ func (mmsd *mmsdService) v1Instances(w http.ResponseWriter, r *http.Request) { } } -func resolveIPAddr(dns string, skip bool) string { - if skip { - return dns - } else { - ip, err := net.ResolveIPAddr("ip", dns) - if err != nil { - return dns - } else { - return ip.String() +// }}} + +func (mmsd *mmsdService) getMarathonApps() []marathon.App { + m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) + if err != nil { + log.Printf("Failed to get marathon.Service: %v\n", err) + return nil + } + + mApps, err := m.GetApps() + var mApps2 []marathon.App + for _, mApp := range mApps { + mApps2 = append(mApps2, *mApp) + } + + return mApps2 +} + +func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { + m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) + if err != nil { + return nil, err + } + + app, err := m.GetApp(appID) + if err != nil { + return nil, err + } + + return app, nil +} + +func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { + var apps []AppCluster + for _, mApp := range mApps { + for portIndex, _ := range mApp.PortDefinitions { + var healthCheck *AppHealthCheck + if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { + var mCommand *string + if mHealthCheck.Command != nil { + mCommand = new(string) + *mCommand = mHealthCheck.Command.Value + } + healthCheck = &AppHealthCheck{ + Protocol: mHealthCheck.Protocol, + Path: mHealthCheck.Path, + Command: mCommand, + GracePeriodSeconds: mHealthCheck.GracePeriodSeconds, + IntervalSeconds: mHealthCheck.IntervalSeconds, + TimeoutSeconds: mHealthCheck.TimeoutSeconds, + MaxConsecutiveFailures: mHealthCheck.MaxConsecutiveFailures, + IgnoreHttp1xx: mHealthCheck.IgnoreHttp1xx, + } + } + + var backends []AppBackend + for _, mTask := range mApp.Tasks { + backends = append(backends, AppBackend{ + Id: mTask.Id, + Host: mTask.Host, + Port: mTask.Ports[portIndex], + State: string(*mTask.State), + }) + } + + app := AppCluster{ + Name: mApp.Id, + Id: PrettifyAppId2(mApp.Id, portIndex), + ServicePort: mApp.PortDefinitions[portIndex].Port, + Protocol: mApp.PortDefinitions[portIndex].Protocol, + PortName: mApp.PortDefinitions[portIndex].Name, + Labels: mApp.PortDefinitions[portIndex].Labels, + HealthCheck: healthCheck, + Backends: backends, + } + apps = append(apps, app) } } + return apps +} + +func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []AppCluster { + var apps []AppCluster + for _, app := range mmsd.apps { + if app.Name == mAppId { + apps = append(apps, app) + } + } + return apps } func (mmsd *mmsdService) setupEventBusListener() { @@ -375,16 +356,6 @@ func (mmsd *mmsdService) setupEventBusListener() { go sse.RunForever() } -func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []AppCluster { - var apps []AppCluster - for _, app := range mmsd.apps { - if app.Name == mAppId { - apps = append(apps, app) - } - } - return apps -} - func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { switch event.TaskStatus { case marathon.TaskRunning: @@ -402,33 +373,24 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { for _, app := range mmsd.findAppsByMarathonId(event.AppId) { for _, task := range app.Backends { if task.Id == event.TaskId { - for _, listener := range mmsd.Listeners { - listener.AddTask(task, app) + for _, handler := range mmsd.Handlers { + handler.AddTask(task, app) } } } } - //. - mmsd.Update(event.AppId, event.TaskId, true) } case marathon.TaskFinished, marathon.TaskFailed, marathon.TaskKilling, marathon.TaskKilled, marathon.TaskLost: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) for _, app := range mmsd.findAppsByMarathonId(event.AppId) { for _, task := range app.Backends { if task.Id == event.TaskId { - for _, listener := range mmsd.Listeners { - listener.RemoveTask(task, app) + for _, handler := range mmsd.Handlers { + handler.RemoveTask(task, app) } } } } - //. - app, err := mmsd.getMarathonApp(event.AppId) - if err != nil { - log.Printf("Failed to fetch Marathon app. %+v. %v\n", event, err) - return - } - mmsd.Remove(event.AppId, event.TaskId, app) } } @@ -436,126 +398,16 @@ func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusCh for _, app := range mmsd.findAppsByMarathonId(event.AppId) { for _, task := range app.Backends { if task.Id == event.TaskId { - for _, listener := range mmsd.Listeners { + for _, handler := range mmsd.Handlers { if event.Alive { - listener.AddTask(task, app) + handler.AddTask(task, app) } else { - listener.RemoveTask(task, app) + handler.RemoveTask(task, app) } } } } } - // EOF - - app, err := mmsd.getMarathonApp(event.AppId) - if err != nil { - log.Printf("Failed to fetch Marathon app. %+v. %v\n", event, err) - return - } - if app == nil { - log.Printf("App %v not found anymore.\n", event.AppId) - return - } - - task := app.GetTaskById(event.TaskId) - if task == nil { - log.Printf("App %v task %v not found anymore.\n", event.AppId, event.TaskId) - mmsd.Remove(event.AppId, event.TaskId, app) - return - } - - // app & task definitely do exist, so propagate health change event - - if event.Alive { - log.Printf("App %v task %v on %v is healthy.\n", event.AppId, event.TaskId, task.Host) - } else { - log.Printf("App %v task %v on %v is unhealthy.\n", event.AppId, event.TaskId, task.Host) - } - - mmsd.Update(event.AppId, event.TaskId, event.Alive) -} - -func (mmsd *mmsdService) getMarathonApps() []marathon.App { - m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) - if err != nil { - log.Printf("Failed to get marathon.Service: %v\n", err) - return nil - } - - mApps, err := m.GetApps() - var mApps2 []marathon.App - for _, mApp := range mApps { - mApps2 = append(mApps2, *mApp) - } - - return mApps2 -} - -func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { - m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) - if err != nil { - return nil, err - } - - app, err := m.GetApp(appID) - if err != nil { - return nil, err - } - - return app, nil -} - -// enable/disable given app:task -func (mmsd *mmsdService) Update(appID string, taskID string, alive bool) { - m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) - if err != nil { - log.Printf("Update: NewService(%q, %v) failed. %v\n", mmsd.MarathonIP, mmsd.MarathonPort, err) - return - } - - app, err := m.GetApp(appID) - if err != nil { - log.Printf("Update: GetApp(%q) failed. %v\n", appID, err) - return - } - - for _, handler := range mmsd.Handlers { - err = handler.Update(app, taskID) - if err != nil { - log.Printf("Update failed. %v\n", err) - } - } -} - -func (mmsd *mmsdService) Remove(appID string, taskID string, app *marathon.App) { - for _, handler := range mmsd.Handlers { - err := handler.Remove(appID, taskID, app) - if err != nil { - log.Printf("Remove failed. %v\n", err) - } - } -} - -func (mmsd *mmsdService) MaybeResetFromTasks(force bool) error { - m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) - if err != nil { - return fmt.Errorf("Could not create new marathon service. %v", err) - } - - apps, err := m.GetApps() - if err != nil { - return fmt.Errorf("Could not get apps. %v", err) - } - - for _, handler := range mmsd.Handlers { - err = handler.Apply(apps, force) - if err != nil { - log.Printf("Failed to apply changes to handler. %v\n", err) - } - } - - return nil } const appVersion = "0.9.12" @@ -619,38 +471,37 @@ func (mmsd *mmsdService) Run() { func (mmsd *mmsdService) applyApps(apps []AppCluster) { mmsd.apps = apps - for _, listener := range mmsd.Listeners { - listener.Apply(apps) + for _, handler := range mmsd.Handlers { + handler.Apply(apps) } } func (mmsd *mmsdService) setupHandlers() { - mmsd.Listeners = append(mmsd.Listeners, &EventLogger{ + mmsd.Handlers = append(mmsd.Handlers, &EventLogger{ Verbose: true, }) - if mmsd.DnsEnabled { - mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ - Verbose: mmsd.Verbose, - ServiceAddr: mmsd.ServiceAddr, - ServicePort: mmsd.DnsPort, - PushSRV: mmsd.DnsPushSRV, - BaseName: mmsd.DnsBaseName, - DnsTTL: mmsd.DnsTTL, - }) - } - - if mmsd.UDPEnabled { - mmsd.Handlers = append(mmsd.Handlers, NewUdpManager( - mmsd.ServiceAddr, - mmsd.Verbose, - mmsd.UDPEnabled, - )) - } + // if mmsd.DnsEnabled { + // mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ + // Verbose: mmsd.Verbose, + // ServiceAddr: mmsd.ServiceAddr, + // ServicePort: mmsd.DnsPort, + // PushSRV: mmsd.DnsPushSRV, + // BaseName: mmsd.DnsBaseName, + // DnsTTL: mmsd.DnsTTL, + // }) + // } + + // if mmsd.UDPEnabled { + // mmsd.Handlers = append(mmsd.Handlers, NewUdpManager( + // mmsd.ServiceAddr, + // mmsd.Verbose, + // mmsd.UDPEnabled, + // )) + // } if mmsd.TCPEnabled { mmsd.Handlers = append(mmsd.Handlers, &HaproxyMgr{ - Enabled: mmsd.TCPEnabled, Verbose: mmsd.Verbose, LocalHealthChecks: mmsd.LocalHealthChecks, FilterGroups: strings.Split(mmsd.FilterGroups, ","), @@ -670,35 +521,21 @@ func (mmsd *mmsdService) setupHandlers() { }) } - if mmsd.FilesEnabled { - mmsd.Handlers = append(mmsd.Handlers, &FilesManager{ - Enabled: mmsd.FilesEnabled, - Verbose: mmsd.Verbose, - BasePath: mmsd.RunStateDir + "/confd", - }) - } + // if mmsd.FilesEnabled { + // mmsd.Handlers = append(mmsd.Handlers, &FilesManager{ + // Verbose: mmsd.Verbose, + // BasePath: mmsd.RunStateDir + "/confd", + // }) + // } for _, handler := range mmsd.Handlers { - err := handler.Setup() - if err != nil { - log.Fatalf("Failed to setup handlers. %v\n", err) - } - } - - // trigger initial run - err := mmsd.MaybeResetFromTasks(true) - if err != nil { - log.Printf("Could not force task state reset. %v\n", err) - } - - for _, listener := range mmsd.Listeners { - listener.Startup() + handler.Startup() } } func (mmsd *mmsdService) shutdown() { - for _, listener := range mmsd.Listeners { - listener.Shutdown() + for _, handler := range mmsd.Handlers { + handler.Shutdown() } } diff --git a/udp_manager.go b/udp_manager.go index cd34b7c..0013d53 100644 --- a/udp_manager.go +++ b/udp_manager.go @@ -11,33 +11,23 @@ import ( ) type UdpManager struct { - Enabled bool Verbose bool BindAddr net.IP Servers map[string]*udpproxy.Frontend } -func NewUdpManager(bindAddr net.IP, verbose bool, enabled bool) *UdpManager { +func NewUdpManager(bindAddr net.IP, verbose bool) *UdpManager { return &UdpManager{ - Enabled: enabled, Verbose: verbose, BindAddr: bindAddr, Servers: make(map[string]*udpproxy.Frontend), } } -func (manager *UdpManager) Setup() error { - return nil -} - -func (manager *UdpManager) IsEnabled() bool { - return manager.Enabled +func (manager *UdpManager) Startup() { } -func (manager *UdpManager) SetEnabled(value bool) { - if value != manager.Enabled { - manager.Enabled = value - } +func (manager *UdpManager) Shutdown() { } func (manager *UdpManager) Apply(apps []*marathon.App, force bool) error { From 92a715c10ec418ce52cacd0612dc8a03caeb4947 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Sat, 17 Sep 2016 22:38:16 +0200 Subject: [PATCH 06/39] move group filtering to core --- main.go | 94 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/main.go b/main.go index 7e41ce8..3e4a047 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,7 @@ type mmsdService struct { Handlers []EventListener quitChannel chan bool RunStateDir string - FilterGroups string + FilterGroups []string LocalHealthChecks bool ManagementAddr net.IP @@ -227,6 +227,15 @@ func (mmsd *mmsdService) v1Instances(w http.ResponseWriter, r *http.Request) { // }}} +func (mmsd *mmsdService) isGroupIncluded(groupName string) bool { + for _, filterGroup := range mmsd.FilterGroups { + if groupName == filterGroup || filterGroup == "*" { + return true + } + } + return false +} + func (mmsd *mmsdService) getMarathonApps() []marathon.App { m, err := marathon.NewService(mmsd.MarathonIP, mmsd.MarathonPort) if err != nil { @@ -260,47 +269,49 @@ func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { var apps []AppCluster for _, mApp := range mApps { - for portIndex, _ := range mApp.PortDefinitions { - var healthCheck *AppHealthCheck - if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { - var mCommand *string - if mHealthCheck.Command != nil { - mCommand = new(string) - *mCommand = mHealthCheck.Command.Value - } - healthCheck = &AppHealthCheck{ - Protocol: mHealthCheck.Protocol, - Path: mHealthCheck.Path, - Command: mCommand, - GracePeriodSeconds: mHealthCheck.GracePeriodSeconds, - IntervalSeconds: mHealthCheck.IntervalSeconds, - TimeoutSeconds: mHealthCheck.TimeoutSeconds, - MaxConsecutiveFailures: mHealthCheck.MaxConsecutiveFailures, - IgnoreHttp1xx: mHealthCheck.IgnoreHttp1xx, + for portIndex, portDef := range mApp.PortDefinitions { + if mmsd.isGroupIncluded(portDef.Labels["lb-group"]) { + var healthCheck *AppHealthCheck + if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { + var mCommand *string + if mHealthCheck.Command != nil { + mCommand = new(string) + *mCommand = mHealthCheck.Command.Value + } + healthCheck = &AppHealthCheck{ + Protocol: mHealthCheck.Protocol, + Path: mHealthCheck.Path, + Command: mCommand, + GracePeriodSeconds: mHealthCheck.GracePeriodSeconds, + IntervalSeconds: mHealthCheck.IntervalSeconds, + TimeoutSeconds: mHealthCheck.TimeoutSeconds, + MaxConsecutiveFailures: mHealthCheck.MaxConsecutiveFailures, + IgnoreHttp1xx: mHealthCheck.IgnoreHttp1xx, + } } - } - var backends []AppBackend - for _, mTask := range mApp.Tasks { - backends = append(backends, AppBackend{ - Id: mTask.Id, - Host: mTask.Host, - Port: mTask.Ports[portIndex], - State: string(*mTask.State), - }) - } + var backends []AppBackend + for _, mTask := range mApp.Tasks { + backends = append(backends, AppBackend{ + Id: mTask.Id, + Host: mTask.Host, + Port: mTask.Ports[portIndex], + State: string(*mTask.State), + }) + } - app := AppCluster{ - Name: mApp.Id, - Id: PrettifyAppId2(mApp.Id, portIndex), - ServicePort: mApp.PortDefinitions[portIndex].Port, - Protocol: mApp.PortDefinitions[portIndex].Protocol, - PortName: mApp.PortDefinitions[portIndex].Name, - Labels: mApp.PortDefinitions[portIndex].Labels, - HealthCheck: healthCheck, - Backends: backends, + app := AppCluster{ + Name: mApp.Id, + Id: PrettifyAppId2(mApp.Id, portIndex), + ServicePort: mApp.PortDefinitions[portIndex].Port, + Protocol: mApp.PortDefinitions[portIndex].Protocol, + PortName: mApp.PortDefinitions[portIndex].Name, + Labels: mApp.PortDefinitions[portIndex].Labels, + HealthCheck: healthCheck, + Backends: backends, + } + apps = append(apps, app) } - apps = append(apps, app) } } return apps @@ -419,12 +430,13 @@ func showVersion() { } func (mmsd *mmsdService) Run() { + var filterGroups = "*" flag.BoolVarP(&mmsd.Verbose, "verbose", "v", mmsd.Verbose, "Set verbosity level") flag.IPVar(&mmsd.MarathonIP, "marathon-ip", mmsd.MarathonIP, "Marathon endpoint TCP IP address") flag.UintVar(&mmsd.MarathonPort, "marathon-port", mmsd.MarathonPort, "Marathon endpoint TCP port number") flag.DurationVar(&mmsd.ReconnectDelay, "reconnect-delay", mmsd.ReconnectDelay, "Marathon reconnect delay") flag.StringVar(&mmsd.RunStateDir, "run-state-dir", mmsd.RunStateDir, "Path to directory to keep run-state") - flag.StringVar(&mmsd.FilterGroups, "filter-groups", mmsd.FilterGroups, "Application group filter") + flag.StringVar(&filterGroups, "filter-groups", filterGroups, "Application group filter") flag.IPVar(&mmsd.ManagedIP, "managed-ip", mmsd.ManagedIP, "IP-address to manage for mmsd") flag.BoolVar(&mmsd.GatewayEnabled, "enable-gateway", mmsd.GatewayEnabled, "Enables gateway support") flag.IPVar(&mmsd.GatewayAddr, "gateway-bind", mmsd.GatewayAddr, "gateway bind address") @@ -454,6 +466,8 @@ func (mmsd *mmsdService) Run() { flag.Parse() + mmsd.FilterGroups = strings.Split(filterGroups, ",") + if *showVersionAndExit { showVersion() os.Exit(0) @@ -504,7 +518,6 @@ func (mmsd *mmsdService) setupHandlers() { mmsd.Handlers = append(mmsd.Handlers, &HaproxyMgr{ Verbose: mmsd.Verbose, LocalHealthChecks: mmsd.LocalHealthChecks, - FilterGroups: strings.Split(mmsd.FilterGroups, ","), ServiceAddr: mmsd.ServiceAddr, GatewayEnabled: mmsd.GatewayEnabled, GatewayAddr: mmsd.GatewayAddr, @@ -556,7 +569,6 @@ func main() { MarathonPort: 8080, ReconnectDelay: time.Second * 4, RunStateDir: "/var/run/mmsd", - FilterGroups: "*", GatewayEnabled: false, GatewayAddr: net.ParseIP("0.0.0.0"), GatewayPortHTTP: 80, From 960b3130b7002a6284a54d9a2e2bef81af17b8bb Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 14:32:12 +0200 Subject: [PATCH 07/39] adds SIGTERM/SIGQUIT handling --- main.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 3e4a047..2e97758 100644 --- a/main.go +++ b/main.go @@ -33,9 +33,11 @@ import ( "net" "net/http" "os" + "os/signal" "path/filepath" "sort" "strings" + "syscall" "time" "github.com/dawanda/go-mesos/marathon" @@ -479,7 +481,13 @@ func (mmsd *mmsdService) Run() { <-mmsd.quitChannel - mmsd.shutdown() + for _, handler := range mmsd.Handlers { + handler.Shutdown() + } +} + +func (mmsd *mmsdService) Shutdown() { + mmsd.quitChannel <- true } func (mmsd *mmsdService) applyApps(apps []AppCluster) { @@ -546,12 +554,6 @@ func (mmsd *mmsdService) setupHandlers() { } } -func (mmsd *mmsdService) shutdown() { - for _, handler := range mmsd.Handlers { - handler.Shutdown() - } -} - func locateExe(name string) string { for _, prefix := range strings.Split(os.Getenv("PATH"), ":") { path := filepath.Join(prefix, name) @@ -592,5 +594,14 @@ func main() { quitChannel: make(chan bool), } + // trap SIGTERM and SIGINT + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGTERM, syscall.SIGINT) + go func() { + s := <-sigc + log.Printf("Caught signal %v. Terminating", s) + mmsd.Shutdown() + }() + mmsd.Run() } From 249718547d6dc2f23f38a64d2deb04b10dd902be Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 14:32:54 +0200 Subject: [PATCH 08/39] code cleanup --- main.go | 58 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/main.go b/main.go index 2e97758..8ce4b6d 100644 --- a/main.go +++ b/main.go @@ -335,38 +335,42 @@ func (mmsd *mmsdService) setupEventBusListener() { var sse = NewEventSource(url, mmsd.ReconnectDelay) - sse.OnOpen = func(event, data string) { - log.Printf("Listening for events from Marathon on %v\n", url) + sse.OnOpen = mmsd.OnMarathonConnected + sse.OnError = mmsd.OnMarathonConnectionFailure + sse.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) + sse.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) - mmsd.applyApps(mmsd.convertMarathonApps(mmsd.getMarathonApps())) - } + go sse.RunForever() +} - sse.OnError = func(event, data string) { - log.Printf("Marathon Event Stream Error. %v. %v\n", event, data) - } +func (mmsd *mmsdService) OnMarathonConnected(event, data string) { + log.Printf("Listening for events from Marathon on %v:%v\n", mmsd.MarathonIP, mmsd.MarathonPort) + mmsd.applyApps(mmsd.convertMarathonApps(mmsd.getMarathonApps())) +} - sse.AddEventListener("status_update_event", func(data string) { - var event marathon.StatusUpdateEvent - err := json.Unmarshal([]byte(data), &event) - if err != nil { - log.Printf("Failed to unmarshal status_update_event. %v\n", err) - log.Printf("status_update_event: %+v\n", data) - } else { - mmsd.statusUpdateEvent(&event) - } - }) +func (mmsd *mmsdService) OnMarathonConnectionFailure(event, data string) { + log.Printf("Marathon Event Stream Error. %v. %v\n", event, data) +} - sse.AddEventListener("health_status_changed_event", func(data string) { - var event marathon.HealthStatusChangedEvent - err := json.Unmarshal([]byte(data), &event) - if err != nil { - log.Printf("Failed to unmarshal health_status_changed_event. %v\n", err) - } else { - mmsd.healthStatusChangedEvent(&event) - } - }) +func (mmsd *mmsdService) StatusUpdateEvent(data string) { + var event marathon.StatusUpdateEvent + err := json.Unmarshal([]byte(data), &event) + if err != nil { + log.Printf("Failed to unmarshal status_update_event. %v\n", err) + log.Printf("status_update_event: %+v\n", data) + } else { + mmsd.statusUpdateEvent(&event) + } +} - go sse.RunForever() +func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { + var event marathon.HealthStatusChangedEvent + err := json.Unmarshal([]byte(data), &event) + if err != nil { + log.Printf("Failed to unmarshal health_status_changed_event. %v\n", err) + } else { + mmsd.healthStatusChangedEvent(&event) + } } func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { From 4e2ec33062ab48ad2537a3b1bcabf3f812231daa Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 14:33:03 +0200 Subject: [PATCH 09/39] comments be comments --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 136f082..5b0f680 100644 --- a/app.go +++ b/app.go @@ -10,7 +10,7 @@ type AppCluster struct { PortName string Labels map[string]string HealthCheck *AppHealthCheck - Backends []AppBackend + Backends []AppBackend // TODO ensured to be always ordered } type AppHealthCheck struct { From 5f2dcf985c99982821505c9d157cafef1c4e981a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 14:33:18 +0200 Subject: [PATCH 10/39] moved isGroupIncluded into the core. so out of here --- haproxy_manager.go | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/haproxy_manager.go b/haproxy_manager.go index f2f03df..9310922 100644 --- a/haproxy_manager.go +++ b/haproxy_manager.go @@ -315,24 +315,13 @@ func (manager *HaproxyMgr) makeConfig(app *marathon.App) (string, error) { manager.appLabels[app.Id][k] = v } - for portIndex, portDef := range app.PortDefinitions { - if manager.isGroupIncluded(portDef.Labels["lb-group"]) { - result += manager.makeConfigForPort(app, portIndex) - } + for portIndex, _ := range app.PortDefinitions { + result += manager.makeConfigForPort(app, portIndex) } return result, nil } -func (manager *HaproxyMgr) isGroupIncluded(groupName string) bool { - for _, filterGroup := range manager.FilterGroups { - if groupName == filterGroup || filterGroup == "*" { - return true - } - } - return false -} - func (manager *HaproxyMgr) makeConfigForPort(app *marathon.App, portIndex int) string { if GetTransportProtocol(app, portIndex) != "tcp" { return "" From b229999240c85ad66ada8bb3130b32b8077c5add Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:16:13 +0200 Subject: [PATCH 11/39] initial porting of haproxy module (into its own new file) --- event_listener.go | 1 + haproxy_manager.go | 86 -------- helper.go | 37 ++++ main.go | 37 +++- mod_haproxy.go | 525 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 590 insertions(+), 96 deletions(-) create mode 100644 mod_haproxy.go diff --git a/event_listener.go b/event_listener.go index 3111b6f..4eaabca 100644 --- a/event_listener.go +++ b/event_listener.go @@ -5,6 +5,7 @@ type EventListener interface { Shutdown() Apply(apps []AppCluster) + AddTask(task AppBackend, app AppCluster) RemoveTask(task AppBackend, app AppCluster) } diff --git a/haproxy_manager.go b/haproxy_manager.go index 9310922..0f0ea95 100644 --- a/haproxy_manager.go +++ b/haproxy_manager.go @@ -61,77 +61,6 @@ var ( ErrBadExit = errors.New("Bad Process Exit.") ) -// {{{ SortedVhostsKeys -type sortedVhosts struct { - m map[string][]string - s []string -} - -func (sm *sortedVhosts) Len() int { - return len(sm.m) -} - -func (sm *sortedVhosts) Less(a, b int) bool { - // return sm.m[sm.s[a]] > sm.m[sm.s[b]] - return sm.s[a] < sm.s[b] -} - -func (sm *sortedVhosts) Swap(a, b int) { - sm.s[a], sm.s[b] = sm.s[b], sm.s[a] -} - -func SortedVhostsKeys(m map[string][]string) []string { - sm := new(sortedVhosts) - sm.m = m - sm.s = make([]string, len(m)) - - i := 0 - for key, _ := range m { - sm.s[i] = key - i++ - } - sort.Sort(sm) - - return sm.s -} - -// }}} -// {{{ SortedStrStrKeys -type sortedStrStrKeys struct { - m map[string]string - s []string -} - -func (sm *sortedStrStrKeys) Len() int { - return len(sm.m) -} - -func (sm *sortedStrStrKeys) Less(a, b int) bool { - // return sm.m[sm.s[a]] > sm.m[sm.s[b]] - return sm.s[a] < sm.s[b] -} - -func (sm *sortedStrStrKeys) Swap(a, b int) { - sm.s[a], sm.s[b] = sm.s[b], sm.s[a] -} - -func SortedStrStrKeys(m map[string]string) []string { - sm := new(sortedStrStrKeys) - sm.m = m - sm.s = make([]string, len(m)) - - i := 0 - for key, _ := range m { - sm.s[i] = key - i++ - } - sort.Sort(sm) - - return sm.s -} - -// }}} - func makeStringArray(s string) []string { if len(s) == 0 { return []string{} @@ -140,21 +69,6 @@ func makeStringArray(s string) []string { } } -func (manager *HaproxyMgr) Startup() { -} - -func (manager *HaproxyMgr) Shutdown() { -} - -func (manager *HaproxyMgr) Apply(apps []AppCluster) { -} - -func (manager *HaproxyMgr) AddTask(task AppBackend, app AppCluster) { -} - -func (manager *HaproxyMgr) RemoveTask(task AppBackend, app AppCluster) { -} - func (manager *HaproxyMgr) OLD_Apply(apps []*marathon.App, force bool) error { manager.appConfigFragments = make(map[string]string) manager.clearAppStateCache() diff --git a/helper.go b/helper.go index b1a733f..dd4762a 100644 --- a/helper.go +++ b/helper.go @@ -9,6 +9,7 @@ import ( "log" "net" "os" + "sort" "strconv" "strings" @@ -259,3 +260,39 @@ func parseRange(input string) (int, int, error) { return begin, end, err } + +// {{{ SortedStrStrKeys +type sortedStrStrKeys struct { + m map[string]string + s []string +} + +func (sm *sortedStrStrKeys) Len() int { + return len(sm.m) +} + +func (sm *sortedStrStrKeys) Less(a, b int) bool { + // return sm.m[sm.s[a]] > sm.m[sm.s[b]] + return sm.s[a] < sm.s[b] +} + +func (sm *sortedStrStrKeys) Swap(a, b int) { + sm.s[a], sm.s[b] = sm.s[b], sm.s[a] +} + +func SortedStrStrKeys(m map[string]string) []string { + sm := new(sortedStrStrKeys) + sm.m = m + sm.s = make([]string, len(m)) + + i := 0 + for key, _ := range m { + sm.s[i] = key + i++ + } + sort.Sort(sm) + + return sm.s +} + +// }}} diff --git a/main.go b/main.go index 8ce4b6d..d7316bc 100644 --- a/main.go +++ b/main.go @@ -100,7 +100,8 @@ type mmsdService struct { DnsPushSRV bool // runtime state - apps []AppCluster + apps []AppCluster + killingTasks map[string]bool // set of tasks currently in killing state } // {{{ HTTP endpoint @@ -268,6 +269,7 @@ func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { return app, nil } +// convertMarathonApps converts an array of marathon.App into a []AppCluster. func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { var apps []AppCluster for _, mApp := range mApps { @@ -337,6 +339,7 @@ func (mmsd *mmsdService) setupEventBusListener() { sse.OnOpen = mmsd.OnMarathonConnected sse.OnError = mmsd.OnMarathonConnectionFailure + //sse.AddEventListener("deployment_info", mmsd.DeploymentStart) sse.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) sse.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) @@ -397,14 +400,27 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { } } } - case marathon.TaskFinished, marathon.TaskFailed, marathon.TaskKilling, marathon.TaskKilled, marathon.TaskLost: + case marathon.TaskKilling: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) - for _, app := range mmsd.findAppsByMarathonId(event.AppId) { - for _, task := range app.Backends { - if task.Id == event.TaskId { - for _, handler := range mmsd.Handlers { - handler.RemoveTask(task, app) - } + mmsd.killingTasks[event.TaskId] = true + mmsd.RemoveTask(event.AppId, event.TaskId, event.TaskStatus) + case marathon.TaskFinished, marathon.TaskFailed, marathon.TaskKilled, marathon.TaskLost: + log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) + if !mmsd.killingTasks[event.TaskId] { + mmsd.RemoveTask(event.AppId, event.TaskId, event.TaskStatus) + } else { + delete(mmsd.killingTasks, event.TaskId) + } + } +} + +func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.TaskStatus) { + for _, app := range mmsd.findAppsByMarathonId(appId) { + for _, task := range app.Backends { + task.State = string(newStatus) + if task.Id == taskId { + for _, handler := range mmsd.Handlers { + handler.RemoveTask(task, app) } } } @@ -527,7 +543,7 @@ func (mmsd *mmsdService) setupHandlers() { // } if mmsd.TCPEnabled { - mmsd.Handlers = append(mmsd.Handlers, &HaproxyMgr{ + mmsd.Handlers = append(mmsd.Handlers, &HaproxyModule{ Verbose: mmsd.Verbose, LocalHealthChecks: mmsd.LocalHealthChecks, ServiceAddr: mmsd.ServiceAddr, @@ -535,7 +551,7 @@ func (mmsd *mmsdService) setupHandlers() { GatewayAddr: mmsd.GatewayAddr, GatewayPortHTTP: mmsd.GatewayPortHTTP, GatewayPortHTTPS: mmsd.GatewayPortHTTPS, - Executable: mmsd.HaproxyBin, + HaproxyExe: mmsd.HaproxyBin, ConfigTailPath: mmsd.HaproxyTailCfg, ConfigPath: filepath.Join(mmsd.RunStateDir, "haproxy.cfg"), OldConfigPath: filepath.Join(mmsd.RunStateDir, "haproxy.cfg.old"), @@ -596,6 +612,7 @@ func main() { DnsBaseName: "mmsd.", DnsTTL: time.Second * 5, quitChannel: make(chan bool), + killingTasks: make(map[string]bool), } // trap SIGTERM and SIGINT diff --git a/mod_haproxy.go b/mod_haproxy.go new file mode 100644 index 0000000..f9fa378 --- /dev/null +++ b/mod_haproxy.go @@ -0,0 +1,525 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net" + "os" + "os/exec" + "runtime" + "sort" + "strconv" + "strings" + "syscall" +) + +const ( +// LB_PROXY_PROTOCOL = "lb-proxy-protocol" +// LB_ACCEPT_PROXY = "lb-accept-proxy" + +// LB_VHOST_HTTP = "lb-vhost" +// LB_VHOST_DEFAULT_HTTP = "lb-vhost-default" +// LB_VHOST_HTTPS = "lb-vhost-ssl" +// LB_VHOST_DEFAULT_HTTPS = "lb-vhost-default-ssl" +) + +// var ( +// ErrBadExit = errors.New("Bad Process Exit.") +// ) + +type HaproxyModule struct { + Verbose bool // log verbose logging messages? + LocalHealthChecks bool // wether or not to perform local health checks or rely on Marathon events. + ServiceAddr net.IP // IP address the load balancers should bind on. + GatewayEnabled bool // Enables/disables HTTP/S gateway support + GatewayAddr net.IP // HTTP gateway bind IP address + GatewayPortHTTP uint // HTTP gateway HTTP port (usually 80) + GatewayPortHTTPS uint // HTTP gateway HTTPS port (usually 443) + HaproxyExe string // path to haproxy executable + ConfigPath string // path to generated config file (/var/run/mmsd/haproxy.cfg) + ConfigTailPath string // path to appended haproxy config file (/etc/mmsd/haproxy-tail.cfg) + OldConfigPath string // path to previousely generated config file + PidFile string // path to PID-file for currently running haproxy executable + ManagementAddr net.IP // haproxy stats listen addr + ManagementPort uint // haproxy stats port + AdminSockPath string // path to haproxy admin socket + + vhostsHTTP map[string][]string // [appId] = []vhost + vhostDefaultHTTP string // default HTTP vhost + vhostsHTTPS map[string][]string // [appId] = []vhost + vhostDefaultHTTPS string // default HTTPS vhost + appConfigCache map[string]string // [appId] = haproxy_config_fragment +} + +func (module *HaproxyModule) Startup() { + log.Printf("Haproxy starting") + + module.vhostsHTTP = make(map[string][]string) + module.vhostDefaultHTTP = "" + + module.vhostsHTTPS = make(map[string][]string) + module.vhostDefaultHTTPS = "" +} + +func (module *HaproxyModule) Shutdown() { + log.Printf("Haproxy shutting down") +} + +func (module *HaproxyModule) Apply(apps []AppCluster) { + log.Printf("Haproxy: apply([]AppCluster)") + + for _, app := range apps { + module.appConfigCache[app.Id] = module.makeConfig(app) + } + + err := module.writeConfig() + if err != nil { + log.Printf("Failed to write config. %v", err) + return + } + + err = module.reloadConfig() + if err != nil { + log.Printf("Reloading configuration failed. %v", err) + } +} + +func (module *HaproxyModule) AddTask(task AppBackend, app AppCluster) { + log.Printf("Haproxy: AddTask()") +} + +func (module *HaproxyModule) RemoveTask(task AppBackend, app AppCluster) { + log.Printf("Haproxy: RemoveTask()") +} + +func (module *HaproxyModule) makeConfig(app AppCluster) string { + result := "" + serverOpts := "" + bindOpts := "" + + if runtime.GOOS == "linux" { // only enable on Linux (known to work) + bindOpts += " defer-accept" + } + + if Atoi(app.Labels[LB_ACCEPT_PROXY], 0) != 0 { + bindOpts += " accept-proxy" + } + + if module.LocalHealthChecks { + serverOpts += " check" + } + + if app.HealthCheck != nil && app.HealthCheck.IntervalSeconds > 0 { + serverOpts += fmt.Sprintf(" inter %v", app.HealthCheck.IntervalSeconds*1000) + } + + switch Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { + case 2: + serverOpts += " send-proxy-v2" + case 1: + serverOpts += " send-proxy" + case 0: + // ignore + default: + log.Printf("Invalid proxy-protocol given for %v: %v - ignoring.", + app.Id, app.Labels["lb-proxy-protocol"]) + } + + switch Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { + case 2: + serverOpts += " send-proxy-v2" + case 1: + serverOpts += " send-proxy" + case 0: + // ignore + default: + log.Printf("Invalid proxy-protocol given for %v: %v - ignoring.", + app.Id, app.Labels["lb-proxy-protocol"]) + } + + var appProtocol = app.Protocol + switch appProtocol { + case "http": + result += fmt.Sprintf( + "frontend __frontend_%v\n"+ + " bind %v:%v%v\n"+ + " option dontlognull\n"+ + " default_backend %v\n"+ + "\n"+ + "backend %v\n"+ + " mode http\n"+ + " balance leastconn\n"+ + " option forwardfor\n"+ + " option http-server-close\n"+ + " option abortonclose\n", + app.Id, module.ServiceAddr, app.ServicePort, bindOpts, app.Id, app.Id) + + if module.LocalHealthChecks && app.HealthCheck != nil { + result += fmt.Sprintf( + " option httpchk GET %v HTTP/1.1\\r\\nHost:\\ %v\n", + app.HealthCheck.Path, "health-check") + } + default: + result += fmt.Sprintf( + "listen %v\n"+ + " bind %v:%v%v\n"+ + " option dontlognull\n"+ + " mode tcp\n"+ + " balance leastconn\n", + app.Id, module.ServiceAddr, app.ServicePort, bindOpts) + } + + result += " option redispatch\n" + result += " retries 1\n" + + for _, task := range app.Backends { + if task.State == "TASK_RUNNING" { + result += fmt.Sprintf( + " server %v:%v %v:%v%v\n", + task.Host, task.Port, // taskLabel == "$host:$port" + SoftResolveIPAddr(task.Host), + task.Port, + serverOpts) + } else { + log.Printf("Haproxy: skip task %v (%v).", task.Id, task.State) + } + } + + result += "\n" + + return result +} + +func (module *HaproxyModule) writeConfig() error { + config := "" + + config += module.makeConfigHead() + + // add apps sorted to config + var clusterNames []string + for name, _ := range module.appConfigCache { + clusterNames = append(clusterNames, name) + } + sort.Strings(clusterNames) + for _, name := range clusterNames { + config += module.appConfigCache[name] + } + + config += module.makeConfigTail() + + // write file + tempConfigFile := fmt.Sprintf("%v.tmp", module.ConfigPath) + err := ioutil.WriteFile(tempConfigFile, []byte(config), 0666) + if err != nil { + return err + } + + err = module.checkConfig(tempConfigFile) + if err != nil { + return err + } + + // if config file previousely did exist, attempt a rename + if _, err := os.Stat(module.ConfigPath); err == nil { + if err = os.Rename(module.ConfigPath, module.OldConfigPath); err != nil { + return err + } + } + + return os.Rename(tempConfigFile, module.ConfigPath) +} + +func (module *HaproxyModule) makeConfigHead() string { + config := "" + + config += fmt.Sprintf( + "# This is an auto generated haproxy configuration!!!\n"+ + "global\n"+ + " maxconn 32768\n"+ + " maxconnrate 32768\n"+ + " log 127.0.0.1 local0\n"+ + " stats socket %v mode 600 level admin\n"+ + "\n"+ + "defaults\n"+ + " maxconn 32768\n"+ + " timeout client 90000\n"+ + " timeout server 90000\n"+ + " timeout connect 90000\n"+ + " timeout queue 90000\n"+ + " timeout http-request 90000\n"+ + "\n", module.AdminSockPath) + + config += fmt.Sprintf( + "listen haproxy\n"+ + " bind %v:%v\n"+ + " mode http\n"+ + " stats enable\n"+ + " stats uri /\n"+ + " stats admin if TRUE\n"+ + " monitor-uri /haproxy?monitor\n"+ + "\n", + module.ManagementAddr, module.ManagementPort) + + if module.GatewayEnabled { + config += module.makeGatewayHTTP() + config += module.makeGatewayHTTPS() + } + return config +} + +func (module *HaproxyModule) makeGatewayHTTP() string { + var ( + suffixRoutes map[string]string = make(map[string]string) + suffixMatches []string + exactRoutes map[string]string = make(map[string]string) + exactMatches []string + vhostDefault string + port uint = module.GatewayPortHTTP + ) + + for _, appID := range SortedVhostsKeys(module.vhostsHTTP) { + vhosts := module.vhostsHTTP[appID] + for _i, vhost := range vhosts { + log.Printf("[haproxy] appID:%v, vhost:%v, i:%v\n", appID, vhost, _i) + matchToken := "vhost_" + vhost + matchToken = strings.Replace(matchToken, ".", "_", -1) + matchToken = strings.Replace(matchToken, "*", "STAR", -1) + + if len(vhost) >= 3 && vhost[0] == '*' && vhost[1] == '.' { + suffixMatches = append(suffixMatches, + fmt.Sprintf(" acl %v hdr_dom(host) -i %v\n", matchToken, strings.SplitN(vhost, ".", 2)[1])) + suffixRoutes[matchToken] = appID + } else { + exactMatches = append(exactMatches, + fmt.Sprintf(" acl %v hdr(host) -i %v\n", matchToken, vhost)) + exactRoutes[matchToken] = appID + } + + if module.vhostDefaultHTTP == appID { + vhostDefault = appID + } + } + } + + var fragment string + fragment += fmt.Sprintf( + "frontend __gateway_http\n"+ + " bind %v:%v\n"+ + " mode http\n"+ + " option httplog\n"+ + " option dontlognull\n"+ + " option forwardfor\n"+ + " option http-server-close\n"+ + " reqadd X-Forwarded-Proto:\\ http\n"+ + "\n", + module.GatewayAddr, + port) + + // write ACL statements + fragment += strings.Join(exactMatches, "") + fragment += strings.Join(suffixMatches, "") + if len(exactMatches) != 0 || len(suffixMatches) != 0 { + fragment += "\n" + } + + for _, acl := range SortedStrStrKeys(exactRoutes) { + appID := exactRoutes[acl] + fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) + } + + for _, acl := range SortedStrStrKeys(suffixRoutes) { + appID := suffixRoutes[acl] + fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) + } + + fragment += "\n" + + if len(vhostDefault) != 0 { + fragment += fmt.Sprintf(" default_backend %v\n\n", vhostDefault) + } + + return fragment +} + +func (module *HaproxyModule) makeGatewayHTTPS() string { + // SNI vhost selector + var ( + suffixRoutes map[string]string = make(map[string]string) + suffixMatches []string + exactRoutes map[string]string = make(map[string]string) + exactMatches []string + vhostDefault string + port uint = module.GatewayPortHTTPS + ) + + for _, appID := range SortedVhostsKeys(module.vhostsHTTPS) { + vhosts := module.vhostsHTTPS[appID] + for _, vhost := range vhosts { + matchToken := "vhost_ssl_" + vhost + matchToken = strings.Replace(matchToken, ".", "_", -1) + matchToken = strings.Replace(matchToken, "*", "STAR", -1) + + if len(vhost) >= 3 && vhost[0] == '*' && vhost[1] == '.' { + suffixMatches = append(suffixMatches, + fmt.Sprintf(" acl %v req_ssl_sni -m dom %v\n", matchToken, strings.SplitN(vhost, ".", 2)[1])) + suffixRoutes[matchToken] = appID + } else { + exactMatches = append(exactMatches, + fmt.Sprintf(" acl %v req_ssl_sni -i %v\n", matchToken, vhost)) + exactRoutes[matchToken] = appID + } + + if module.vhostDefaultHTTPS == appID { + vhostDefault = appID + } + } + } + + var fragment string + fragment += fmt.Sprintf( + "frontend __gateway_https\n"+ + " bind %v:%v\n"+ + " mode tcp\n"+ + " tcp-request inspect-delay 5s\n"+ + " tcp-request content accept if { req_ssl_hello_type 1 }\n"+ + "\n", + module.GatewayAddr, + port) + + // write ACL statements + fragment += strings.Join(exactMatches, "") + fragment += strings.Join(suffixMatches, "") + if len(exactMatches) != 0 || len(suffixMatches) != 0 { + fragment += "\n" + } + + for _, acl := range SortedStrStrKeys(exactRoutes) { + appID := exactRoutes[acl] + fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) + } + + for _, acl := range SortedStrStrKeys(suffixRoutes) { + appID := suffixRoutes[acl] + fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) + } + + fragment += "\n" + + if len(vhostDefault) != 0 { + fragment += fmt.Sprintf(" default_backend %v\n\n", vhostDefault) + } + + return fragment +} + +func (module *HaproxyModule) makeConfigTail() string { + if len(module.ConfigTailPath) == 0 { + return "" + } + + tail, err := ioutil.ReadFile(module.ConfigTailPath) + if err != nil { + log.Printf("Failed to read config tail %v. %v", module.ConfigTailPath, err) + return "" + } + + return string(tail) +} + +func (module *HaproxyModule) reloadConfig() error { + if FileIsIdentical(module.ConfigPath, module.OldConfigPath) { + log.Printf("[haproxy] config file not changed. ignoring reload\n") + return nil + } + + pidStr, err := ioutil.ReadFile(module.PidFile) + if err != nil { + return module.startProcess() + } + + pid, err := strconv.Atoi(strings.TrimSpace(string(pidStr))) + if err != nil { + return err + } + + err = syscall.Kill(pid, syscall.Signal(0)) + if err != nil { + // process doesn't exist; start up process + return module.startProcess() + } else { + // process does exist; send SIGHUP to reload + return module.reloadProcess(pid) + } +} + +func (module *HaproxyModule) checkConfig(path string) error { + return module.exec("checking configuration", + "-f", path, "-p", module.PidFile, "-c") +} + +func (module *HaproxyModule) startProcess() error { + return module.exec("starting up process", + "-f", module.ConfigPath, "-p", module.PidFile, "-D", "-q") +} + +func (module *HaproxyModule) reloadProcess(pid int) error { + return module.exec("reloading configuration", + "-f", module.ConfigPath, "-p", module.PidFile, "-D", "-sf", fmt.Sprint(pid)) +} + +func (module *HaproxyModule) exec(logMessage string, args ...string) error { + proc := exec.Command(module.HaproxyExe, args...) + output, err := proc.CombinedOutput() + + log.Printf("[haproxy] %v: %v %v\n", logMessage, module.HaproxyExe, args) + + exitCode := proc.ProcessState.Sys().(syscall.WaitStatus) + if exitCode != 0 { + log.Printf("[haproxy] Bad exit code %v.\n", exitCode) + err = ErrBadExit + } + + if len(output) != 0 && module.Verbose { + log.Println("[haproxy] command output:") + log.Println(strings.TrimSpace(string(output))) + } + + return err +} + +// {{{ SortedVhostsKeys +type sortedVhosts struct { + m map[string][]string + s []string +} + +func (sm *sortedVhosts) Len() int { + return len(sm.m) +} + +func (sm *sortedVhosts) Less(a, b int) bool { + // return sm.m[sm.s[a]] > sm.m[sm.s[b]] + return sm.s[a] < sm.s[b] +} + +func (sm *sortedVhosts) Swap(a, b int) { + sm.s[a], sm.s[b] = sm.s[b], sm.s[a] +} + +func SortedVhostsKeys(m map[string][]string) []string { + sm := new(sortedVhosts) + sm.m = m + sm.s = make([]string, len(m)) + + i := 0 + for key, _ := range m { + sm.s[i] = key + i++ + } + sort.Sort(sm) + + return sm.s +} + +// }}} From eaa5e53279ad27275131be16cf31aa0baf50d167 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:23:05 +0200 Subject: [PATCH 12/39] haproxy: adds gw settings support, more cleanups --- haproxy_manager.go | 22 ------------------- helper.go | 8 +++++++ mod_haproxy.go | 53 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 32 deletions(-) diff --git a/haproxy_manager.go b/haproxy_manager.go index 0f0ea95..096b025 100644 --- a/haproxy_manager.go +++ b/haproxy_manager.go @@ -6,7 +6,6 @@ package main // TODO: support local-health checks *or* marathon-based health check propagation (--local-health-checks=false) import ( - "errors" "fmt" "io/ioutil" "log" @@ -48,27 +47,6 @@ type HaproxyMgr struct { vhostDefaultHTTPS string } -const ( - LB_PROXY_PROTOCOL = "lb-proxy-protocol" - LB_ACCEPT_PROXY = "lb-accept-proxy" - LB_VHOST_HTTP = "lb-vhost" - LB_VHOST_DEFAULT_HTTP = "lb-vhost-default" - LB_VHOST_HTTPS = "lb-vhost-ssl" - LB_VHOST_DEFAULT_HTTPS = "lb-vhost-default-ssl" -) - -var ( - ErrBadExit = errors.New("Bad Process Exit.") -) - -func makeStringArray(s string) []string { - if len(s) == 0 { - return []string{} - } else { - return strings.Split(s, ",") - } -} - func (manager *HaproxyMgr) OLD_Apply(apps []*marathon.App, force bool) error { manager.appConfigFragments = make(map[string]string) manager.clearAppStateCache() diff --git a/helper.go b/helper.go index dd4762a..c9e157b 100644 --- a/helper.go +++ b/helper.go @@ -261,6 +261,14 @@ func parseRange(input string) (int, int, error) { return begin, end, err } +func makeStringArray(s string) []string { + if len(s) == 0 { + return []string{} + } else { + return strings.Split(s, ",") + } +} + // {{{ SortedStrStrKeys type sortedStrStrKeys struct { m map[string]string diff --git a/mod_haproxy.go b/mod_haproxy.go index f9fa378..2d25c01 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "io/ioutil" "log" @@ -15,18 +16,17 @@ import ( ) const ( -// LB_PROXY_PROTOCOL = "lb-proxy-protocol" -// LB_ACCEPT_PROXY = "lb-accept-proxy" - -// LB_VHOST_HTTP = "lb-vhost" -// LB_VHOST_DEFAULT_HTTP = "lb-vhost-default" -// LB_VHOST_HTTPS = "lb-vhost-ssl" -// LB_VHOST_DEFAULT_HTTPS = "lb-vhost-default-ssl" + LB_PROXY_PROTOCOL = "lb-proxy-protocol" + LB_ACCEPT_PROXY = "lb-accept-proxy" + LB_VHOST_HTTP = "lb-vhost" + LB_VHOST_DEFAULT_HTTP = "lb-vhost-default" + LB_VHOST_HTTPS = "lb-vhost-ssl" + LB_VHOST_DEFAULT_HTTPS = "lb-vhost-default-ssl" ) -// var ( -// ErrBadExit = errors.New("Bad Process Exit.") -// ) +var ( + ErrBadExit = errors.New("Bad Process Exit.") +) type HaproxyModule struct { Verbose bool // log verbose logging messages? @@ -94,6 +94,9 @@ func (module *HaproxyModule) RemoveTask(task AppBackend, app AppCluster) { } func (module *HaproxyModule) makeConfig(app AppCluster) string { + module.updateGatewaySettings(app) + + // generate haproxy config fragment result := "" serverOpts := "" bindOpts := "" @@ -191,6 +194,36 @@ func (module *HaproxyModule) makeConfig(app AppCluster) string { return result } +func (module *HaproxyModule) updateGatewaySettings(app AppCluster) { + // update HTTP virtual hosting + var lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTP]) + if len(lbVirtualHosts) != 0 { + module.vhostsHTTP[app.Id] = lbVirtualHosts + if app.Labels[LB_VHOST_DEFAULT_HTTP] == "1" { + module.vhostDefaultHTTP = app.Id + } + } else { + delete(module.vhostsHTTP, app.Id) + if module.vhostDefaultHTTP == app.Id { + module.vhostDefaultHTTP = "" + } + } + + // update HTTPS virtual hosting + lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTPS]) + if len(lbVirtualHosts) != 0 { + module.vhostsHTTPS[app.Id] = lbVirtualHosts + if app.Labels[LB_VHOST_DEFAULT_HTTPS] == "1" { + module.vhostDefaultHTTPS = app.Id + } + } else { + delete(module.vhostsHTTPS, app.Id) + if module.vhostDefaultHTTPS == app.Id { + module.vhostDefaultHTTPS = "" + } + } +} + func (module *HaproxyModule) writeConfig() error { config := "" From 629e3a4e863856dddf8c62bcdc1c752bed18b642 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:23:57 +0200 Subject: [PATCH 13/39] haproxy: filter by protocol (tcp/http) --- mod_haproxy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mod_haproxy.go b/mod_haproxy.go index 2d25c01..861fd70 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -70,7 +70,9 @@ func (module *HaproxyModule) Apply(apps []AppCluster) { log.Printf("Haproxy: apply([]AppCluster)") for _, app := range apps { - module.appConfigCache[app.Id] = module.makeConfig(app) + if app.Protocol == "TCP" || app.Protocol == "HTTP" { + module.appConfigCache[app.Id] = module.makeConfig(app) + } } err := module.writeConfig() From c9c1efdb5bcb9ec771e8bf690d121a54b8ea3a40 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:27:38 +0200 Subject: [PATCH 14/39] snip --- mod_haproxy.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mod_haproxy.go b/mod_haproxy.go index 861fd70..39a9305 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -70,7 +70,7 @@ func (module *HaproxyModule) Apply(apps []AppCluster) { log.Printf("Haproxy: apply([]AppCluster)") for _, app := range apps { - if app.Protocol == "TCP" || app.Protocol == "HTTP" { + if module.supportsProtocol(app.Protocol) { module.appConfigCache[app.Id] = module.makeConfig(app) } } @@ -88,6 +88,9 @@ func (module *HaproxyModule) Apply(apps []AppCluster) { } func (module *HaproxyModule) AddTask(task AppBackend, app AppCluster) { + if !module.supportsProtocol(app.Protocol) { + return + } log.Printf("Haproxy: AddTask()") } @@ -523,6 +526,15 @@ func (module *HaproxyModule) exec(logMessage string, args ...string) error { return err } +func (module *HaproxyModule) supportsProtocol(proto string) bool { + if proto == "TCP" || proto == "HTTP" { + return true + } else { + log.Printf("Haproxy: Protocol not supported: %v") + return false + } +} + // {{{ SortedVhostsKeys type sortedVhosts struct { m map[string][]string From 8a6aae30d511c198b2b605f0d05e08c558aeed85 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:30:55 +0200 Subject: [PATCH 15/39] core: also import main apps labels and then port based labels on top (potentially overriding main labels) --- main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index d7316bc..e617615 100644 --- a/main.go +++ b/main.go @@ -304,13 +304,21 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster }) } + labels := make(map[string]string) + for k, v := range mApp.Labels { + labels[k] = v + } + for k, v := range mApp.PortDefinitions[portIndex].Labels { + labels[k] = v + } + app := AppCluster{ Name: mApp.Id, Id: PrettifyAppId2(mApp.Id, portIndex), ServicePort: mApp.PortDefinitions[portIndex].Port, Protocol: mApp.PortDefinitions[portIndex].Protocol, PortName: mApp.PortDefinitions[portIndex].Name, - Labels: mApp.PortDefinitions[portIndex].Labels, + Labels: labels, HealthCheck: healthCheck, Backends: backends, } From 5c85d6465ef276f1a688b975b986d383892c9c57 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 17:54:21 +0200 Subject: [PATCH 16/39] haproxy: got it working. --- mod_haproxy.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mod_haproxy.go b/mod_haproxy.go index 39a9305..6db037a 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -60,6 +60,8 @@ func (module *HaproxyModule) Startup() { module.vhostsHTTPS = make(map[string][]string) module.vhostDefaultHTTPS = "" + + module.appConfigCache = make(map[string]string) } func (module *HaproxyModule) Shutdown() { @@ -146,9 +148,9 @@ func (module *HaproxyModule) makeConfig(app AppCluster) string { app.Id, app.Labels["lb-proxy-protocol"]) } - var appProtocol = app.Protocol + var appProtocol = module.getAppProtocol(app) switch appProtocol { - case "http": + case "HTTP": result += fmt.Sprintf( "frontend __frontend_%v\n"+ " bind %v:%v%v\n"+ @@ -345,7 +347,6 @@ func (module *HaproxyModule) makeGatewayHTTP() string { "frontend __gateway_http\n"+ " bind %v:%v\n"+ " mode http\n"+ - " option httplog\n"+ " option dontlognull\n"+ " option forwardfor\n"+ " option http-server-close\n"+ @@ -526,11 +527,23 @@ func (module *HaproxyModule) exec(logMessage string, args ...string) error { return err } +func (module *HaproxyModule) getAppProtocol(app AppCluster) string { + if app.HealthCheck != nil { + if app.HealthCheck.Protocol != "COMMAND" { + return strings.ToUpper(app.HealthCheck.Protocol) + } + } + + return strings.ToUpper(app.Protocol) +} + func (module *HaproxyModule) supportsProtocol(proto string) bool { + proto = strings.ToUpper(proto) + if proto == "TCP" || proto == "HTTP" { return true } else { - log.Printf("Haproxy: Protocol not supported: %v") + log.Printf("Haproxy: Protocol not supported: %v", proto) return false } } From cb6e63bd22cb0702eaefe04a5034a1ad788d2faa Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 18:59:50 +0200 Subject: [PATCH 17/39] do pointers for AppCluster and related & streamline file naming. --- event_listener.go | 6 +- haproxy_manager.go | 729 -------------------------- main.go | 38 +- event_logger.go => mod_eventlogger.go | 14 +- mod_haproxy.go | 15 +- 5 files changed, 38 insertions(+), 764 deletions(-) delete mode 100644 haproxy_manager.go rename event_logger.go => mod_eventlogger.go (57%) diff --git a/event_listener.go b/event_listener.go index 4eaabca..dcd0620 100644 --- a/event_listener.go +++ b/event_listener.go @@ -4,8 +4,8 @@ type EventListener interface { Startup() Shutdown() - Apply(apps []AppCluster) + Apply(apps []*AppCluster) - AddTask(task AppBackend, app AppCluster) - RemoveTask(task AppBackend, app AppCluster) + AddTask(task *AppBackend, app *AppCluster) + RemoveTask(task *AppBackend, app *AppCluster) } diff --git a/haproxy_manager.go b/haproxy_manager.go deleted file mode 100644 index 096b025..0000000 --- a/haproxy_manager.go +++ /dev/null @@ -1,729 +0,0 @@ -package main - -// TODO: sort cluster names in cfg output -// TODO: sort backend names in cfg output -// TODO: avoid spam-reloading the haproxy binary (due to massive scaling) -// TODO: support local-health checks *or* marathon-based health check propagation (--local-health-checks=false) - -import ( - "fmt" - "io/ioutil" - "log" - "net" - "os" - "os/exec" - "runtime" - "sort" - "strconv" - "strings" - "syscall" - - "github.com/dawanda/go-mesos/marathon" -) - -type HaproxyMgr struct { - Verbose bool - LocalHealthChecks bool - FilterGroups []string - ServiceAddr net.IP - GatewayEnabled bool - GatewayAddr net.IP - GatewayPortHTTP uint - GatewayPortHTTPS uint - Executable string - ConfigPath string - ConfigTailPath string - OldConfigPath string - PidFile string - ManagementAddr net.IP - ManagementPort uint - AdminSockPath string - appConfigFragments map[string]string // [appId] = haproxy_config_fragment - appLabels map[string]map[string]string // [appId][key] = value - appStateCache map[string]map[string]*marathon.Task // [appId][task] = Task - vhosts map[string][]string // [appId] = []vhost - vhostDefault string - vhostsHTTPS map[string][]string - vhostDefaultHTTPS string -} - -func (manager *HaproxyMgr) OLD_Apply(apps []*marathon.App, force bool) error { - manager.appConfigFragments = make(map[string]string) - manager.clearAppStateCache() - - manager.vhosts = make(map[string][]string) - manager.vhostDefault = "" - - manager.vhostsHTTPS = make(map[string][]string) - manager.vhostDefaultHTTPS = "" - - for _, app := range apps { - config, err := manager.makeConfig(app) - if err != nil { - return err - } - manager.appConfigFragments[app.Id] = config - for _, task := range app.Tasks { - manager.setAppStateCacheEntry(&task) - } - } - - err := manager.writeConfig() - if err != nil { - return err - } - - return manager.reloadConfig(false) -} - -func (manager *HaproxyMgr) OLD_Remove(appID string, taskID string, app *marathon.App) error { - if app != nil { - // make sure we *remove* the task from the cluster - config, err := manager.makeConfig(app) - if err != nil { - return err - } - if len(app.Tasks) > 0 { - // app removed one task, still at least one alive - manager.appConfigFragments[appID] = config - } else { - // app suspended (or scaled down to zero) - delete(manager.appConfigFragments, appID) - } - } else { - // app destroyed fully - delete(manager.appConfigFragments, appID) - } - manager.removeAppStateCacheEntry(appID, taskID) - - err := manager.writeConfig() - if err != nil { - return err - } - - return manager.reloadConfig(false) -} - -func isAppJustSpawned(app *marathon.App) bool { - return false // XXX removed -} - -func (manager *HaproxyMgr) OLD_Update(app *marathon.App, taskID string) error { - // collect list of task labels as we formatted them in haproxy.cfg. - var instanceNames []string - for portIndex, servicePort := range app.Ports { - if GetTransportProtocol(app, portIndex) == "tcp" { - appID := PrettifyAppId(app.Id, portIndex, servicePort) - cachedTask := manager.getAppStateCacheEntry(app.Id, taskID) - if cachedTask == nil { - cachedTask = app.GetTaskById(taskID) - } - if cachedTask != nil { - cachedTaskLabel := fmt.Sprintf("%v/%v:%v", appID, cachedTask.Host, cachedTask.Ports[portIndex]) - instanceNames = append(instanceNames, cachedTaskLabel) - } - } - } - - config, err := manager.makeConfig(app) - if err != nil { - return err - } - - manager.appConfigFragments[app.Id] = config - for _, task := range app.Tasks { - manager.setAppStateCacheEntry(&task) - } - - err = manager.writeConfig() - if err != nil { - return err - } - - task := app.GetTaskById(taskID) - - // go right away reload the config if that is the first start of the - // underlying task and we got just health - if task != nil && task.IsAlive() { - // no health checks defined or app got just spawned the first time? - if len(app.HealthChecks) == 0 || isAppJustSpawned(app) { - log.Printf("[haproxy] App %v on host %v becomes healthy (or alive) first time. force reload config.\n", - app.Id, task.Host) - return manager.reloadConfig(true) - } - } - - // upstream server is already present, so send en enable or disable command - // to all app clusters of this name ($app-$portIndex-$servicePort/$taskLabel) - var updateCommandFmt string - if task != nil && task.IsAlive() { - updateCommandFmt = "enable server %v\n" - } else { - updateCommandFmt = "disable server %v\n" - } - - for _, instanceName := range instanceNames { - err := manager.sendCommandf(updateCommandFmt, instanceName) - if err != nil { - return err - } - } - - return nil -} - -func (manager *HaproxyMgr) sendCommandf(cmdFmt string, args ...interface{}) error { - log.Printf("[haproxy] "+cmdFmt, args...) - cmd := fmt.Sprintf(cmdFmt, args...) - conn, err := net.DialUnix("unix", nil, &net.UnixAddr{manager.AdminSockPath, "unix"}) - if err != nil { - return err - } - defer conn.Close() - - _, err = conn.Write([]byte(cmd)) - if err != nil { - return err - } - - var response []byte = make([]byte, 32768) - _, err = conn.Read(response) - if err != nil { - return err - } - - return nil -} - -func (manager *HaproxyMgr) makeConfig(app *marathon.App) (string, error) { - var result string - - // import application labels - if manager.appLabels == nil { - manager.appLabels = make(map[string]map[string]string) - } - manager.appLabels[app.Id] = make(map[string]string) - for k, v := range app.Labels { - manager.appLabels[app.Id][k] = v - } - - for portIndex, _ := range app.PortDefinitions { - result += manager.makeConfigForPort(app, portIndex) - } - - return result, nil -} - -func (manager *HaproxyMgr) makeConfigForPort(app *marathon.App, portIndex int) string { - if GetTransportProtocol(app, portIndex) != "tcp" { - return "" - } - - var portDef = app.PortDefinitions[portIndex] - var servicePort = portDef.Port - var appID = PrettifyAppId(app.Id, portIndex, servicePort) - var bindAddr = manager.ServiceAddr - var healthCheck = GetHealthCheckForPortIndex(app.HealthChecks, portIndex) - var appProtocol = GetApplicationProtocol(app, portIndex) - - var lbVirtualHosts = makeStringArray(portDef.Labels[LB_VHOST_HTTP]) - if len(lbVirtualHosts) != 0 { - manager.vhosts[appID] = lbVirtualHosts - if portDef.Labels[LB_VHOST_DEFAULT_HTTP] == "1" { - manager.vhostDefault = appID - } - } else { - delete(manager.vhosts, appID) - if manager.vhostDefault == appID { - manager.vhostDefault = "" - } - } - - lbVirtualHosts = makeStringArray(portDef.Labels[LB_VHOST_HTTPS]) - if len(lbVirtualHosts) != 0 { - manager.vhostsHTTPS[appID] = lbVirtualHosts - if portDef.Labels[LB_VHOST_DEFAULT_HTTPS] == "1" { - manager.vhostDefaultHTTPS = appID - } - } else { - delete(manager.vhostsHTTPS, appID) - if manager.vhostDefaultHTTPS == appID { - manager.vhostDefaultHTTPS = "" - } - } - - result := "" - bindOpts := "" - - if runtime.GOOS == "linux" { // only enable on Linux (known to work) - bindOpts += " defer-accept" - } - - if Atoi(portDef.Labels[LB_ACCEPT_PROXY], 0) != 0 { - bindOpts += " accept-proxy" - } - - serverOpts := "" - - if manager.LocalHealthChecks { - serverOpts += " check" - } - - if healthCheck.IntervalSeconds > 0 { - serverOpts += fmt.Sprintf(" inter %v", healthCheck.IntervalSeconds*1000) - } - - switch Atoi(portDef.Labels[LB_PROXY_PROTOCOL], 0) { - case 2: - serverOpts += " send-proxy-v2" - case 1: - serverOpts += " send-proxy" - case 0: - // ignore - default: - log.Printf("Invalid proxy-protocol given for %v: %v - ignoring.", - app.Id, app.Labels["lb-proxy-protocol"]) - } - - switch appProtocol { - case "http": - result += fmt.Sprintf( - "frontend __frontend_%v\n"+ - " bind %v:%v%v\n"+ - " option dontlognull\n"+ - " default_backend %v\n"+ - "\n"+ - "backend %v\n"+ - " mode http\n"+ - " balance leastconn\n"+ - " option forwardfor\n"+ - " option http-server-close\n"+ - " option abortonclose\n"+ - " option httpchk GET %v HTTP/1.1\\r\\nHost:\\ %v\n", - appID, bindAddr, servicePort, bindOpts, appID, appID, - healthCheck.Path, "health-check") - case "redis-master", "redis-server", "redis": - result += fmt.Sprintf( - "listen %v\n"+ - " bind %v:%v%v\n"+ - " option dontlognull\n"+ - " mode tcp\n"+ - " balance leastconn\n"+ - " option tcp-check\n"+ - " tcp-check connect\n"+ - " tcp-check send PING\\r\\n\n"+ - " tcp-check expect string +PONG\n"+ - " tcp-check send info\\ replication\\r\\n\n"+ - " tcp-check expect string role:master\n"+ - " tcp-check send QUIT\\r\\n\n"+ - " tcp-check expect string +OK\n", - appID, bindAddr, servicePort, bindOpts) - case "smtp": - result += fmt.Sprintf( - "listen %v\n"+ - " bind %v:%v%v\n"+ - " option dontlognull\n"+ - " mode tcp\n"+ - " balance leastconn\n"+ - " option tcp-check\n"+ - " option smtpchk EHLO localhost\n", - appID, bindAddr, servicePort, bindOpts) - default: - result += fmt.Sprintf( - "listen %v\n"+ - " bind %v:%v%v\n"+ - " option dontlognull\n"+ - " mode tcp\n"+ - " balance leastconn\n", - appID, bindAddr, servicePort, bindOpts) - } - - result += " option redispatch\n" - result += " retries 1\n" - - for _, task := range sortTasks(app.Tasks, portIndex) { - // Include the task iff it is in running state or a state is not provided. - // The latter is kept for backwards compatibility with older - // Marathon services - if task.State == nil || *task.State == marathon.TaskRunning { - result += fmt.Sprintf( - " server %v:%v %v:%v%v\n", - task.Host, task.Ports[portIndex], // taskLabel == "$host:$port" - SoftResolveIPAddr(task.Host), - task.Ports[portIndex], - serverOpts) - } - } - - result += "\n" - - return result -} - -func (manager *HaproxyMgr) writeConfig() error { - config, err := manager.makeConfigHead() - if err != nil { - return err - } - - var clusterNames []string - for name, _ := range manager.appConfigFragments { - clusterNames = append(clusterNames, name) - } - sort.Strings(clusterNames) - for _, name := range clusterNames { - config += manager.appConfigFragments[name] - } - - tail, err := manager.makeConfigTail() - if err != nil { - return err - } - config += tail - - tempConfigFile := fmt.Sprintf("%v.tmp", manager.ConfigPath) - err = ioutil.WriteFile(tempConfigFile, []byte(config), 0666) - if err != nil { - return err - } - - err = manager.checkConfig(tempConfigFile) - if err != nil { - return err - } - - // if config file previousely did exist, attempt a rename - if _, err := os.Stat(manager.ConfigPath); err == nil { - if err = os.Rename(manager.ConfigPath, manager.OldConfigPath); err != nil { - return err - } - } - - return os.Rename(tempConfigFile, manager.ConfigPath) -} - -func (manager *HaproxyMgr) makeConfigHead() (string, error) { - headerFragment := fmt.Sprintf( - "# This is an auto generated haproxy configuration!!!\n"+ - "global\n"+ - " maxconn 32768\n"+ - " maxconnrate 32768\n"+ - " log 127.0.0.1 local0\n"+ - " stats socket %v mode 600 level admin\n"+ - "\n"+ - "defaults\n"+ - " maxconn 32768\n"+ - " timeout client 90000\n"+ - " timeout server 90000\n"+ - " timeout connect 90000\n"+ - " timeout queue 90000\n"+ - " timeout http-request 90000\n"+ - "\n", manager.AdminSockPath) - - mgntFragment := fmt.Sprintf( - "listen haproxy\n"+ - " bind %v:%v\n"+ - " mode http\n"+ - " stats enable\n"+ - " stats uri /\n"+ - " stats admin if TRUE\n"+ - " monitor-uri /haproxy?monitor\n"+ - "\n", - manager.ManagementAddr, manager.ManagementPort) - - if manager.GatewayEnabled { - gatewayHTTP := manager.makeGatewayHTTP() - gatewayHTTPS := manager.makeGatewayHTTPS() - return headerFragment + mgntFragment + gatewayHTTP + gatewayHTTPS, nil - } else { - return headerFragment + mgntFragment, nil - } -} - -func (manager *HaproxyMgr) makeGatewayHTTP() string { - var ( - suffixRoutes map[string]string = make(map[string]string) - suffixMatches []string - exactRoutes map[string]string = make(map[string]string) - exactMatches []string - vhostDefault string - port uint = manager.GatewayPortHTTP - ) - - for _, appID := range SortedVhostsKeys(manager.vhosts) { - vhosts := manager.vhosts[appID] - for _i, vhost := range vhosts { - log.Printf("[haproxy] appID:%v, vhost:%v, i:%v\n", appID, vhost, _i) - matchToken := "vhost_" + vhost - matchToken = strings.Replace(matchToken, ".", "_", -1) - matchToken = strings.Replace(matchToken, "*", "STAR", -1) - - if len(vhost) >= 3 && vhost[0] == '*' && vhost[1] == '.' { - suffixMatches = append(suffixMatches, - fmt.Sprintf(" acl %v hdr_dom(host) -i %v\n", matchToken, strings.SplitN(vhost, ".", 2)[1])) - suffixRoutes[matchToken] = appID - } else { - exactMatches = append(exactMatches, - fmt.Sprintf(" acl %v hdr(host) -i %v\n", matchToken, vhost)) - exactRoutes[matchToken] = appID - } - - if manager.vhostDefault == appID { - vhostDefault = appID - } - } - } - - var fragment string - fragment += fmt.Sprintf( - "frontend __gateway_http\n"+ - " bind %v:%v\n"+ - " mode http\n"+ - " option httplog\n"+ - " option dontlognull\n"+ - " option forwardfor\n"+ - " option http-server-close\n"+ - " reqadd X-Forwarded-Proto:\\ http\n"+ - "\n", - manager.GatewayAddr, - port) - - // write ACL statements - fragment += strings.Join(exactMatches, "") - fragment += strings.Join(suffixMatches, "") - if len(exactMatches) != 0 || len(suffixMatches) != 0 { - fragment += "\n" - } - - for _, acl := range SortedStrStrKeys(exactRoutes) { - appID := exactRoutes[acl] - fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) - } - - for _, acl := range SortedStrStrKeys(suffixRoutes) { - appID := suffixRoutes[acl] - fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) - } - - fragment += "\n" - - if len(vhostDefault) != 0 { - fragment += fmt.Sprintf(" default_backend %v\n\n", vhostDefault) - } - - return fragment -} - -func (manager *HaproxyMgr) makeGatewayHTTPS() string { - // SNI vhost selector - var ( - suffixRoutes map[string]string = make(map[string]string) - suffixMatches []string - exactRoutes map[string]string = make(map[string]string) - exactMatches []string - vhostDefault string - port uint = manager.GatewayPortHTTPS - ) - - for _, appID := range SortedVhostsKeys(manager.vhostsHTTPS) { - vhosts := manager.vhostsHTTPS[appID] - for _, vhost := range vhosts { - matchToken := "vhost_ssl_" + vhost - matchToken = strings.Replace(matchToken, ".", "_", -1) - matchToken = strings.Replace(matchToken, "*", "STAR", -1) - - if len(vhost) >= 3 && vhost[0] == '*' && vhost[1] == '.' { - suffixMatches = append(suffixMatches, - fmt.Sprintf(" acl %v req_ssl_sni -m dom %v\n", matchToken, strings.SplitN(vhost, ".", 2)[1])) - suffixRoutes[matchToken] = appID - } else { - exactMatches = append(exactMatches, - fmt.Sprintf(" acl %v req_ssl_sni -i %v\n", matchToken, vhost)) - exactRoutes[matchToken] = appID - } - - if manager.vhostDefaultHTTPS == appID { - vhostDefault = appID - } - } - } - - var fragment string - fragment += fmt.Sprintf( - "frontend __gateway_https\n"+ - " bind %v:%v\n"+ - " mode tcp\n"+ - " tcp-request inspect-delay 5s\n"+ - " tcp-request content accept if { req_ssl_hello_type 1 }\n"+ - "\n", - manager.GatewayAddr, - port) - - // write ACL statements - fragment += strings.Join(exactMatches, "") - fragment += strings.Join(suffixMatches, "") - if len(exactMatches) != 0 || len(suffixMatches) != 0 { - fragment += "\n" - } - - for _, acl := range SortedStrStrKeys(exactRoutes) { - appID := exactRoutes[acl] - fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) - } - - for _, acl := range SortedStrStrKeys(suffixRoutes) { - appID := suffixRoutes[acl] - fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) - } - - fragment += "\n" - - if len(vhostDefault) != 0 { - fragment += fmt.Sprintf(" default_backend %v\n\n", vhostDefault) - } - - return fragment -} - -func (manager *HaproxyMgr) makeConfigTail() (string, error) { - if len(manager.ConfigTailPath) == 0 { - return "", nil - } - - tail, err := ioutil.ReadFile(manager.ConfigTailPath) - if err != nil { - return "", err - } - - return string(tail), nil -} - -func (manager *HaproxyMgr) reloadConfig(force bool) error { - if !force && FileIsIdentical(manager.ConfigPath, manager.OldConfigPath) { - log.Printf("[haproxy] config file not changed. ignoring reload\n") - return nil - } - - pidStr, err := ioutil.ReadFile(manager.PidFile) - if err != nil { - return manager.startProcess() - } - pid, err := strconv.Atoi(strings.TrimSpace(string(pidStr))) - if err != nil { - return err - } - - err = syscall.Kill(pid, syscall.Signal(0)) - if err != nil { - // process doesn't exist; start up process - return manager.startProcess() - } else { - // process does exist; send SIGHUP to reload - return manager.reloadProcess(pid) - } -} - -func (manager *HaproxyMgr) checkConfig(path string) error { - return manager.exec("checking configuration", - "-f", path, "-p", manager.PidFile, "-c") -} - -func (manager *HaproxyMgr) startProcess() error { - return manager.exec("starting up process", - "-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-q") -} - -func (manager *HaproxyMgr) reloadProcess(pid int) error { - return manager.exec("reloading configuration", - "-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-sf", fmt.Sprint(pid)) -} - -func (manager *HaproxyMgr) exec(logMessage string, args ...string) error { - proc := exec.Command(manager.Executable, args...) - output, err := proc.CombinedOutput() - - log.Printf("[haproxy] %v: %v %v\n", logMessage, manager.Executable, args) - - exitCode := proc.ProcessState.Sys().(syscall.WaitStatus) - if exitCode != 0 { - log.Printf("[haproxy] Bad exit code %v.\n", exitCode) - err = ErrBadExit - } - - if len(output) != 0 && manager.Verbose { - log.Println("[haproxy] command output:") - log.Println(strings.TrimSpace(string(output))) - } - - return err -} - -func (manager *HaproxyMgr) clearAppStateCache() { - manager.appStateCache = make(map[string]map[string]*marathon.Task) -} - -func (manager *HaproxyMgr) getAppStateCacheEntry(appID, taskID string) *marathon.Task { - if app, ok := manager.appStateCache[appID]; ok { - if task, ok := app[taskID]; ok { - return task - } - } - return nil -} - -func (manager *HaproxyMgr) setAppStateCacheEntry(task *marathon.Task) { - app, ok := manager.appStateCache[task.AppId] - if !ok { - app = make(map[string]*marathon.Task) - manager.appStateCache[task.AppId] = app - } - app[task.Id] = task -} - -func (manager *HaproxyMgr) removeAppStateCacheEntry(appID, taskID string) { - if len(manager.appStateCache[appID]) != 0 { - delete(manager.appStateCache[appID], taskID) - - if len(manager.appStateCache[appID]) == 0 { - delete(manager.appStateCache, appID) - } - } -} - -// {{{ SortedTaskList - -type SortedTaskList struct { - Tasks []marathon.Task - PortIndex int -} - -func (tasks SortedTaskList) Len() int { - return len(tasks.Tasks) -} - -func (tasks SortedTaskList) Less(i, j int) bool { - var a = &tasks.Tasks[i] - var b = &tasks.Tasks[j] - - if len(a.Ports) < tasks.PortIndex || len(b.Ports) < tasks.PortIndex { - // XXX That's a very case; when you redeploy your app with the port count - // changed, you might run into here. - return a.Host < b.Host - } - - return a.Host < b.Host || a.Ports[tasks.PortIndex] < b.Ports[tasks.PortIndex] -} - -func (tasks SortedTaskList) Swap(i, j int) { - tmp := tasks.Tasks[i] - tasks.Tasks[i] = tasks.Tasks[j] - tasks.Tasks[j] = tmp -} - -// }}} - -func sortTasks(tasks []marathon.Task, portIndex int) []marathon.Task { - stl := SortedTaskList{Tasks: tasks, PortIndex: portIndex} - sort.Sort(stl) - return stl.Tasks -} diff --git a/main.go b/main.go index e617615..3dc1dd2 100644 --- a/main.go +++ b/main.go @@ -45,13 +45,6 @@ import ( flag "github.com/ogier/pflag" ) -type mmsdHandler interface { - Setup() error - Apply(apps []*marathon.App, force bool) error - Update(app *marathon.App, taskID string) error - Remove(appID string, taskID string, app *marathon.App) error -} - type mmsdService struct { HttpApiPort uint Verbose bool @@ -100,7 +93,7 @@ type mmsdService struct { DnsPushSRV bool // runtime state - apps []AppCluster + apps []*AppCluster killingTasks map[string]bool // set of tasks currently in killing state } @@ -270,8 +263,8 @@ func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { } // convertMarathonApps converts an array of marathon.App into a []AppCluster. -func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster { - var apps []AppCluster +func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster { + var apps []*AppCluster for _, mApp := range mApps { for portIndex, portDef := range mApp.PortDefinitions { if mmsd.isGroupIncluded(portDef.Labels["lb-group"]) { @@ -312,7 +305,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster labels[k] = v } - app := AppCluster{ + app := &AppCluster{ Name: mApp.Id, Id: PrettifyAppId2(mApp.Id, portIndex), ServicePort: mApp.PortDefinitions[portIndex].Port, @@ -329,8 +322,8 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []AppCluster return apps } -func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []AppCluster { - var apps []AppCluster +func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*AppCluster { + var apps []*AppCluster for _, app := range mmsd.apps { if app.Name == mAppId { apps = append(apps, app) @@ -399,10 +392,17 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { // So we consider thie TASK_RUNNING state as healthy-notice. if len(app.HealthChecks) == 0 { for _, app := range mmsd.findAppsByMarathonId(event.AppId) { + // app.Backends = append(app.Backends, AppBackend{ + // Id: event.TaskId, + // State: string(event.TaskStatus), + // Host: event.Host, + // Port: 0, + // }) + for _, task := range app.Backends { if task.Id == event.TaskId { for _, handler := range mmsd.Handlers { - handler.AddTask(task, app) + handler.AddTask(&task, app) } } } @@ -428,7 +428,7 @@ func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.Tas task.State = string(newStatus) if task.Id == taskId { for _, handler := range mmsd.Handlers { - handler.RemoveTask(task, app) + handler.RemoveTask(&task, app) } } } @@ -441,9 +441,9 @@ func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusCh if task.Id == event.TaskId { for _, handler := range mmsd.Handlers { if event.Alive { - handler.AddTask(task, app) + handler.AddTask(&task, app) } else { - handler.RemoveTask(task, app) + handler.RemoveTask(&task, app) } } } @@ -518,7 +518,7 @@ func (mmsd *mmsdService) Shutdown() { mmsd.quitChannel <- true } -func (mmsd *mmsdService) applyApps(apps []AppCluster) { +func (mmsd *mmsdService) applyApps(apps []*AppCluster) { mmsd.apps = apps for _, handler := range mmsd.Handlers { @@ -527,7 +527,7 @@ func (mmsd *mmsdService) applyApps(apps []AppCluster) { } func (mmsd *mmsdService) setupHandlers() { - mmsd.Handlers = append(mmsd.Handlers, &EventLogger{ + mmsd.Handlers = append(mmsd.Handlers, &EventLoggerModule{ Verbose: true, }) diff --git a/event_logger.go b/mod_eventlogger.go similarity index 57% rename from event_logger.go rename to mod_eventlogger.go index c5ff4d7..84ce4d0 100644 --- a/event_logger.go +++ b/mod_eventlogger.go @@ -6,21 +6,21 @@ import ( "log" ) -/* EventLogger adds simple event logging to the logger. +/* EventLoggerModule adds simple event logging to the logger. */ -type EventLogger struct { +type EventLoggerModule struct { Verbose bool } -func (logger *EventLogger) Startup() { +func (logger *EventLoggerModule) Startup() { log.Printf("Initialize\n") } -func (logger *EventLogger) Shutdown() { +func (logger *EventLoggerModule) Shutdown() { log.Printf("Shutdown\n") } -func (logger *EventLogger) Apply(apps []AppCluster) { +func (logger *EventLoggerModule) Apply(apps []*AppCluster) { if logger.Verbose { out, err := json.MarshalIndent(apps, "", " ") if err != nil { @@ -35,10 +35,10 @@ func (logger *EventLogger) Apply(apps []AppCluster) { } } -func (logger *EventLogger) AddTask(task AppBackend, app AppCluster) { +func (logger *EventLoggerModule) AddTask(task *AppBackend, app *AppCluster) { log.Printf("Task Add: %v: %v %v\n", task.State, app.Id, task.Host) } -func (logger *EventLogger) RemoveTask(task AppBackend, app AppCluster) { +func (logger *EventLoggerModule) RemoveTask(task *AppBackend, app *AppCluster) { log.Printf("Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) } diff --git a/mod_haproxy.go b/mod_haproxy.go index 6db037a..3617789 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -22,6 +22,7 @@ const ( LB_VHOST_DEFAULT_HTTP = "lb-vhost-default" LB_VHOST_HTTPS = "lb-vhost-ssl" LB_VHOST_DEFAULT_HTTPS = "lb-vhost-default-ssl" + LB_DISABLED = "lb-disabled" ) var ( @@ -68,7 +69,7 @@ func (module *HaproxyModule) Shutdown() { log.Printf("Haproxy shutting down") } -func (module *HaproxyModule) Apply(apps []AppCluster) { +func (module *HaproxyModule) Apply(apps []*AppCluster) { log.Printf("Haproxy: apply([]AppCluster)") for _, app := range apps { @@ -89,18 +90,20 @@ func (module *HaproxyModule) Apply(apps []AppCluster) { } } -func (module *HaproxyModule) AddTask(task AppBackend, app AppCluster) { +func (module *HaproxyModule) AddTask(task *AppBackend, app *AppCluster) { if !module.supportsProtocol(app.Protocol) { return } log.Printf("Haproxy: AddTask()") + // TODO } -func (module *HaproxyModule) RemoveTask(task AppBackend, app AppCluster) { +func (module *HaproxyModule) RemoveTask(task *AppBackend, app *AppCluster) { log.Printf("Haproxy: RemoveTask()") + // TODO } -func (module *HaproxyModule) makeConfig(app AppCluster) string { +func (module *HaproxyModule) makeConfig(app *AppCluster) string { module.updateGatewaySettings(app) // generate haproxy config fragment @@ -201,7 +204,7 @@ func (module *HaproxyModule) makeConfig(app AppCluster) string { return result } -func (module *HaproxyModule) updateGatewaySettings(app AppCluster) { +func (module *HaproxyModule) updateGatewaySettings(app *AppCluster) { // update HTTP virtual hosting var lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTP]) if len(lbVirtualHosts) != 0 { @@ -527,7 +530,7 @@ func (module *HaproxyModule) exec(logMessage string, args ...string) error { return err } -func (module *HaproxyModule) getAppProtocol(app AppCluster) string { +func (module *HaproxyModule) getAppProtocol(app *AppCluster) string { if app.HealthCheck != nil { if app.HealthCheck.Protocol != "COMMAND" { return strings.ToUpper(app.HealthCheck.Protocol) From 2ab446b6f8d4df70c8adcc0b6127597ac4fe2c96 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 19:07:05 +0200 Subject: [PATCH 18/39] some API documentation --- event_listener.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/event_listener.go b/event_listener.go index dcd0620..b46cce3 100644 --- a/event_listener.go +++ b/event_listener.go @@ -1,11 +1,23 @@ package main +// EventListener provides an interface for hooking into +// standard service discovery API calls, such as adding and removing +// backends from load balancers (app clusters). type EventListener interface { + // Startup is invoked upon application startup Startup() + + // Shutdown is invoked upon application shutdown Shutdown() + // Apply installs the load balancer for all apps. + // This function is invoked upon startup to synchronize with the current + // state. Apply(apps []*AppCluster) + // AddTask must add the given backend to the cluster. AddTask(task *AppBackend, app *AppCluster) + + // RemoveTask must remove the given backend from the cluster. RemoveTask(task *AppBackend, app *AppCluster) } From a8e1418ddc51a5e123b1efb32600ddc91d592b64 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 19:18:17 +0200 Subject: [PATCH 19/39] some initial (general purpose) tests, for helper.go --- helper_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 helper_test.go diff --git a/helper_test.go b/helper_test.go new file mode 100644 index 0000000..c767253 --- /dev/null +++ b/helper_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" +) + +func TestContains(t *testing.T) { + if Contains([]string{"a", "b"}, "b") == false { + t.Error("'b' not found.") + } + if Contains([]string{"a", "b"}, "aa") == true { + t.Error("'aa' found. weird.") + } +} + +func TestAtoi(t *testing.T) { + // test success + if i := Atoi("123", 17); i != 123 { + t.Errorf("Failed to run Atoi on \"123\". Got %v instead.", i) + } + // test default + if i := Atoi("blah", 17); i != 17 { + t.Errorf("Failed to run Atoi on \"\". Got %v instead.", i) + } + // test default + if i := Atoi("", 17); i != 17 { + t.Errorf("Failed to run Atoi on \"\". Got %v instead.", i) + } +} + +func TestFindMissing(t *testing.T) { + a := []string{"bar", "foo", "tar"} + b := []string{"foo", "com"} + c := FindMissing(a, b) + t.Logf("result: %+v", c) + if len(c) != 2 { + t.Fatalf("Result size differs. Expected 2 but got %v", len(c)) + } + if c[0] != "bar" { + t.Errorf("\"bar\" not found, but \"%v\".", c[0]) + } + if c[1] != "tar" { + t.Errorf("\"tar\" not found, but \"%v\".", c[1]) + } +} From 484a3b7542776bf8eabe4f6d7a0ddfcce1385c20 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 22:13:10 +0200 Subject: [PATCH 20/39] port files module --- files_manager.go | 134 ----------------------------------------------- helper.go | 16 ++++++ main.go | 23 ++++---- mod_files.go | 128 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 145 deletions(-) delete mode 100644 files_manager.go create mode 100644 mod_files.go diff --git a/files_manager.go b/files_manager.go deleted file mode 100644 index a47a749..0000000 --- a/files_manager.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "io/ioutil" - "log" - "os" - "path/filepath" - - "github.com/dawanda/go-mesos/marathon" -) - -type FilesManager struct { - Verbose bool - BasePath string -} - -func (upstream *FilesManager) Log(msg string) { - if upstream.Verbose { - log.Printf("upstream: %v\n", msg) - } -} - -func (manager *FilesManager) Startup() { -} - -func (manager *FilesManager) Shutdown() { -} - -func (upstream *FilesManager) Remove(appID string, taskID string, app *marathon.App) error { - if app != nil { - _, err := upstream.writeApp(app) - return err - } else { - // TODO: remove files for app-$portIndex - return nil - } -} - -func (upstream *FilesManager) Update(app *marathon.App, taskID string) error { - _, err := upstream.writeApp(app) - return err -} - -func (upstream *FilesManager) Apply(apps []*marathon.App, force bool) error { - err := os.MkdirAll(upstream.BasePath, 0770) - if err != nil { - return err - } - - var newFiles []string - oldFiles, err := upstream.collectFiles() - if err != nil { - return err - } - - for _, app := range apps { - filenames, _ := upstream.writeApp(app) - newFiles = append(newFiles, filenames...) - } - - // check for superfluous files - diff := FindMissing(oldFiles, newFiles) - for _, superfluous := range diff { - upstream.Log(fmt.Sprintf("Removing superfluous file: %v\n", superfluous)) - os.Remove(superfluous) - } - - return nil -} - -func (upstream *FilesManager) writeApp(app *marathon.App) ([]string, error) { - var files []string - - for portIndex, port := range app.Ports { - app_id := PrettifyAppId(app.Id, portIndex, port) - cfgfile := filepath.Join(upstream.BasePath, app_id+".instances") - tmpfile := cfgfile + ".tmp" - - err := upstream.writeFile(tmpfile, app_id, portIndex, app) - if err != nil { - return files, err - } - files = append(files, cfgfile) - - if _, err := os.Stat(cfgfile); os.IsNotExist(err) { - upstream.Log(fmt.Sprintf("new %v", cfgfile)) - os.Rename(tmpfile, cfgfile) - } else if !FileIsIdentical(tmpfile, cfgfile) { - upstream.Log(fmt.Sprintf("refresh %v", cfgfile)) - os.Rename(tmpfile, cfgfile) - } else { - // new file is identical to already existing one - os.Remove(tmpfile) - } - } - return files, nil -} - -func (upstream *FilesManager) writeFile(filename string, appId string, - portIndex int, app *marathon.App) error { - - var b bytes.Buffer - b.WriteString(fmt.Sprintf("Service-Name: %v\r\n", appId)) - b.WriteString(fmt.Sprintf("Service-Port: %v\r\n", app.Ports[portIndex])) - b.WriteString(fmt.Sprintf("Service-Transport-Proto: %v\r\n", GetTransportProtocol(app, portIndex))) - b.WriteString(fmt.Sprintf("Service-Application-Proto: %v\r\n", GetApplicationProtocol(app, portIndex))) - if proto := GetHealthCheckProtocol(app, portIndex); len(proto) != 0 { - b.WriteString(fmt.Sprintf("Health-Check-Proto: %v\r\n", GetApplicationProtocol(app, portIndex))) - } - b.WriteString("\r\n") - - for _, task := range app.Tasks { - b.WriteString(fmt.Sprintf("%v:%v\n", task.Host, task.Ports[portIndex])) - } - - return ioutil.WriteFile(filename, b.Bytes(), 0660) -} - -func (upstream *FilesManager) collectFiles() ([]string, error) { - fileInfos, err := ioutil.ReadDir(upstream.BasePath) - if err != nil { - upstream.Log(fmt.Sprintf("Error reading directory %v. %v", upstream.BasePath, err)) - return nil, err - } - - var fileNames []string - for _, fileInfo := range fileInfos { - fileNames = append(fileNames, filepath.Join(upstream.BasePath, fileInfo.Name())) - } - - return fileNames, nil -} diff --git a/helper.go b/helper.go index c9e157b..d26f036 100644 --- a/helper.go +++ b/helper.go @@ -135,6 +135,22 @@ func FindMissing(slice1, slice2 []string) (missing []string) { return } +func GetApplicationProtocol1(app *AppCluster) string { + if proto := strings.ToLower(app.Labels["proto"]); len(proto) != 0 { + return proto + } + + if app.HealthCheck != nil && len(app.HealthCheck.Protocol) != 0 { + return strings.ToLower(app.HealthCheck.Protocol) + } + + if len(app.Protocol) != 0 { + return strings.ToLower(app.Protocol) + } + + return "tcp" +} + func GetApplicationProtocol(app *marathon.App, portIndex int) (proto string) { if proto = strings.ToLower(app.Labels["proto"]); len(proto) != 0 { return diff --git a/main.go b/main.go index 3dc1dd2..d98ae1c 100644 --- a/main.go +++ b/main.go @@ -305,10 +305,11 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster labels[k] = v } + servicePort := mApp.PortDefinitions[portIndex].Port app := &AppCluster{ Name: mApp.Id, - Id: PrettifyAppId2(mApp.Id, portIndex), - ServicePort: mApp.PortDefinitions[portIndex].Port, + Id: PrettifyAppId(mApp.Id, portIndex, servicePort), + ServicePort: servicePort, Protocol: mApp.PortDefinitions[portIndex].Protocol, PortName: mApp.PortDefinitions[portIndex].Name, Labels: labels, @@ -527,9 +528,9 @@ func (mmsd *mmsdService) applyApps(apps []*AppCluster) { } func (mmsd *mmsdService) setupHandlers() { - mmsd.Handlers = append(mmsd.Handlers, &EventLoggerModule{ - Verbose: true, - }) + // mmsd.Handlers = append(mmsd.Handlers, &EventLoggerModule{ + // Verbose: true, + // }) // if mmsd.DnsEnabled { // mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ @@ -570,12 +571,12 @@ func (mmsd *mmsdService) setupHandlers() { }) } - // if mmsd.FilesEnabled { - // mmsd.Handlers = append(mmsd.Handlers, &FilesManager{ - // Verbose: mmsd.Verbose, - // BasePath: mmsd.RunStateDir + "/confd", - // }) - // } + if mmsd.FilesEnabled { + mmsd.Handlers = append(mmsd.Handlers, &FilesManager{ + Verbose: mmsd.Verbose, + BasePath: mmsd.RunStateDir + "/confd", + }) + } for _, handler := range mmsd.Handlers { handler.Startup() diff --git a/mod_files.go b/mod_files.go new file mode 100644 index 0000000..f57670c --- /dev/null +++ b/mod_files.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" +) + +type FilesManager struct { + Verbose bool + BasePath string +} + +func (upstream *FilesManager) Log(msg string) { + if upstream.Verbose { + log.Printf("upstream: %v\n", msg) + } +} + +func (manager *FilesManager) Startup() { +} + +func (manager *FilesManager) Shutdown() { +} + +func (manager *FilesManager) RemoveTask(task *AppBackend, app *AppCluster) { + if app != nil { + manager.writeApp(app) + } else { + // TODO: remove files for app-$portIndex + } +} + +func (upstream *FilesManager) AddTask(task *AppBackend, app *AppCluster) { + upstream.writeApp(app) +} + +func (upstream *FilesManager) Apply(apps []*AppCluster) { + err := os.MkdirAll(upstream.BasePath, 0770) + if err != nil { + log.Printf("Failed to mkdir. %v", err) + return + } + + var newFiles []string + oldFiles, err := upstream.collectFiles() + if err != nil { + log.Printf("Failed to collect files. %v", err) + return + } + + for _, app := range apps { + filenames, _ := upstream.writeApp(app) + newFiles = append(newFiles, filenames...) + } + + // check for superfluous files + diff := FindMissing(oldFiles, newFiles) + for _, superfluous := range diff { + upstream.Log(fmt.Sprintf("Removing superfluous file: %v\n", superfluous)) + os.Remove(superfluous) + } +} + +func (upstream *FilesManager) writeApp(app *AppCluster) ([]string, error) { + var files []string + + app_id := app.Id + cfgfile := filepath.Join(upstream.BasePath, app_id+".instances") + tmpfile := cfgfile + ".tmp" + + err := upstream.writeFile(tmpfile, app_id, app) + if err != nil { + return files, err + } + files = append(files, cfgfile) + + if _, err := os.Stat(cfgfile); os.IsNotExist(err) { + upstream.Log(fmt.Sprintf("new %v", cfgfile)) + os.Rename(tmpfile, cfgfile) + } else if !FileIsIdentical(tmpfile, cfgfile) { + upstream.Log(fmt.Sprintf("refresh %v", cfgfile)) + os.Rename(tmpfile, cfgfile) + } else { + // new file is identical to already existing one + os.Remove(tmpfile) + } + return files, nil +} + +func (upstream *FilesManager) writeFile(filename string, appId string, + app *AppCluster) error { + + var b bytes.Buffer + b.WriteString(fmt.Sprintf("Service-Name: %v\r\n", appId)) + b.WriteString(fmt.Sprintf("Service-Port: %v\r\n", app.ServicePort)) + b.WriteString(fmt.Sprintf("Service-Transport-Proto: %v\r\n", app.Protocol)) + b.WriteString(fmt.Sprintf("Service-Application-Proto: %v\r\n", GetApplicationProtocol1(app))) + if app.HealthCheck != nil && len(app.HealthCheck.Protocol) != 0 { + b.WriteString(fmt.Sprintf("Health-Check-Proto: %v\r\n", strings.ToLower(app.HealthCheck.Protocol))) + } + b.WriteString("\r\n") + + for _, task := range app.Backends { + b.WriteString(fmt.Sprintf("%v:%v\n", task.Host, task.Port)) + } + + return ioutil.WriteFile(filename, b.Bytes(), 0660) +} + +func (upstream *FilesManager) collectFiles() ([]string, error) { + fileInfos, err := ioutil.ReadDir(upstream.BasePath) + if err != nil { + upstream.Log(fmt.Sprintf("Error reading directory %v. %v", upstream.BasePath, err)) + return nil, err + } + + var fileNames []string + for _, fileInfo := range fileInfos { + fileNames = append(fileNames, filepath.Join(upstream.BasePath, fileInfo.Name())) + } + + return fileNames, nil +} From c457449706c9441c7c8d2e4d0058d16f713f09ee Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 22:29:30 +0200 Subject: [PATCH 21/39] documentation update --- event_listener.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/event_listener.go b/event_listener.go index b46cce3..fd6fe54 100644 --- a/event_listener.go +++ b/event_listener.go @@ -11,13 +11,20 @@ type EventListener interface { Shutdown() // Apply installs the load balancer for all apps. + // // This function is invoked upon startup to synchronize with the current // state. Apply(apps []*AppCluster) // AddTask must add the given backend to the cluster. + // + // It is assured that the task to be added is also already added to the + // given AppCluster. AddTask(task *AppBackend, app *AppCluster) // RemoveTask must remove the given backend from the cluster. + // + // It is ensured that the task to be removed is not present in the given + // AppCluster. RemoveTask(task *AppBackend, app *AppCluster) } From f6709d10f270fd775a2c6a92c033cdadd2221638 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Mon, 19 Sep 2016 22:30:56 +0200 Subject: [PATCH 22/39] core: ensure RemoveTask also removes it from app cluster's task list --- main.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index d98ae1c..4481417 100644 --- a/main.go +++ b/main.go @@ -423,17 +423,23 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { } } -func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.TaskStatus) { +func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.TaskStatus) bool { for _, app := range mmsd.findAppsByMarathonId(appId) { - for _, task := range app.Backends { - task.State = string(newStatus) + for i, task := range app.Backends { if task.Id == taskId { + // update task state, and remove task out of app cluster's task list + task.State = string(newStatus) + app.Backends = append(app.Backends[:0], app.Backends[i+1:]...) + for _, handler := range mmsd.Handlers { handler.RemoveTask(&task, app) } + + return true } } } + return false } func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusChangedEvent) { From 3f0588e0798dc4998678b697dcc3574a51c4a8e4 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 09:29:22 +0200 Subject: [PATCH 23/39] wip --- main.go | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index 4481417..a98321d 100644 --- a/main.go +++ b/main.go @@ -392,14 +392,15 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { // XXX Only update propagate no health checks have been configured. // So we consider thie TASK_RUNNING state as healthy-notice. if len(app.HealthChecks) == 0 { + // for portIndex, port := range event.Ports { + // app.Backends = append(app.Backends, AppBackend{ + // Id: event.TaskId, + // State: string(event.TaskStatus), + // Host: event.Host, + // Port: port, + // }) + // } for _, app := range mmsd.findAppsByMarathonId(event.AppId) { - // app.Backends = append(app.Backends, AppBackend{ - // Id: event.TaskId, - // State: string(event.TaskStatus), - // Host: event.Host, - // Port: 0, - // }) - for _, task := range app.Backends { if task.Id == event.TaskId { for _, handler := range mmsd.Handlers { @@ -443,18 +444,10 @@ func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.Tas } func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusChangedEvent) { - for _, app := range mmsd.findAppsByMarathonId(event.AppId) { - for _, task := range app.Backends { - if task.Id == event.TaskId { - for _, handler := range mmsd.Handlers { - if event.Alive { - handler.AddTask(&task, app) - } else { - handler.RemoveTask(&task, app) - } - } - } - } + if event.Alive { + // mmsd.AddTask(event.AppId, event.TaskId) + } else { + // mmsd.RemoveTask(event.AppId, event.TaskId) } } From f07d0d8e3c907dd1213dcc5212fb62fea6ae5f24 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 17:32:24 +0200 Subject: [PATCH 24/39] wip --- Godeps/Godeps.json | 2 +- app.go | 17 +++---- main.go | 110 ++++++++++++++++++++++++++------------------- 3 files changed, 74 insertions(+), 55 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 2f817bb..8e5e839 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -4,7 +4,7 @@ "Deps": [ { "ImportPath": "github.com/dawanda/go-mesos/marathon", - "Rev": "41be4aa9c9f5170cb339043d8f21eb66d8a07807" + "Rev": "56dfa5705fdac2336ac685e96a4882af4629e346" }, { "ImportPath": "github.com/gorilla/context", diff --git a/app.go b/app.go index 5b0f680..1d5c75f 100644 --- a/app.go +++ b/app.go @@ -3,14 +3,15 @@ package main // Marathon independant application definitions type AppCluster struct { - Name string - Id string - ServicePort uint - Protocol string - PortName string - Labels map[string]string - HealthCheck *AppHealthCheck - Backends []AppBackend // TODO ensured to be always ordered + Name string // Marathon's human reaable AppId (i.e. /path/to/app) + Id string // globally unique ID that identifies this application + ServicePort uint // service port to listen to in service discovery + Protocol string // service protocol + PortName string // user filled port name + Labels map[string]string // key/value pairs of labels (port-local | global) + HealthCheck *AppHealthCheck // healthcheck, if available, or nil + Backends []AppBackend // ordered list of backend tasks + PortIndex int // Marathon's application port index } type AppHealthCheck struct { diff --git a/main.go b/main.go index a98321d..e7d1c0d 100644 --- a/main.go +++ b/main.go @@ -296,6 +296,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster State: string(*mTask.State), }) } + // TODO: sort backends ASC labels := make(map[string]string) for k, v := range mApp.Labels { @@ -315,6 +316,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster Labels: labels, HealthCheck: healthCheck, Backends: backends, + PortIndex: portIndex, } apps = append(apps, app) } @@ -323,6 +325,14 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster return apps } +func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *AppCluster { + log.Printf("Application %v with port index %v not found.", + appId, portIndex) + return nil +} + +// findAppsByMarathonId returns list of all applications that belong to the +// given Marathon App mAppId. func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*AppCluster { var apps []*AppCluster for _, app := range mmsd.apps { @@ -357,28 +367,16 @@ func (mmsd *mmsdService) OnMarathonConnectionFailure(event, data string) { log.Printf("Marathon Event Stream Error. %v. %v\n", event, data) } +// StatusUpdateEvent is invoked by SSE when exactly this named event is fired. func (mmsd *mmsdService) StatusUpdateEvent(data string) { var event marathon.StatusUpdateEvent err := json.Unmarshal([]byte(data), &event) if err != nil { log.Printf("Failed to unmarshal status_update_event. %v\n", err) log.Printf("status_update_event: %+v\n", data) - } else { - mmsd.statusUpdateEvent(&event) - } -} - -func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { - var event marathon.HealthStatusChangedEvent - err := json.Unmarshal([]byte(data), &event) - if err != nil { - log.Printf("Failed to unmarshal health_status_changed_event. %v\n", err) - } else { - mmsd.healthStatusChangedEvent(&event) + return } -} -func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { switch event.TaskStatus { case marathon.TaskRunning: app, err := mmsd.getMarathonApp(event.AppId) @@ -387,28 +385,23 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { event.AppId, event.TaskId, event.Host, err) return } - log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) + + log.Printf( + "App %v task %v on %v changed status. %v. %v\n", + event.AppId, event.TaskId, event.Host, event.TaskStatus, event.Message) // XXX Only update propagate no health checks have been configured. // So we consider thie TASK_RUNNING state as healthy-notice. if len(app.HealthChecks) == 0 { - // for portIndex, port := range event.Ports { - // app.Backends = append(app.Backends, AppBackend{ - // Id: event.TaskId, - // State: string(event.TaskStatus), - // Host: event.Host, - // Port: port, - // }) - // } - for _, app := range mmsd.findAppsByMarathonId(event.AppId) { - for _, task := range app.Backends { - if task.Id == event.TaskId { - for _, handler := range mmsd.Handlers { - handler.AddTask(&task, app) - } - } - } - } + mmsd.AddTask( + event.AppId, + event.TaskId, + string(event.TaskStatus), + event.Host, + event.Ports) + } else { + // TODO: add to mmsd.unhealthyTasks[] to remember host:port mappings + // for the moment when this task becomes live } case marathon.TaskKilling: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) @@ -424,7 +417,41 @@ func (mmsd *mmsdService) statusUpdateEvent(event *marathon.StatusUpdateEvent) { } } +// HealthStatusChangedEvent is invoked by SSE when exactly this named event is fired. +func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { + var event marathon.HealthStatusChangedEvent + err := json.Unmarshal([]byte(data), &event) + if err != nil { + log.Printf("Failed to unmarshal health_status_changed_event. %v\n", err) + } else if event.Alive { + // TODO mmsd.AddTask(event.AppId, event.TaskId) + } else { + mmsd.RemoveTask(event.AppId, event.TaskId, "TASK_RUNNING") + } +} + +// AddTask ensures the given task is added to the app and all handlers are +// notified as well. +func (mmsd *mmsdService) AddTask(appId, taskId, taskStatus, host string, ports []uint) { + for portIndex, port := range ports { + if app := mmsd.getAppByMarathonId(appId, portIndex); app != nil { + task := AppBackend{ + Id: taskId, + Host: host, + Port: port, + State: string(taskStatus), + } + app.Backends = append(app.Backends, task) + + for _, handler := range mmsd.Handlers { + handler.AddTask(&task, app) + } + } + } +} + func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.TaskStatus) bool { + var found uint for _, app := range mmsd.findAppsByMarathonId(appId) { for i, task := range app.Backends { if task.Id == taskId { @@ -435,20 +462,11 @@ func (mmsd *mmsdService) RemoveTask(appId, taskId string, newStatus marathon.Tas for _, handler := range mmsd.Handlers { handler.RemoveTask(&task, app) } - - return true + found++ } } } - return false -} - -func (mmsd *mmsdService) healthStatusChangedEvent(event *marathon.HealthStatusChangedEvent) { - if event.Alive { - // mmsd.AddTask(event.AppId, event.TaskId) - } else { - // mmsd.RemoveTask(event.AppId, event.TaskId) - } + return found > 0 } const appVersion = "0.9.12" @@ -603,9 +621,9 @@ func main() { GatewayAddr: net.ParseIP("0.0.0.0"), GatewayPortHTTP: 80, GatewayPortHTTPS: 443, - FilesEnabled: true, - UDPEnabled: true, - TCPEnabled: true, + FilesEnabled: false, + UDPEnabled: false, + TCPEnabled: false, LocalHealthChecks: true, HaproxyBin: locateExe("haproxy"), HaproxyTailCfg: "/etc/mmsd/haproxy-tail.cfg", From 0b6edaf1cb2690a6421676ec171152b62e56fb0e Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Tue, 20 Sep 2016 09:41:01 +0200 Subject: [PATCH 25/39] Go linter warning fixes --- main.go | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/main.go b/main.go index e7d1c0d..6bbcd07 100644 --- a/main.go +++ b/main.go @@ -86,11 +86,11 @@ type mmsdService struct { UDPEnabled bool // DNS service discovery - DnsEnabled bool - DnsPort uint - DnsBaseName string - DnsTTL time.Duration - DnsPushSRV bool + DNSEnabled bool + DNSPort uint + DNSBaseName string + DNSTTL time.Duration + DNSPushSRV bool // runtime state apps []*AppCluster @@ -498,11 +498,11 @@ func (mmsd *mmsdService) Run() { flag.StringVar(&mmsd.HaproxyTailCfg, "haproxy-cfgtail", mmsd.HaproxyTailCfg, "path to haproxy tail config file") flag.IPVar(&mmsd.ServiceAddr, "haproxy-bind", mmsd.ServiceAddr, "haproxy management port") flag.UintVar(&mmsd.HaproxyPort, "haproxy-port", mmsd.HaproxyPort, "haproxy management port") - flag.BoolVar(&mmsd.DnsEnabled, "enable-dns", mmsd.DnsEnabled, "Enables DNS-based service discovery") - flag.UintVar(&mmsd.DnsPort, "dns-port", mmsd.DnsPort, "DNS service discovery port") - flag.BoolVar(&mmsd.DnsPushSRV, "dns-push-srv", mmsd.DnsPushSRV, "DNS service discovery to also push SRV on A") - flag.StringVar(&mmsd.DnsBaseName, "dns-basename", mmsd.DnsBaseName, "DNS service discovery's base name") - flag.DurationVar(&mmsd.DnsTTL, "dns-ttl", mmsd.DnsTTL, "DNS service discovery's reply message TTL") + flag.BoolVar(&mmsd.DNSEnabled, "enable-dns", mmsd.DNSEnabled, "Enables DNS-based service discovery") + flag.UintVar(&mmsd.DNSPort, "dns-port", mmsd.DNSPort, "DNS service discovery port") + flag.BoolVar(&mmsd.DNSPushSRV, "dns-push-srv", mmsd.DNSPushSRV, "DNS service discovery to also push SRV on A") + flag.StringVar(&mmsd.DNSBaseName, "dns-basename", mmsd.DNSBaseName, "DNS service discovery's base name") + flag.DurationVar(&mmsd.DNSTTL, "dns-ttl", mmsd.DNSTTL, "DNS service discovery's reply message TTL") showVersionAndExit := flag.BoolP("version", "V", false, "Shows version and exits") flag.Usage = func() { @@ -549,14 +549,14 @@ func (mmsd *mmsdService) setupHandlers() { // Verbose: true, // }) - // if mmsd.DnsEnabled { + // if mmsd.DNSEnabled { // mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ // Verbose: mmsd.Verbose, // ServiceAddr: mmsd.ServiceAddr, - // ServicePort: mmsd.DnsPort, - // PushSRV: mmsd.DnsPushSRV, - // BaseName: mmsd.DnsBaseName, - // DnsTTL: mmsd.DnsTTL, + // ServicePort: mmsd.DNSPort, + // PushSRV: mmsd.DNSPushSRV, + // BaseName: mmsd.DNSBaseName, + // DNSTTL: mmsd.DNSTTL, // }) // } @@ -632,11 +632,11 @@ func main() { ServiceAddr: net.ParseIP("0.0.0.0"), HttpApiPort: 8082, Verbose: false, - DnsEnabled: false, - DnsPort: 53, - DnsPushSRV: false, - DnsBaseName: "mmsd.", - DnsTTL: time.Second * 5, + DNSEnabled: false, + DNSPort: 53, + DNSPushSRV: false, + DNSBaseName: "mmsd.", + DNSTTL: time.Second * 5, quitChannel: make(chan bool), killingTasks: make(map[string]bool), } From 70d61805f45b1fd70a080ee9e97cb2e67ef25417 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Tue, 20 Sep 2016 17:49:42 +0200 Subject: [PATCH 26/39] WIP rewrite DNS module and packaging --- event_listener.go | 8 +- helper.go | 3 +- main.go | 48 ++++----- mod_eventlogger.go | 8 +- mod_files.go | 12 ++- mod_haproxy.go | 14 +-- app.go => module_api/app.go | 2 +- modules/dns.go | 200 ++++++++++++++++++++++++++++++++++++ 8 files changed, 253 insertions(+), 42 deletions(-) rename app.go => module_api/app.go (98%) create mode 100644 modules/dns.go diff --git a/event_listener.go b/event_listener.go index fd6fe54..3e4c02f 100644 --- a/event_listener.go +++ b/event_listener.go @@ -1,5 +1,7 @@ package main +import "github.com/dawanda/mmsd/module_api" + // EventListener provides an interface for hooking into // standard service discovery API calls, such as adding and removing // backends from load balancers (app clusters). @@ -14,17 +16,17 @@ type EventListener interface { // // This function is invoked upon startup to synchronize with the current // state. - Apply(apps []*AppCluster) + Apply(apps []*module_api.AppCluster) // AddTask must add the given backend to the cluster. // // It is assured that the task to be added is also already added to the // given AppCluster. - AddTask(task *AppBackend, app *AppCluster) + AddTask(task *module_api.AppBackend, app *module_api.AppCluster) // RemoveTask must remove the given backend from the cluster. // // It is ensured that the task to be removed is not present in the given // AppCluster. - RemoveTask(task *AppBackend, app *AppCluster) + RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) } diff --git a/helper.go b/helper.go index d26f036..2e2bcc5 100644 --- a/helper.go +++ b/helper.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/dawanda/go-mesos/marathon" + "github.com/dawanda/mmsd/module_api" ) var ( @@ -135,7 +136,7 @@ func FindMissing(slice1, slice2 []string) (missing []string) { return } -func GetApplicationProtocol1(app *AppCluster) string { +func GetApplicationProtocol1(app *module_api.AppCluster) string { if proto := strings.ToLower(app.Labels["proto"]); len(proto) != 0 { return proto } diff --git a/main.go b/main.go index 6bbcd07..0450745 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,8 @@ import ( "time" "github.com/dawanda/go-mesos/marathon" + "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/modules" "github.com/gorilla/mux" flag "github.com/ogier/pflag" ) @@ -93,7 +95,7 @@ type mmsdService struct { DNSPushSRV bool // runtime state - apps []*AppCluster + apps []*module_api.AppCluster killingTasks map[string]bool // set of tasks currently in killing state } @@ -263,19 +265,19 @@ func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { } // convertMarathonApps converts an array of marathon.App into a []AppCluster. -func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster { - var apps []*AppCluster +func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*module_api.AppCluster { + var apps []*module_api.AppCluster for _, mApp := range mApps { for portIndex, portDef := range mApp.PortDefinitions { if mmsd.isGroupIncluded(portDef.Labels["lb-group"]) { - var healthCheck *AppHealthCheck + var healthCheck *module_api.AppHealthCheck if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { var mCommand *string if mHealthCheck.Command != nil { mCommand = new(string) *mCommand = mHealthCheck.Command.Value } - healthCheck = &AppHealthCheck{ + healthCheck = &module_api.AppHealthCheck{ Protocol: mHealthCheck.Protocol, Path: mHealthCheck.Path, Command: mCommand, @@ -287,9 +289,9 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster } } - var backends []AppBackend + var backends []module_api.AppBackend for _, mTask := range mApp.Tasks { - backends = append(backends, AppBackend{ + backends = append(backends, module_api.AppBackend{ Id: mTask.Id, Host: mTask.Host, Port: mTask.Ports[portIndex], @@ -307,7 +309,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster } servicePort := mApp.PortDefinitions[portIndex].Port - app := &AppCluster{ + app := &module_api.AppCluster{ Name: mApp.Id, Id: PrettifyAppId(mApp.Id, portIndex, servicePort), ServicePort: servicePort, @@ -325,7 +327,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*AppCluster return apps } -func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *AppCluster { +func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *module_api.AppCluster { log.Printf("Application %v with port index %v not found.", appId, portIndex) return nil @@ -333,8 +335,8 @@ func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *AppClu // findAppsByMarathonId returns list of all applications that belong to the // given Marathon App mAppId. -func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*AppCluster { - var apps []*AppCluster +func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*module_api.AppCluster { + var apps []*module_api.AppCluster for _, app := range mmsd.apps { if app.Name == mAppId { apps = append(apps, app) @@ -435,7 +437,7 @@ func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { func (mmsd *mmsdService) AddTask(appId, taskId, taskStatus, host string, ports []uint) { for portIndex, port := range ports { if app := mmsd.getAppByMarathonId(appId, portIndex); app != nil { - task := AppBackend{ + task := module_api.AppBackend{ Id: taskId, Host: host, Port: port, @@ -536,7 +538,7 @@ func (mmsd *mmsdService) Shutdown() { mmsd.quitChannel <- true } -func (mmsd *mmsdService) applyApps(apps []*AppCluster) { +func (mmsd *mmsdService) applyApps(apps []*module_api.AppCluster) { mmsd.apps = apps for _, handler := range mmsd.Handlers { @@ -549,16 +551,16 @@ func (mmsd *mmsdService) setupHandlers() { // Verbose: true, // }) - // if mmsd.DNSEnabled { - // mmsd.Handlers = append(mmsd.Handlers, &DnsManager{ - // Verbose: mmsd.Verbose, - // ServiceAddr: mmsd.ServiceAddr, - // ServicePort: mmsd.DNSPort, - // PushSRV: mmsd.DNSPushSRV, - // BaseName: mmsd.DNSBaseName, - // DNSTTL: mmsd.DNSTTL, - // }) - // } + if mmsd.DNSEnabled { + mmsd.Handlers = append(mmsd.Handlers, &modules.DNSModule{ + Verbose: mmsd.Verbose, + ServiceAddr: mmsd.ServiceAddr, + ServicePort: mmsd.DNSPort, + PushSRV: mmsd.DNSPushSRV, + BaseName: mmsd.DNSBaseName, + DNSTTL: mmsd.DNSTTL, + }) + } // if mmsd.UDPEnabled { // mmsd.Handlers = append(mmsd.Handlers, NewUdpManager( diff --git a/mod_eventlogger.go b/mod_eventlogger.go index 84ce4d0..d604a09 100644 --- a/mod_eventlogger.go +++ b/mod_eventlogger.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "log" + + "github.com/dawanda/mmsd/module_api" ) /* EventLoggerModule adds simple event logging to the logger. @@ -20,7 +22,7 @@ func (logger *EventLoggerModule) Shutdown() { log.Printf("Shutdown\n") } -func (logger *EventLoggerModule) Apply(apps []*AppCluster) { +func (logger *EventLoggerModule) Apply(apps []*module_api.AppCluster) { if logger.Verbose { out, err := json.MarshalIndent(apps, "", " ") if err != nil { @@ -35,10 +37,10 @@ func (logger *EventLoggerModule) Apply(apps []*AppCluster) { } } -func (logger *EventLoggerModule) AddTask(task *AppBackend, app *AppCluster) { +func (logger *EventLoggerModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { log.Printf("Task Add: %v: %v %v\n", task.State, app.Id, task.Host) } -func (logger *EventLoggerModule) RemoveTask(task *AppBackend, app *AppCluster) { +func (logger *EventLoggerModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { log.Printf("Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) } diff --git a/mod_files.go b/mod_files.go index f57670c..faa0e3d 100644 --- a/mod_files.go +++ b/mod_files.go @@ -8,6 +8,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/dawanda/mmsd/module_api" ) type FilesManager struct { @@ -27,7 +29,7 @@ func (manager *FilesManager) Startup() { func (manager *FilesManager) Shutdown() { } -func (manager *FilesManager) RemoveTask(task *AppBackend, app *AppCluster) { +func (manager *FilesManager) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { if app != nil { manager.writeApp(app) } else { @@ -35,11 +37,11 @@ func (manager *FilesManager) RemoveTask(task *AppBackend, app *AppCluster) { } } -func (upstream *FilesManager) AddTask(task *AppBackend, app *AppCluster) { +func (upstream *FilesManager) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { upstream.writeApp(app) } -func (upstream *FilesManager) Apply(apps []*AppCluster) { +func (upstream *FilesManager) Apply(apps []*module_api.AppCluster) { err := os.MkdirAll(upstream.BasePath, 0770) if err != nil { log.Printf("Failed to mkdir. %v", err) @@ -66,7 +68,7 @@ func (upstream *FilesManager) Apply(apps []*AppCluster) { } } -func (upstream *FilesManager) writeApp(app *AppCluster) ([]string, error) { +func (upstream *FilesManager) writeApp(app *module_api.AppCluster) ([]string, error) { var files []string app_id := app.Id @@ -93,7 +95,7 @@ func (upstream *FilesManager) writeApp(app *AppCluster) ([]string, error) { } func (upstream *FilesManager) writeFile(filename string, appId string, - app *AppCluster) error { + app *module_api.AppCluster) error { var b bytes.Buffer b.WriteString(fmt.Sprintf("Service-Name: %v\r\n", appId)) diff --git a/mod_haproxy.go b/mod_haproxy.go index 3617789..fb0d4e9 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -13,6 +13,8 @@ import ( "strconv" "strings" "syscall" + + "github.com/dawanda/mmsd/module_api" ) const ( @@ -69,7 +71,7 @@ func (module *HaproxyModule) Shutdown() { log.Printf("Haproxy shutting down") } -func (module *HaproxyModule) Apply(apps []*AppCluster) { +func (module *HaproxyModule) Apply(apps []*module_api.AppCluster) { log.Printf("Haproxy: apply([]AppCluster)") for _, app := range apps { @@ -90,7 +92,7 @@ func (module *HaproxyModule) Apply(apps []*AppCluster) { } } -func (module *HaproxyModule) AddTask(task *AppBackend, app *AppCluster) { +func (module *HaproxyModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { if !module.supportsProtocol(app.Protocol) { return } @@ -98,12 +100,12 @@ func (module *HaproxyModule) AddTask(task *AppBackend, app *AppCluster) { // TODO } -func (module *HaproxyModule) RemoveTask(task *AppBackend, app *AppCluster) { +func (module *HaproxyModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { log.Printf("Haproxy: RemoveTask()") // TODO } -func (module *HaproxyModule) makeConfig(app *AppCluster) string { +func (module *HaproxyModule) makeConfig(app *module_api.AppCluster) string { module.updateGatewaySettings(app) // generate haproxy config fragment @@ -204,7 +206,7 @@ func (module *HaproxyModule) makeConfig(app *AppCluster) string { return result } -func (module *HaproxyModule) updateGatewaySettings(app *AppCluster) { +func (module *HaproxyModule) updateGatewaySettings(app *module_api.AppCluster) { // update HTTP virtual hosting var lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTP]) if len(lbVirtualHosts) != 0 { @@ -530,7 +532,7 @@ func (module *HaproxyModule) exec(logMessage string, args ...string) error { return err } -func (module *HaproxyModule) getAppProtocol(app *AppCluster) string { +func (module *HaproxyModule) getAppProtocol(app *module_api.AppCluster) string { if app.HealthCheck != nil { if app.HealthCheck.Protocol != "COMMAND" { return strings.ToUpper(app.HealthCheck.Protocol) diff --git a/app.go b/module_api/app.go similarity index 98% rename from app.go rename to module_api/app.go index 1d5c75f..36c2a2a 100644 --- a/app.go +++ b/module_api/app.go @@ -1,4 +1,4 @@ -package main +package module_api // Marathon independant application definitions diff --git a/modules/dns.go b/modules/dns.go new file mode 100644 index 0000000..6fd7292 --- /dev/null +++ b/modules/dns.go @@ -0,0 +1,200 @@ +package modules + +// DNS based service discovery +// --------------------------------------------------------------------------- +// +// Marathon: "/path/to/application" +// DNS-query: application.to.path.$basedomain (A, AAAA, TXT, SRV) +// DNS-reply: +// A => list of IPv4 addresses +// AAAA => list of IPv6 addresses +// SRV => ip:port array per task +// TXT => app labels + +// TODO: DNS forwarding +// TODO: DNS proxy cache (for speeding up) + +import ( + "fmt" + "log" + "net" + "strings" + "sync" + "time" + + "github.com/dawanda/mmsd/module_api" + "github.com/miekg/dns" +) + +type DNSModule struct { + Verbose bool + ServiceAddr net.IP + ServicePort uint + BaseName string + DNSTTL time.Duration + PushSRV bool + udpServer *dns.Server + tcpServer *dns.Server + db map[string]*dbEntry + dbMutex sync.Mutex +} + +type dbEntry struct { + ipAddresses []net.IP + app *module_api.AppCluster +} + +func (module *DNSModule) Startup() { + dns.HandleFunc(module.BaseName, module.dnsHandler) + + go func() { + module.udpServer = &dns.Server{ + Addr: fmt.Sprintf("%v:%v", module.ServiceAddr, module.ServicePort), + Net: "udp", + TsigSecret: nil, + } + err := module.udpServer.ListenAndServe() + if err != nil { + log.Fatal(err) + } + }() + + go func() { + module.tcpServer = &dns.Server{ + Addr: fmt.Sprintf("%v:%v", module.ServiceAddr, module.ServicePort), + Net: "tcp", + TsigSecret: nil, + } + err := module.tcpServer.ListenAndServe() + if err != nil { + log.Fatal(err) + } + }() +} + +func (module *DNSModule) Shutdown() { + module.udpServer.Shutdown() + module.tcpServer.Shutdown() +} + +func (module *DNSModule) Apply(apps []*module_api.AppCluster) { + module.dbMutex.Lock() + module.db = make(map[string]*dbEntry) + module.dbMutex.Unlock() + + for _, app := range apps { + if err := module.update(app); err != nil { + return + } + } +} + +func (module *DNSModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { + module.update(app) +} + +func (module *DNSModule) update(app *module_api.AppCluster) error { + var ipAddresses []net.IP + + for _, backend := range app.Backends { + ip, err := net.ResolveIPAddr("ip", backend.Host) + if err != nil { + return err + } + ipAddresses = append(ipAddresses, ip.IP) + } + + var reversed = module.makeDnsNameFromAppId(app.Id) + var entry = &dbEntry{ + ipAddresses: ipAddresses, + app: app, + } + + module.dbMutex.Lock() + module.db[reversed] = entry + module.dbMutex.Unlock() + + return nil +} + +func (module *DNSModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { + module.update(app) +} + +func (module *DNSModule) dnsHandler(w dns.ResponseWriter, req *dns.Msg) { + m := new(dns.Msg) + m.SetReply(req) + + name := req.Question[0].Name + name = strings.TrimSuffix(name, "."+module.BaseName) + + module.dbMutex.Lock() + entry, ok := module.db[name] + module.dbMutex.Unlock() + + if ok { + switch req.Question[0].Qtype { + case dns.TypeSRV: + m.Answer = module.makeAllSRV(entry) + case dns.TypeA: + m.Answer = module.makeAllA(entry) + if module.PushSRV { + m.Extra = module.makeAllSRV(entry) + } + } + } + + w.WriteMsg(m) +} + +func (module *DNSModule) makeAllA(entry *dbEntry) []dns.RR { + var result []dns.RR + + for _, ip := range entry.ipAddresses { + rr := &dns.A{ + Hdr: dns.RR_Header{ + Ttl: uint32(module.DNSTTL.Seconds()), + Name: module.BaseName, + Class: dns.ClassINET, + Rrtype: dns.TypeA, + }, + A: ip.To4(), + } + result = append(result, rr) + } + + return result +} + +func (module *DNSModule) makeAllSRV(entry *dbEntry) []dns.RR { + var result []dns.RR + + for _, task := range entry.app.Backends { + rr := &dns.SRV{ + Hdr: dns.RR_Header{ + Ttl: uint32(module.DNSTTL.Seconds()), + Name: module.BaseName, + Class: dns.ClassINET, + Rrtype: dns.TypeSRV, + }, + Port: uint16(task.Port), + Target: task.Host + ".", + Weight: 1, + Priority: 1, + } + result = append(result, rr) + } + + return result +} + +func (module *DNSModule) makeDnsNameFromAppId(appID string) string { + var parts = strings.Split(appID, "/")[1:] + var reversedParts []string + for i := range parts { + reversedParts = append(reversedParts, parts[len(parts)-i-1]) + } + var reversed = strings.Join(reversedParts, ".") + + return reversed +} From dc89ef1d104bd148e21615a56c0034f42ac56f01 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 18:32:27 +0200 Subject: [PATCH 27/39] Rename module_api to core. looks prettier ^o^ --- {module_api => core}/app.go | 2 +- event_listener.go | 8 ++++---- helper.go | 4 ++-- main.go | 28 ++++++++++++++-------------- mod_eventlogger.go | 8 ++++---- mod_files.go | 12 ++++++------ mod_haproxy.go | 14 +++++++------- modules/dns.go | 12 ++++++------ 8 files changed, 44 insertions(+), 44 deletions(-) rename {module_api => core}/app.go (98%) diff --git a/module_api/app.go b/core/app.go similarity index 98% rename from module_api/app.go rename to core/app.go index 36c2a2a..81a0e0a 100644 --- a/module_api/app.go +++ b/core/app.go @@ -1,4 +1,4 @@ -package module_api +package core // Marathon independant application definitions diff --git a/event_listener.go b/event_listener.go index 3e4c02f..6e56424 100644 --- a/event_listener.go +++ b/event_listener.go @@ -1,6 +1,6 @@ package main -import "github.com/dawanda/mmsd/module_api" +import "github.com/dawanda/mmsd/core" // EventListener provides an interface for hooking into // standard service discovery API calls, such as adding and removing @@ -16,17 +16,17 @@ type EventListener interface { // // This function is invoked upon startup to synchronize with the current // state. - Apply(apps []*module_api.AppCluster) + Apply(apps []*core.AppCluster) // AddTask must add the given backend to the cluster. // // It is assured that the task to be added is also already added to the // given AppCluster. - AddTask(task *module_api.AppBackend, app *module_api.AppCluster) + AddTask(task *core.AppBackend, app *core.AppCluster) // RemoveTask must remove the given backend from the cluster. // // It is ensured that the task to be removed is not present in the given // AppCluster. - RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) + RemoveTask(task *core.AppBackend, app *core.AppCluster) } diff --git a/helper.go b/helper.go index 2e2bcc5..032b7f3 100644 --- a/helper.go +++ b/helper.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/dawanda/go-mesos/marathon" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" ) var ( @@ -136,7 +136,7 @@ func FindMissing(slice1, slice2 []string) (missing []string) { return } -func GetApplicationProtocol1(app *module_api.AppCluster) string { +func GetApplicationProtocol1(app *core.AppCluster) string { if proto := strings.ToLower(app.Labels["proto"]); len(proto) != 0 { return proto } diff --git a/main.go b/main.go index 0450745..5d28fd4 100644 --- a/main.go +++ b/main.go @@ -41,7 +41,7 @@ import ( "time" "github.com/dawanda/go-mesos/marathon" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" "github.com/dawanda/mmsd/modules" "github.com/gorilla/mux" flag "github.com/ogier/pflag" @@ -95,7 +95,7 @@ type mmsdService struct { DNSPushSRV bool // runtime state - apps []*module_api.AppCluster + apps []*core.AppCluster killingTasks map[string]bool // set of tasks currently in killing state } @@ -265,19 +265,19 @@ func (mmsd *mmsdService) getMarathonApp(appID string) (*marathon.App, error) { } // convertMarathonApps converts an array of marathon.App into a []AppCluster. -func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*module_api.AppCluster { - var apps []*module_api.AppCluster +func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*core.AppCluster { + var apps []*core.AppCluster for _, mApp := range mApps { for portIndex, portDef := range mApp.PortDefinitions { if mmsd.isGroupIncluded(portDef.Labels["lb-group"]) { - var healthCheck *module_api.AppHealthCheck + var healthCheck *core.AppHealthCheck if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { var mCommand *string if mHealthCheck.Command != nil { mCommand = new(string) *mCommand = mHealthCheck.Command.Value } - healthCheck = &module_api.AppHealthCheck{ + healthCheck = &core.AppHealthCheck{ Protocol: mHealthCheck.Protocol, Path: mHealthCheck.Path, Command: mCommand, @@ -289,9 +289,9 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*module_api } } - var backends []module_api.AppBackend + var backends []core.AppBackend for _, mTask := range mApp.Tasks { - backends = append(backends, module_api.AppBackend{ + backends = append(backends, core.AppBackend{ Id: mTask.Id, Host: mTask.Host, Port: mTask.Ports[portIndex], @@ -309,7 +309,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*module_api } servicePort := mApp.PortDefinitions[portIndex].Port - app := &module_api.AppCluster{ + app := &core.AppCluster{ Name: mApp.Id, Id: PrettifyAppId(mApp.Id, portIndex, servicePort), ServicePort: servicePort, @@ -327,7 +327,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*module_api return apps } -func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *module_api.AppCluster { +func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *core.AppCluster { log.Printf("Application %v with port index %v not found.", appId, portIndex) return nil @@ -335,8 +335,8 @@ func (mmsd *mmsdService) getAppByMarathonId(appId string, portIndex int) *module // findAppsByMarathonId returns list of all applications that belong to the // given Marathon App mAppId. -func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*module_api.AppCluster { - var apps []*module_api.AppCluster +func (mmsd *mmsdService) findAppsByMarathonId(mAppId string) []*core.AppCluster { + var apps []*core.AppCluster for _, app := range mmsd.apps { if app.Name == mAppId { apps = append(apps, app) @@ -437,7 +437,7 @@ func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { func (mmsd *mmsdService) AddTask(appId, taskId, taskStatus, host string, ports []uint) { for portIndex, port := range ports { if app := mmsd.getAppByMarathonId(appId, portIndex); app != nil { - task := module_api.AppBackend{ + task := core.AppBackend{ Id: taskId, Host: host, Port: port, @@ -538,7 +538,7 @@ func (mmsd *mmsdService) Shutdown() { mmsd.quitChannel <- true } -func (mmsd *mmsdService) applyApps(apps []*module_api.AppCluster) { +func (mmsd *mmsdService) applyApps(apps []*core.AppCluster) { mmsd.apps = apps for _, handler := range mmsd.Handlers { diff --git a/mod_eventlogger.go b/mod_eventlogger.go index d604a09..1cd34fe 100644 --- a/mod_eventlogger.go +++ b/mod_eventlogger.go @@ -5,7 +5,7 @@ import ( "fmt" "log" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" ) /* EventLoggerModule adds simple event logging to the logger. @@ -22,7 +22,7 @@ func (logger *EventLoggerModule) Shutdown() { log.Printf("Shutdown\n") } -func (logger *EventLoggerModule) Apply(apps []*module_api.AppCluster) { +func (logger *EventLoggerModule) Apply(apps []*core.AppCluster) { if logger.Verbose { out, err := json.MarshalIndent(apps, "", " ") if err != nil { @@ -37,10 +37,10 @@ func (logger *EventLoggerModule) Apply(apps []*module_api.AppCluster) { } } -func (logger *EventLoggerModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (logger *EventLoggerModule) AddTask(task *core.AppBackend, app *core.AppCluster) { log.Printf("Task Add: %v: %v %v\n", task.State, app.Id, task.Host) } -func (logger *EventLoggerModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (logger *EventLoggerModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { log.Printf("Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) } diff --git a/mod_files.go b/mod_files.go index faa0e3d..ea24471 100644 --- a/mod_files.go +++ b/mod_files.go @@ -9,7 +9,7 @@ import ( "path/filepath" "strings" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" ) type FilesManager struct { @@ -29,7 +29,7 @@ func (manager *FilesManager) Startup() { func (manager *FilesManager) Shutdown() { } -func (manager *FilesManager) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (manager *FilesManager) RemoveTask(task *core.AppBackend, app *core.AppCluster) { if app != nil { manager.writeApp(app) } else { @@ -37,11 +37,11 @@ func (manager *FilesManager) RemoveTask(task *module_api.AppBackend, app *module } } -func (upstream *FilesManager) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (upstream *FilesManager) AddTask(task *core.AppBackend, app *core.AppCluster) { upstream.writeApp(app) } -func (upstream *FilesManager) Apply(apps []*module_api.AppCluster) { +func (upstream *FilesManager) Apply(apps []*core.AppCluster) { err := os.MkdirAll(upstream.BasePath, 0770) if err != nil { log.Printf("Failed to mkdir. %v", err) @@ -68,7 +68,7 @@ func (upstream *FilesManager) Apply(apps []*module_api.AppCluster) { } } -func (upstream *FilesManager) writeApp(app *module_api.AppCluster) ([]string, error) { +func (upstream *FilesManager) writeApp(app *core.AppCluster) ([]string, error) { var files []string app_id := app.Id @@ -95,7 +95,7 @@ func (upstream *FilesManager) writeApp(app *module_api.AppCluster) ([]string, er } func (upstream *FilesManager) writeFile(filename string, appId string, - app *module_api.AppCluster) error { + app *core.AppCluster) error { var b bytes.Buffer b.WriteString(fmt.Sprintf("Service-Name: %v\r\n", appId)) diff --git a/mod_haproxy.go b/mod_haproxy.go index fb0d4e9..639f0a9 100644 --- a/mod_haproxy.go +++ b/mod_haproxy.go @@ -14,7 +14,7 @@ import ( "strings" "syscall" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" ) const ( @@ -71,7 +71,7 @@ func (module *HaproxyModule) Shutdown() { log.Printf("Haproxy shutting down") } -func (module *HaproxyModule) Apply(apps []*module_api.AppCluster) { +func (module *HaproxyModule) Apply(apps []*core.AppCluster) { log.Printf("Haproxy: apply([]AppCluster)") for _, app := range apps { @@ -92,7 +92,7 @@ func (module *HaproxyModule) Apply(apps []*module_api.AppCluster) { } } -func (module *HaproxyModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (module *HaproxyModule) AddTask(task *core.AppBackend, app *core.AppCluster) { if !module.supportsProtocol(app.Protocol) { return } @@ -100,12 +100,12 @@ func (module *HaproxyModule) AddTask(task *module_api.AppBackend, app *module_ap // TODO } -func (module *HaproxyModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (module *HaproxyModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { log.Printf("Haproxy: RemoveTask()") // TODO } -func (module *HaproxyModule) makeConfig(app *module_api.AppCluster) string { +func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { module.updateGatewaySettings(app) // generate haproxy config fragment @@ -206,7 +206,7 @@ func (module *HaproxyModule) makeConfig(app *module_api.AppCluster) string { return result } -func (module *HaproxyModule) updateGatewaySettings(app *module_api.AppCluster) { +func (module *HaproxyModule) updateGatewaySettings(app *core.AppCluster) { // update HTTP virtual hosting var lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTP]) if len(lbVirtualHosts) != 0 { @@ -532,7 +532,7 @@ func (module *HaproxyModule) exec(logMessage string, args ...string) error { return err } -func (module *HaproxyModule) getAppProtocol(app *module_api.AppCluster) string { +func (module *HaproxyModule) getAppProtocol(app *core.AppCluster) string { if app.HealthCheck != nil { if app.HealthCheck.Protocol != "COMMAND" { return strings.ToUpper(app.HealthCheck.Protocol) diff --git a/modules/dns.go b/modules/dns.go index 6fd7292..70d119b 100644 --- a/modules/dns.go +++ b/modules/dns.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/dawanda/mmsd/module_api" + "github.com/dawanda/mmsd/core" "github.com/miekg/dns" ) @@ -41,7 +41,7 @@ type DNSModule struct { type dbEntry struct { ipAddresses []net.IP - app *module_api.AppCluster + app *core.AppCluster } func (module *DNSModule) Startup() { @@ -77,7 +77,7 @@ func (module *DNSModule) Shutdown() { module.tcpServer.Shutdown() } -func (module *DNSModule) Apply(apps []*module_api.AppCluster) { +func (module *DNSModule) Apply(apps []*core.AppCluster) { module.dbMutex.Lock() module.db = make(map[string]*dbEntry) module.dbMutex.Unlock() @@ -89,11 +89,11 @@ func (module *DNSModule) Apply(apps []*module_api.AppCluster) { } } -func (module *DNSModule) AddTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (module *DNSModule) AddTask(task *core.AppBackend, app *core.AppCluster) { module.update(app) } -func (module *DNSModule) update(app *module_api.AppCluster) error { +func (module *DNSModule) update(app *core.AppCluster) error { var ipAddresses []net.IP for _, backend := range app.Backends { @@ -117,7 +117,7 @@ func (module *DNSModule) update(app *module_api.AppCluster) error { return nil } -func (module *DNSModule) RemoveTask(task *module_api.AppBackend, app *module_api.AppCluster) { +func (module *DNSModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { module.update(app) } From cc58b40d6454332b653935220fa5bd6c547e429a Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 19:55:56 +0200 Subject: [PATCH 28/39] renamed modules_api to core, created util & sse namespace, and adapted the rest. --- event_listener.go => core/event_listener.go | 10 +++--- main.go | 32 +++++++++++--------- mod_eventlogger.go => modules/eventlogger.go | 2 +- mod_files.go => modules/files.go | 25 ++++++++++++--- mod_haproxy.go => modules/haproxy.go | 25 +++++++-------- udp_manager.go => modules/udp.go | 7 +++-- sse.go => sse/sse.go | 2 +- helper.go => util/helper.go | 25 +++------------ helper_test.go => util/helper_test.go | 2 +- 9 files changed, 66 insertions(+), 64 deletions(-) rename event_listener.go => core/event_listener.go (80%) rename mod_eventlogger.go => modules/eventlogger.go (98%) rename mod_files.go => modules/files.go (85%) rename mod_haproxy.go => modules/haproxy.go (95%) rename udp_manager.go => modules/udp.go (94%) rename sse.go => sse/sse.go (99%) rename helper.go => util/helper.go (90%) rename helper_test.go => util/helper_test.go (98%) diff --git a/event_listener.go b/core/event_listener.go similarity index 80% rename from event_listener.go rename to core/event_listener.go index 6e56424..3174b16 100644 --- a/event_listener.go +++ b/core/event_listener.go @@ -1,6 +1,4 @@ -package main - -import "github.com/dawanda/mmsd/core" +package core // EventListener provides an interface for hooking into // standard service discovery API calls, such as adding and removing @@ -16,17 +14,17 @@ type EventListener interface { // // This function is invoked upon startup to synchronize with the current // state. - Apply(apps []*core.AppCluster) + Apply(apps []*AppCluster) // AddTask must add the given backend to the cluster. // // It is assured that the task to be added is also already added to the // given AppCluster. - AddTask(task *core.AppBackend, app *core.AppCluster) + AddTask(task *AppBackend, app *AppCluster) // RemoveTask must remove the given backend from the cluster. // // It is ensured that the task to be removed is not present in the given // AppCluster. - RemoveTask(task *core.AppBackend, app *core.AppCluster) + RemoveTask(task *AppBackend, app *AppCluster) } diff --git a/main.go b/main.go index 5d28fd4..100f698 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,8 @@ import ( "github.com/dawanda/go-mesos/marathon" "github.com/dawanda/mmsd/core" "github.com/dawanda/mmsd/modules" + "github.com/dawanda/mmsd/sse" + "github.com/dawanda/mmsd/util" "github.com/gorilla/mux" flag "github.com/ogier/pflag" ) @@ -50,7 +52,7 @@ import ( type mmsdService struct { HttpApiPort uint Verbose bool - Handlers []EventListener + Handlers []core.EventListener quitChannel chan bool RunStateDir string FilterGroups []string @@ -154,7 +156,7 @@ func (mmsd *mmsdService) v1Instances(w http.ResponseWriter, r *http.Request) { noResolve := r.URL.Query().Get("noresolve") == "1" withServerID := r.URL.Query().Get("withid") == "1" - portBegin, portEnd, err := parseRange(r.URL.Query().Get("portIndex")) + portBegin, portEnd, err := util.ParseRange(r.URL.Query().Get("portIndex")) if err != nil { w.WriteHeader(http.StatusBadRequest) log.Printf("error parsing range. %v\n", err) @@ -204,10 +206,10 @@ func (mmsd *mmsdService) v1Instances(w http.ResponseWriter, r *http.Request) { for _, task := range app.Tasks { item := "" if withServerID { - item += fmt.Sprintf("%v:", Hash(task.SlaveId)) + item += fmt.Sprintf("%v:", util.Hash(task.SlaveId)) } - item += resolveIPAddr(task.Host, noResolve) + item += util.ResolveIPAddr(task.Host, noResolve) for portIndex := portBegin; portIndex <= portEnd; portIndex++ { item += fmt.Sprintf(":%d", task.Ports[portIndex]) @@ -271,7 +273,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*core.AppCl for portIndex, portDef := range mApp.PortDefinitions { if mmsd.isGroupIncluded(portDef.Labels["lb-group"]) { var healthCheck *core.AppHealthCheck - if mHealthCheck := FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { + if mHealthCheck := util.FindHealthCheckForPortIndex(mApp.HealthChecks, portIndex); mHealthCheck != nil { var mCommand *string if mHealthCheck.Command != nil { mCommand = new(string) @@ -311,7 +313,7 @@ func (mmsd *mmsdService) convertMarathonApps(mApps []marathon.App) []*core.AppCl servicePort := mApp.PortDefinitions[portIndex].Port app := &core.AppCluster{ Name: mApp.Id, - Id: PrettifyAppId(mApp.Id, portIndex, servicePort), + Id: util.PrettifyAppId(mApp.Id, portIndex, servicePort), ServicePort: servicePort, Protocol: mApp.PortDefinitions[portIndex].Protocol, PortName: mApp.PortDefinitions[portIndex].Name, @@ -349,15 +351,15 @@ func (mmsd *mmsdService) setupEventBusListener() { var url = fmt.Sprintf("http://%v:%v/v2/events", mmsd.MarathonIP, mmsd.MarathonPort) - var sse = NewEventSource(url, mmsd.ReconnectDelay) + var stream = sse.NewEventSource(url, mmsd.ReconnectDelay) - sse.OnOpen = mmsd.OnMarathonConnected - sse.OnError = mmsd.OnMarathonConnectionFailure - //sse.AddEventListener("deployment_info", mmsd.DeploymentStart) - sse.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) - sse.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) + stream.OnOpen = mmsd.OnMarathonConnected + stream.OnError = mmsd.OnMarathonConnectionFailure + //stream.AddEventListener("deployment_info", mmsd.DeploymentStart) + stream.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) + stream.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) - go sse.RunForever() + go stream.RunForever() } func (mmsd *mmsdService) OnMarathonConnected(event, data string) { @@ -571,7 +573,7 @@ func (mmsd *mmsdService) setupHandlers() { // } if mmsd.TCPEnabled { - mmsd.Handlers = append(mmsd.Handlers, &HaproxyModule{ + mmsd.Handlers = append(mmsd.Handlers, &modules.HaproxyModule{ Verbose: mmsd.Verbose, LocalHealthChecks: mmsd.LocalHealthChecks, ServiceAddr: mmsd.ServiceAddr, @@ -591,7 +593,7 @@ func (mmsd *mmsdService) setupHandlers() { } if mmsd.FilesEnabled { - mmsd.Handlers = append(mmsd.Handlers, &FilesManager{ + mmsd.Handlers = append(mmsd.Handlers, &modules.FilesManager{ Verbose: mmsd.Verbose, BasePath: mmsd.RunStateDir + "/confd", }) diff --git a/mod_eventlogger.go b/modules/eventlogger.go similarity index 98% rename from mod_eventlogger.go rename to modules/eventlogger.go index 1cd34fe..2e8d75d 100644 --- a/mod_eventlogger.go +++ b/modules/eventlogger.go @@ -1,4 +1,4 @@ -package main +package modules import ( "encoding/json" diff --git a/mod_files.go b/modules/files.go similarity index 85% rename from mod_files.go rename to modules/files.go index ea24471..ea0d625 100644 --- a/mod_files.go +++ b/modules/files.go @@ -1,4 +1,4 @@ -package main +package modules import ( "bytes" @@ -10,6 +10,7 @@ import ( "strings" "github.com/dawanda/mmsd/core" + "github.com/dawanda/mmsd/util" ) type FilesManager struct { @@ -61,7 +62,7 @@ func (upstream *FilesManager) Apply(apps []*core.AppCluster) { } // check for superfluous files - diff := FindMissing(oldFiles, newFiles) + diff := util.FindMissing(oldFiles, newFiles) for _, superfluous := range diff { upstream.Log(fmt.Sprintf("Removing superfluous file: %v\n", superfluous)) os.Remove(superfluous) @@ -84,7 +85,7 @@ func (upstream *FilesManager) writeApp(app *core.AppCluster) ([]string, error) { if _, err := os.Stat(cfgfile); os.IsNotExist(err) { upstream.Log(fmt.Sprintf("new %v", cfgfile)) os.Rename(tmpfile, cfgfile) - } else if !FileIsIdentical(tmpfile, cfgfile) { + } else if !util.FileIsIdentical(tmpfile, cfgfile) { upstream.Log(fmt.Sprintf("refresh %v", cfgfile)) os.Rename(tmpfile, cfgfile) } else { @@ -94,6 +95,22 @@ func (upstream *FilesManager) writeApp(app *core.AppCluster) ([]string, error) { return files, nil } +func getApplicationProtocol1(app *core.AppCluster) string { + if proto := strings.ToLower(app.Labels["proto"]); len(proto) != 0 { + return proto + } + + if app.HealthCheck != nil && len(app.HealthCheck.Protocol) != 0 { + return strings.ToLower(app.HealthCheck.Protocol) + } + + if len(app.Protocol) != 0 { + return strings.ToLower(app.Protocol) + } + + return "tcp" +} + func (upstream *FilesManager) writeFile(filename string, appId string, app *core.AppCluster) error { @@ -101,7 +118,7 @@ func (upstream *FilesManager) writeFile(filename string, appId string, b.WriteString(fmt.Sprintf("Service-Name: %v\r\n", appId)) b.WriteString(fmt.Sprintf("Service-Port: %v\r\n", app.ServicePort)) b.WriteString(fmt.Sprintf("Service-Transport-Proto: %v\r\n", app.Protocol)) - b.WriteString(fmt.Sprintf("Service-Application-Proto: %v\r\n", GetApplicationProtocol1(app))) + b.WriteString(fmt.Sprintf("Service-Application-Proto: %v\r\n", getApplicationProtocol1(app))) if app.HealthCheck != nil && len(app.HealthCheck.Protocol) != 0 { b.WriteString(fmt.Sprintf("Health-Check-Proto: %v\r\n", strings.ToLower(app.HealthCheck.Protocol))) } diff --git a/mod_haproxy.go b/modules/haproxy.go similarity index 95% rename from mod_haproxy.go rename to modules/haproxy.go index 639f0a9..c1b6420 100644 --- a/mod_haproxy.go +++ b/modules/haproxy.go @@ -1,4 +1,4 @@ -package main +package modules import ( "errors" @@ -15,6 +15,7 @@ import ( "syscall" "github.com/dawanda/mmsd/core" + "github.com/dawanda/mmsd/util" ) const ( @@ -117,7 +118,7 @@ func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { bindOpts += " defer-accept" } - if Atoi(app.Labels[LB_ACCEPT_PROXY], 0) != 0 { + if util.Atoi(app.Labels[LB_ACCEPT_PROXY], 0) != 0 { bindOpts += " accept-proxy" } @@ -129,7 +130,7 @@ func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { serverOpts += fmt.Sprintf(" inter %v", app.HealthCheck.IntervalSeconds*1000) } - switch Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { + switch util.Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { case 2: serverOpts += " send-proxy-v2" case 1: @@ -141,7 +142,7 @@ func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { app.Id, app.Labels["lb-proxy-protocol"]) } - switch Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { + switch util.Atoi(app.Labels[LB_PROXY_PROTOCOL], 0) { case 2: serverOpts += " send-proxy-v2" case 1: @@ -193,7 +194,7 @@ func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { result += fmt.Sprintf( " server %v:%v %v:%v%v\n", task.Host, task.Port, // taskLabel == "$host:$port" - SoftResolveIPAddr(task.Host), + util.SoftResolveIPAddr(task.Host), task.Port, serverOpts) } else { @@ -208,7 +209,7 @@ func (module *HaproxyModule) makeConfig(app *core.AppCluster) string { func (module *HaproxyModule) updateGatewaySettings(app *core.AppCluster) { // update HTTP virtual hosting - var lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTP]) + var lbVirtualHosts = util.MakeStringArray(app.Labels[LB_VHOST_HTTP]) if len(lbVirtualHosts) != 0 { module.vhostsHTTP[app.Id] = lbVirtualHosts if app.Labels[LB_VHOST_DEFAULT_HTTP] == "1" { @@ -222,7 +223,7 @@ func (module *HaproxyModule) updateGatewaySettings(app *core.AppCluster) { } // update HTTPS virtual hosting - lbVirtualHosts = makeStringArray(app.Labels[LB_VHOST_HTTPS]) + lbVirtualHosts = util.MakeStringArray(app.Labels[LB_VHOST_HTTPS]) if len(lbVirtualHosts) != 0 { module.vhostsHTTPS[app.Id] = lbVirtualHosts if app.Labels[LB_VHOST_DEFAULT_HTTPS] == "1" { @@ -367,12 +368,12 @@ func (module *HaproxyModule) makeGatewayHTTP() string { fragment += "\n" } - for _, acl := range SortedStrStrKeys(exactRoutes) { + for _, acl := range util.SortedStrStrKeys(exactRoutes) { appID := exactRoutes[acl] fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) } - for _, acl := range SortedStrStrKeys(suffixRoutes) { + for _, acl := range util.SortedStrStrKeys(suffixRoutes) { appID := suffixRoutes[acl] fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) } @@ -438,12 +439,12 @@ func (module *HaproxyModule) makeGatewayHTTPS() string { fragment += "\n" } - for _, acl := range SortedStrStrKeys(exactRoutes) { + for _, acl := range util.SortedStrStrKeys(exactRoutes) { appID := exactRoutes[acl] fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) } - for _, acl := range SortedStrStrKeys(suffixRoutes) { + for _, acl := range util.SortedStrStrKeys(suffixRoutes) { appID := suffixRoutes[acl] fragment += fmt.Sprintf(" use_backend %v if %v\n", appID, acl) } @@ -472,7 +473,7 @@ func (module *HaproxyModule) makeConfigTail() string { } func (module *HaproxyModule) reloadConfig() error { - if FileIsIdentical(module.ConfigPath, module.OldConfigPath) { + if util.FileIsIdentical(module.ConfigPath, module.OldConfigPath) { log.Printf("[haproxy] config file not changed. ignoring reload\n") return nil } diff --git a/udp_manager.go b/modules/udp.go similarity index 94% rename from udp_manager.go rename to modules/udp.go index 0013d53..b1b23ba 100644 --- a/udp_manager.go +++ b/modules/udp.go @@ -1,4 +1,4 @@ -package main +package modules import ( "fmt" @@ -8,6 +8,7 @@ import ( "github.com/dawanda/go-mesos/marathon" "github.com/dawanda/mmsd/udpproxy" + "github.com/dawanda/mmsd/util" ) type UdpManager struct { @@ -43,7 +44,7 @@ func (manager *UdpManager) Apply(apps []*marathon.App, force bool) error { func (manager *UdpManager) GetFrontend(app *marathon.App, portIndex int, replace bool) (*udpproxy.Frontend, error) { servicePort := app.PortDefinitions[portIndex].Port - name := PrettifyAppId(app.Id, portIndex, servicePort) + name := util.PrettifyAppId(app.Id, portIndex, servicePort) server, ok := manager.Servers[name] if ok { @@ -91,7 +92,7 @@ func (manager *UdpManager) removeApp(appID string) error { func (manager *UdpManager) applyApp(app *marathon.App) error { for portIndex := range app.Ports { - if GetTransportProtocol(app, portIndex) == "udp" { + if util.GetTransportProtocol(app, portIndex) == "udp" { fe, err := manager.GetFrontend(app, portIndex, true) if err != nil { log.Printf("Error spawning UDP frontend. %v\n", err) diff --git a/sse.go b/sse/sse.go similarity index 99% rename from sse.go rename to sse/sse.go index dbd9789..ee89030 100644 --- a/sse.go +++ b/sse/sse.go @@ -1,4 +1,4 @@ -package main +package sse import ( "bufio" diff --git a/helper.go b/util/helper.go similarity index 90% rename from helper.go rename to util/helper.go index 032b7f3..a4d8909 100644 --- a/helper.go +++ b/util/helper.go @@ -1,4 +1,4 @@ -package main +package util import ( "bytes" @@ -14,7 +14,6 @@ import ( "strings" "github.com/dawanda/go-mesos/marathon" - "github.com/dawanda/mmsd/core" ) var ( @@ -136,22 +135,6 @@ func FindMissing(slice1, slice2 []string) (missing []string) { return } -func GetApplicationProtocol1(app *core.AppCluster) string { - if proto := strings.ToLower(app.Labels["proto"]); len(proto) != 0 { - return proto - } - - if app.HealthCheck != nil && len(app.HealthCheck.Protocol) != 0 { - return strings.ToLower(app.HealthCheck.Protocol) - } - - if len(app.Protocol) != 0 { - return strings.ToLower(app.Protocol) - } - - return "tcp" -} - func GetApplicationProtocol(app *marathon.App, portIndex int) (proto string) { if proto = strings.ToLower(app.Labels["proto"]); len(proto) != 0 { return @@ -221,7 +204,7 @@ func Hash(s string) uint32 { return h.Sum32() } -func resolveIPAddr(dns string, skip bool) string { +func ResolveIPAddr(dns string, skip bool) string { if skip { return dns } else { @@ -234,7 +217,7 @@ func resolveIPAddr(dns string, skip bool) string { } } -func parseRange(input string) (int, int, error) { +func ParseRange(input string) (int, int, error) { if len(input) == 0 { return 0, 0, nil } @@ -278,7 +261,7 @@ func parseRange(input string) (int, int, error) { return begin, end, err } -func makeStringArray(s string) []string { +func MakeStringArray(s string) []string { if len(s) == 0 { return []string{} } else { diff --git a/helper_test.go b/util/helper_test.go similarity index 98% rename from helper_test.go rename to util/helper_test.go index c767253..185d062 100644 --- a/helper_test.go +++ b/util/helper_test.go @@ -1,4 +1,4 @@ -package main +package util import ( "testing" From c189804ad37be8e81dffcdd3ff990d9115e60d5b Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 20:11:55 +0200 Subject: [PATCH 29/39] slightly improved eventlogger output (just prefixes, yeah) --- modules/eventlogger.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/eventlogger.go b/modules/eventlogger.go index 2e8d75d..5855ac4 100644 --- a/modules/eventlogger.go +++ b/modules/eventlogger.go @@ -15,32 +15,32 @@ type EventLoggerModule struct { } func (logger *EventLoggerModule) Startup() { - log.Printf("Initialize\n") + log.Printf("eventlogger: Initialize\n") } func (logger *EventLoggerModule) Shutdown() { - log.Printf("Shutdown\n") + log.Printf("eventlogger: Shutdown\n") } func (logger *EventLoggerModule) Apply(apps []*core.AppCluster) { if logger.Verbose { out, err := json.MarshalIndent(apps, "", " ") if err != nil { - log.Printf("Marshal failed. %v\n", err) + log.Printf("eventlogger: Marshal failed. %v\n", err) } else { - fmt.Printf("%v\n", string(out)) + fmt.Printf("eventlogger: %v\n", string(out)) } } for _, app := range apps { - log.Printf("Apply: %v\n", app.Id) + log.Printf("eventlogger: Apply: %v\n", app.Id) } } func (logger *EventLoggerModule) AddTask(task *core.AppBackend, app *core.AppCluster) { - log.Printf("Task Add: %v: %v %v\n", task.State, app.Id, task.Host) + log.Printf("eventlogger: Task Add: %v: %v %v\n", task.State, app.Id, task.Host) } func (logger *EventLoggerModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { - log.Printf("Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) + log.Printf("eventlogger: Task Remove: %v: %v %v\n", task.State, app.Id, task.Host) } From e42b90eb4a8a99e7156d4ee1cde99518e2c1bc35 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Tue, 20 Sep 2016 20:12:10 +0200 Subject: [PATCH 30/39] adds eventlogger module to CLI param list --- main.go | 66 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/main.go b/main.go index 100f698..1f91b00 100644 --- a/main.go +++ b/main.go @@ -65,6 +65,8 @@ type mmsdService struct { // service discovery IP-management ManagedIP net.IP + EventLoggerEnabled bool + // file based service discovery FilesEnabled bool @@ -494,6 +496,7 @@ func (mmsd *mmsdService) Run() { flag.IPVar(&mmsd.GatewayAddr, "gateway-bind", mmsd.GatewayAddr, "gateway bind address") flag.UintVar(&mmsd.GatewayPortHTTP, "gateway-port-http", mmsd.GatewayPortHTTP, "gateway port for HTTP") flag.UintVar(&mmsd.GatewayPortHTTPS, "gateway-port-https", mmsd.GatewayPortHTTPS, "gateway port for HTTPS") + flag.BoolVar(&mmsd.EventLoggerEnabled, "enable-eventlogger", mmsd.EventLoggerEnabled, "enables eventlogger module") flag.BoolVar(&mmsd.FilesEnabled, "enable-files", mmsd.FilesEnabled, "enables file based service discovery") flag.BoolVar(&mmsd.UDPEnabled, "enable-udp", mmsd.UDPEnabled, "enables UDP load balancing") flag.BoolVar(&mmsd.TCPEnabled, "enable-tcp", mmsd.TCPEnabled, "enables haproxy TCP load balancing") @@ -549,9 +552,11 @@ func (mmsd *mmsdService) applyApps(apps []*core.AppCluster) { } func (mmsd *mmsdService) setupHandlers() { - // mmsd.Handlers = append(mmsd.Handlers, &EventLoggerModule{ - // Verbose: true, - // }) + if mmsd.EventLoggerEnabled { + mmsd.Handlers = append(mmsd.Handlers, &modules.EventLoggerModule{ + Verbose: mmsd.Verbose, + }) + } if mmsd.DNSEnabled { mmsd.Handlers = append(mmsd.Handlers, &modules.DNSModule{ @@ -616,33 +621,34 @@ func locateExe(name string) string { func main() { var mmsd = mmsdService{ - MarathonScheme: "http", - MarathonIP: net.ParseIP("127.0.0.1"), - MarathonPort: 8080, - ReconnectDelay: time.Second * 4, - RunStateDir: "/var/run/mmsd", - GatewayEnabled: false, - GatewayAddr: net.ParseIP("0.0.0.0"), - GatewayPortHTTP: 80, - GatewayPortHTTPS: 443, - FilesEnabled: false, - UDPEnabled: false, - TCPEnabled: false, - LocalHealthChecks: true, - HaproxyBin: locateExe("haproxy"), - HaproxyTailCfg: "/etc/mmsd/haproxy-tail.cfg", - HaproxyPort: 8081, - ManagementAddr: net.ParseIP("0.0.0.0"), - ServiceAddr: net.ParseIP("0.0.0.0"), - HttpApiPort: 8082, - Verbose: false, - DNSEnabled: false, - DNSPort: 53, - DNSPushSRV: false, - DNSBaseName: "mmsd.", - DNSTTL: time.Second * 5, - quitChannel: make(chan bool), - killingTasks: make(map[string]bool), + MarathonScheme: "http", + MarathonIP: net.ParseIP("127.0.0.1"), + MarathonPort: 8080, + ReconnectDelay: time.Second * 4, + RunStateDir: "/var/run/mmsd", + GatewayEnabled: false, + GatewayAddr: net.ParseIP("0.0.0.0"), + GatewayPortHTTP: 80, + GatewayPortHTTPS: 443, + EventLoggerEnabled: false, + FilesEnabled: false, + UDPEnabled: false, + TCPEnabled: false, + LocalHealthChecks: true, + HaproxyBin: locateExe("haproxy"), + HaproxyTailCfg: "/etc/mmsd/haproxy-tail.cfg", + HaproxyPort: 8081, + ManagementAddr: net.ParseIP("0.0.0.0"), + ServiceAddr: net.ParseIP("0.0.0.0"), + HttpApiPort: 8082, + Verbose: false, + DNSEnabled: false, + DNSPort: 53, + DNSPushSRV: false, + DNSBaseName: "mmsd.", + DNSTTL: time.Second * 5, + quitChannel: make(chan bool), + killingTasks: make(map[string]bool), } // trap SIGTERM and SIGINT From 6edc0cc5a873eef2637f232d0a176668868b0946 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 21 Sep 2016 00:39:10 +0200 Subject: [PATCH 31/39] haproxy module cleanup --- modules/haproxy.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/haproxy.go b/modules/haproxy.go index c1b6420..57530c1 100644 --- a/modules/haproxy.go +++ b/modules/haproxy.go @@ -57,8 +57,6 @@ type HaproxyModule struct { } func (module *HaproxyModule) Startup() { - log.Printf("Haproxy starting") - module.vhostsHTTP = make(map[string][]string) module.vhostDefaultHTTP = "" @@ -69,18 +67,19 @@ func (module *HaproxyModule) Startup() { } func (module *HaproxyModule) Shutdown() { - log.Printf("Haproxy shutting down") } func (module *HaproxyModule) Apply(apps []*core.AppCluster) { - log.Printf("Haproxy: apply([]AppCluster)") - for _, app := range apps { if module.supportsProtocol(app.Protocol) { module.appConfigCache[app.Id] = module.makeConfig(app) } } + module.Commit() +} + +func (module *HaproxyModule) Commit() { err := module.writeConfig() if err != nil { log.Printf("Failed to write config. %v", err) From 7afe6b934f2f54ad4fc7046422d1f217c3077fdb Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 21 Sep 2016 00:53:15 +0200 Subject: [PATCH 32/39] conform to hipster bus-words --- main.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 1f91b00..d1470db 100644 --- a/main.go +++ b/main.go @@ -353,15 +353,17 @@ func (mmsd *mmsdService) setupEventBusListener() { var url = fmt.Sprintf("http://%v:%v/v2/events", mmsd.MarathonIP, mmsd.MarathonPort) - var stream = sse.NewEventSource(url, mmsd.ReconnectDelay) + var bus = sse.NewEventSource(url, mmsd.ReconnectDelay) - stream.OnOpen = mmsd.OnMarathonConnected - stream.OnError = mmsd.OnMarathonConnectionFailure - //stream.AddEventListener("deployment_info", mmsd.DeploymentStart) - stream.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) - stream.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) + bus.OnOpen = mmsd.OnMarathonConnected + bus.OnError = mmsd.OnMarathonConnectionFailure + //bus.AddEventListener("deployment_info", mmsd.DeploymentStart) + //bus.AddEventListener("deployment_success", mmsd.DeploymentSuccess) + //bus.AddEventListener("deployment_failed", mmsd.DeploymentFailed) + bus.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) + bus.AddEventListener("health_status_changed_event", mmsd.HealthStatusChangedEvent) - go stream.RunForever() + go bus.RunForever() } func (mmsd *mmsdService) OnMarathonConnected(event, data string) { From 2cbea5afb9d78ff2680dc04422d1dce702446687 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Wed, 21 Sep 2016 01:06:03 +0200 Subject: [PATCH 33/39] wip --- main.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index d1470db..eb515f4 100644 --- a/main.go +++ b/main.go @@ -99,8 +99,9 @@ type mmsdService struct { DNSPushSRV bool // runtime state - apps []*core.AppCluster - killingTasks map[string]bool // set of tasks currently in killing state + apps []*core.AppCluster // cache of latest state of all app clusters + taskEvents map[string]marathon.StatusUpdateEvent // map of last task status update events + killingTasks map[string]bool // set of tasks currently in killing state } // {{{ HTTP endpoint @@ -408,8 +409,9 @@ func (mmsd *mmsdService) StatusUpdateEvent(data string) { event.Host, event.Ports) } else { - // TODO: add to mmsd.unhealthyTasks[] to remember host:port mappings - // for the moment when this task becomes live + // Remember the update event so as soon as the task becomes healthy, + // we can send a full AddTask() event to the handlers. + mmsd.taskEvents[event.TaskId] = event } case marathon.TaskKilling: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) @@ -417,6 +419,7 @@ func (mmsd *mmsdService) StatusUpdateEvent(data string) { mmsd.RemoveTask(event.AppId, event.TaskId, event.TaskStatus) case marathon.TaskFinished, marathon.TaskFailed, marathon.TaskKilled, marathon.TaskLost: log.Printf("App %v task %v on %v changed status. %v.\n", event.AppId, event.TaskId, event.Host, event.TaskStatus) + delete(mmsd.taskEvents, event.TaskId) if !mmsd.killingTasks[event.TaskId] { mmsd.RemoveTask(event.AppId, event.TaskId, event.TaskStatus) } else { @@ -432,7 +435,11 @@ func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { if err != nil { log.Printf("Failed to unmarshal health_status_changed_event. %v\n", err) } else if event.Alive { - // TODO mmsd.AddTask(event.AppId, event.TaskId) + if taskUpdateEvent, ok := mmsd.taskEvents[event.TaskId]; ok { + log.Printf("taskUpdateEvent: %+v", taskUpdateEvent) + // TODO: fee mmsd.apps[appIds] with tasks + // TODO mmsd.AddTask(event.AppId, event.TaskId) + } } else { mmsd.RemoveTask(event.AppId, event.TaskId, "TASK_RUNNING") } @@ -650,6 +657,7 @@ func main() { DNSBaseName: "mmsd.", DNSTTL: time.Second * 5, quitChannel: make(chan bool), + taskEvents: make(map[string]marathon.StatusUpdateEvent), killingTasks: make(map[string]bool), } From 67a38ac78cb8eee17b499e6dbdb9053866d67ba0 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Wed, 21 Sep 2016 12:50:32 +0200 Subject: [PATCH 34/39] The Marathon ID is stored in Name --- modules/dns.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/dns.go b/modules/dns.go index 70d119b..8e3dd17 100644 --- a/modules/dns.go +++ b/modules/dns.go @@ -45,6 +45,8 @@ type dbEntry struct { } func (module *DNSModule) Startup() { + log.Printf("DNS Server listen on %v:%v UDP", module.ServiceAddr, module.ServicePort) + log.Printf("DNS Server listen on %v:%v TCP", module.ServiceAddr, module.ServicePort) dns.HandleFunc(module.BaseName, module.dnsHandler) go func() { @@ -83,13 +85,15 @@ func (module *DNSModule) Apply(apps []*core.AppCluster) { module.dbMutex.Unlock() for _, app := range apps { - if err := module.update(app); err != nil { + err := module.update(app) + if err != nil { return } } } func (module *DNSModule) AddTask(task *core.AppBackend, app *core.AppCluster) { + log.Printf("DNS AddTask : %v, %v", task, app) module.update(app) } @@ -104,7 +108,7 @@ func (module *DNSModule) update(app *core.AppCluster) error { ipAddresses = append(ipAddresses, ip.IP) } - var reversed = module.makeDnsNameFromAppId(app.Id) + var reversed = module.makeDnsNameFromAppName(app.Name) var entry = &dbEntry{ ipAddresses: ipAddresses, app: app, @@ -118,6 +122,7 @@ func (module *DNSModule) update(app *core.AppCluster) error { } func (module *DNSModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { + log.Printf("DNS RemoveTask : %v, %v", task, app) module.update(app) } @@ -188,8 +193,8 @@ func (module *DNSModule) makeAllSRV(entry *dbEntry) []dns.RR { return result } -func (module *DNSModule) makeDnsNameFromAppId(appID string) string { - var parts = strings.Split(appID, "/")[1:] +func (module *DNSModule) makeDnsNameFromAppName(appName string) string { + var parts = strings.Split(appName, "/")[1:] var reversedParts []string for i := range parts { reversedParts = append(reversedParts, parts[len(parts)-i-1]) From 9e84035a2308fb530f4c5c9af9cacc759380abc6 Mon Sep 17 00:00:00 2001 From: Christian Parpart Date: Thu, 22 Sep 2016 14:22:29 +0200 Subject: [PATCH 35/39] wip wrt. deployment_info --- main.go | 30 ++++++++++++++++++++++++++++-- util/helper.go | 21 +++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index eb515f4..a9848e1 100644 --- a/main.go +++ b/main.go @@ -358,7 +358,7 @@ func (mmsd *mmsdService) setupEventBusListener() { bus.OnOpen = mmsd.OnMarathonConnected bus.OnError = mmsd.OnMarathonConnectionFailure - //bus.AddEventListener("deployment_info", mmsd.DeploymentStart) + bus.AddEventListener("deployment_info", mmsd.DeploymentStarted) //bus.AddEventListener("deployment_success", mmsd.DeploymentSuccess) //bus.AddEventListener("deployment_failed", mmsd.DeploymentFailed) bus.AddEventListener("status_update_event", mmsd.StatusUpdateEvent) @@ -376,6 +376,24 @@ func (mmsd *mmsdService) OnMarathonConnectionFailure(event, data string) { log.Printf("Marathon Event Stream Error. %v. %v\n", event, data) } +func (mmsd *mmsdService) DeploymentStarted(data string) { + var event marathon.DeploymentInfoEvent + err := json.Unmarshal([]byte(data), &event) + if err != nil { + log.Printf("Failed to unmarshal DeploymentInfoEvent. %v", err) + log.Printf("deployment_info: %v", data) + return + } + + log.Printf("app: %v", data) + // for _, action := range event.CurrentStep.Actions { + // log.Printf("action: %+v", action) + // app := util.FindMarathonAppById(event.Plan.Target.Apps, action.App) + // log.Printf("app: %v", util.ConvertToJsonString(app)) + // } + // log.Printf("apps => %+v", event.Plan.Target.Apps) +} + // StatusUpdateEvent is invoked by SSE when exactly this named event is fired. func (mmsd *mmsdService) StatusUpdateEvent(data string) { var event marathon.StatusUpdateEvent @@ -437,7 +455,15 @@ func (mmsd *mmsdService) HealthStatusChangedEvent(data string) { } else if event.Alive { if taskUpdateEvent, ok := mmsd.taskEvents[event.TaskId]; ok { log.Printf("taskUpdateEvent: %+v", taskUpdateEvent) - // TODO: fee mmsd.apps[appIds] with tasks + // for portIndex, port := range taskUpdateEvent.Ports { + // task := &core.AppBackend{ + // Id: event.TaskId, + // Host: taskUpdateEvent.Host, + // Port: port, + // State: string(taskUpdateEvent.TaskStatus), + // } + // } + // TODO: feed mmsd.apps[appIds] with tasks // TODO mmsd.AddTask(event.AppId, event.TaskId) } } else { diff --git a/util/helper.go b/util/helper.go index a4d8909..864c4e0 100644 --- a/util/helper.go +++ b/util/helper.go @@ -2,6 +2,7 @@ package util import ( "bytes" + "encoding/json" "errors" "fmt" "hash/fnv" @@ -304,3 +305,23 @@ func SortedStrStrKeys(m map[string]string) []string { } // }}} + +func ConvertToJsonString(obj interface{}) string { + out, err := json.MarshalIndent(&obj, "", " ") + if err == nil { + return string(out) + } else { + return "" + } +} + +func FindMarathonAppById(apps []marathon.App, id string) *marathon.App { + for i, mi := range apps { + if mi.Id == id { + return &apps[i] + } else { + log.Printf("FindMarathonAppById: skip %v", mi.Id) + } + } + return nil +} From a81b363c2faa6659233ae7031723e78b98c0e6db Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 24 Feb 2017 14:57:56 +0100 Subject: [PATCH 36/39] It was not return the fqdn for the request --- modules/dns.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/dns.go b/modules/dns.go index 8e3dd17..13de988 100644 --- a/modules/dns.go +++ b/modules/dns.go @@ -44,9 +44,9 @@ type dbEntry struct { app *core.AppCluster } +// Startup bring up a DNS server list UDP and TCP connections func (module *DNSModule) Startup() { - log.Printf("DNS Server listen on %v:%v UDP", module.ServiceAddr, module.ServicePort) - log.Printf("DNS Server listen on %v:%v TCP", module.ServiceAddr, module.ServicePort) + log.Printf("DNS Server base name: %s", module.BaseName) dns.HandleFunc(module.BaseName, module.dnsHandler) go func() { @@ -55,6 +55,7 @@ func (module *DNSModule) Startup() { Net: "udp", TsigSecret: nil, } + log.Printf("DNS Server listen on %v:%v UDP", module.ServiceAddr, module.ServicePort) err := module.udpServer.ListenAndServe() if err != nil { log.Fatal(err) @@ -67,6 +68,7 @@ func (module *DNSModule) Startup() { Net: "tcp", TsigSecret: nil, } + log.Printf("DNS Server listen on %v:%v TCP", module.ServiceAddr, module.ServicePort) err := module.tcpServer.ListenAndServe() if err != nil { log.Fatal(err) @@ -140,11 +142,11 @@ func (module *DNSModule) dnsHandler(w dns.ResponseWriter, req *dns.Msg) { if ok { switch req.Question[0].Qtype { case dns.TypeSRV: - m.Answer = module.makeAllSRV(entry) + m.Answer = module.makeAllSRV(req.Question[0].Name, entry) case dns.TypeA: - m.Answer = module.makeAllA(entry) + m.Answer = module.makeAllA(req.Question[0].Name, entry) if module.PushSRV { - m.Extra = module.makeAllSRV(entry) + m.Extra = module.makeAllSRV(req.Question[0].Name, entry) } } } @@ -152,14 +154,14 @@ func (module *DNSModule) dnsHandler(w dns.ResponseWriter, req *dns.Msg) { w.WriteMsg(m) } -func (module *DNSModule) makeAllA(entry *dbEntry) []dns.RR { +func (module *DNSModule) makeAllA(name string, entry *dbEntry) []dns.RR { var result []dns.RR for _, ip := range entry.ipAddresses { rr := &dns.A{ Hdr: dns.RR_Header{ Ttl: uint32(module.DNSTTL.Seconds()), - Name: module.BaseName, + Name: name, Class: dns.ClassINET, Rrtype: dns.TypeA, }, @@ -171,14 +173,14 @@ func (module *DNSModule) makeAllA(entry *dbEntry) []dns.RR { return result } -func (module *DNSModule) makeAllSRV(entry *dbEntry) []dns.RR { +func (module *DNSModule) makeAllSRV(name string, entry *dbEntry) []dns.RR { var result []dns.RR for _, task := range entry.app.Backends { rr := &dns.SRV{ Hdr: dns.RR_Header{ Ttl: uint32(module.DNSTTL.Seconds()), - Name: module.BaseName, + Name: name, Class: dns.ClassINET, Rrtype: dns.TypeSRV, }, From 441b0dbbd00f789e46b91c8a85fddaec381ea1fb Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Fri, 24 Feb 2017 18:33:26 +0100 Subject: [PATCH 37/39] Exported methods without a comment --- modules/dns.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/dns.go b/modules/dns.go index 13de988..ad484ad 100644 --- a/modules/dns.go +++ b/modules/dns.go @@ -26,6 +26,7 @@ import ( "github.com/miekg/dns" ) +// DNSModule provide that is pluginable in the EventListener type DNSModule struct { Verbose bool ServiceAddr net.IP @@ -76,11 +77,13 @@ func (module *DNSModule) Startup() { }() } +// Shutdown stop the DNS server func (module *DNSModule) Shutdown() { module.udpServer.Shutdown() module.tcpServer.Shutdown() } +// Apply bootstrap the data store from list of AppCluster func (module *DNSModule) Apply(apps []*core.AppCluster) { module.dbMutex.Lock() module.db = make(map[string]*dbEntry) @@ -94,6 +97,7 @@ func (module *DNSModule) Apply(apps []*core.AppCluster) { } } +// AddTask replace or add the AppCluster in the data store func (module *DNSModule) AddTask(task *core.AppBackend, app *core.AppCluster) { log.Printf("DNS AddTask : %v, %v", task, app) module.update(app) @@ -110,7 +114,7 @@ func (module *DNSModule) update(app *core.AppCluster) error { ipAddresses = append(ipAddresses, ip.IP) } - var reversed = module.makeDnsNameFromAppName(app.Name) + var reversed = module.makeDNSNameFromAppName(app.Name) var entry = &dbEntry{ ipAddresses: ipAddresses, app: app, @@ -123,6 +127,7 @@ func (module *DNSModule) update(app *core.AppCluster) error { return nil } +// RemoveTask replace or remove the AppCluster from data store func (module *DNSModule) RemoveTask(task *core.AppBackend, app *core.AppCluster) { log.Printf("DNS RemoveTask : %v, %v", task, app) module.update(app) @@ -195,7 +200,7 @@ func (module *DNSModule) makeAllSRV(name string, entry *dbEntry) []dns.RR { return result } -func (module *DNSModule) makeDnsNameFromAppName(appName string) string { +func (module *DNSModule) makeDNSNameFromAppName(appName string) string { var parts = strings.Split(appName, "/")[1:] var reversedParts []string for i := range parts { From 8921ded5e4dc070abd5a7d13ac7b2e1bcf7eefd2 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Thu, 9 Mar 2017 16:40:10 +0100 Subject: [PATCH 38/39] Remove old code --- dns_manager.go | 216 ------------------------------------------------- 1 file changed, 216 deletions(-) delete mode 100644 dns_manager.go diff --git a/dns_manager.go b/dns_manager.go deleted file mode 100644 index d122179..0000000 --- a/dns_manager.go +++ /dev/null @@ -1,216 +0,0 @@ -package main - -// DNS based service discovery -// --------------------------------------------------------------------------- -// -// Marathon: "/path/to/application" -// DNS-query: application.to.path.$basedomain (A, AAAA, TXT, SRV) -// DNS-reply: -// A => list of IPv4 addresses -// AAAA => list of IPv6 addresses -// SRV => ip:port array per task -// TXT => app labels - -// TODO: DNS forwarding -// TODO: DNS proxy cache (for speeding up) - -import ( - "fmt" - "log" - "net" - "strings" - "sync" - "time" - - "github.com/dawanda/go-mesos/marathon" - "github.com/miekg/dns" -) - -type DnsManager struct { - Verbose bool - ServiceAddr net.IP - ServicePort uint - BaseName string - DnsTTL time.Duration - PushSRV bool - udpServer *dns.Server - tcpServer *dns.Server - db map[string]*appEntry - dbMutex sync.Mutex -} - -type appEntry struct { - ipAddresses []net.IP - app *marathon.App -} - -func (manager *DnsManager) Startup() { - dns.HandleFunc(manager.BaseName, manager.dnsHandler) - - go func() { - manager.udpServer = &dns.Server{ - Addr: fmt.Sprintf("%v:%v", manager.ServiceAddr, manager.ServicePort), - Net: "udp", - TsigSecret: nil, - } - err := manager.udpServer.ListenAndServe() - if err != nil { - log.Fatal(err) - } - }() - - go func() { - manager.tcpServer = &dns.Server{ - Addr: fmt.Sprintf("%v:%v", manager.ServiceAddr, manager.ServicePort), - Net: "tcp", - TsigSecret: nil, - } - err := manager.tcpServer.ListenAndServe() - if err != nil { - log.Fatal(err) - } - }() -} - -func (manager *DnsManager) Shutdown() { - manager.udpServer.Shutdown() - manager.tcpServer.Shutdown() -} - -func (manager *DnsManager) Log(msg string) { - if manager.Verbose { - log.Printf("[dns]: %v\n", msg) - } -} - -func (manager *DnsManager) Apply(apps []*marathon.App, force bool) error { - manager.dbMutex.Lock() - manager.db = make(map[string]*appEntry) - manager.dbMutex.Unlock() - - for _, app := range apps { - if err := manager.updateApp(app); err != nil { - return err - } - } - - return nil -} - -func (manager *DnsManager) Update(app *marathon.App, taskID string) error { - return manager.updateApp(app) -} - -func (manager *DnsManager) updateApp(app *marathon.App) error { - var ipAddresses []net.IP - - for _, task := range app.Tasks { - ip, err := net.ResolveIPAddr("ip", task.Host) - if err != nil { - return err - } - ipAddresses = append(ipAddresses, ip.IP) - } - - var reversed = manager.makeDnsNameFromAppId(app.Id) - var entry = &appEntry{ - ipAddresses: ipAddresses, - app: app, - } - - manager.dbMutex.Lock() - manager.db[reversed] = entry - manager.dbMutex.Unlock() - - return nil -} - -func (manager *DnsManager) makeDnsNameFromAppId(appID string) string { - var parts = strings.Split(appID, "/")[1:] - var reversedParts []string - for i := range parts { - reversedParts = append(reversedParts, parts[len(parts)-i-1]) - } - var reversed = strings.Join(reversedParts, ".") - - return reversed -} - -func (manager *DnsManager) Remove(appID string, taskID string, app *marathon.App) error { - if app == nil { - manager.dbMutex.Lock() - delete(manager.db, manager.makeDnsNameFromAppId(appID)) - manager.dbMutex.Unlock() - return nil - } - return manager.updateApp(app) -} - -func (manager *DnsManager) dnsHandler(w dns.ResponseWriter, req *dns.Msg) { - m := new(dns.Msg) - m.SetReply(req) - - name := req.Question[0].Name - name = strings.TrimSuffix(name, "."+manager.BaseName) - - manager.dbMutex.Lock() - entry, ok := manager.db[name] - manager.dbMutex.Unlock() - - if ok { - switch req.Question[0].Qtype { - case dns.TypeSRV: - m.Answer = manager.makeAllSRV(entry) - case dns.TypeA: - m.Answer = manager.makeAllA(entry) - if manager.PushSRV { - m.Extra = manager.makeAllSRV(entry) - } - } - } - - w.WriteMsg(m) -} - -func (manager *DnsManager) makeAllA(entry *appEntry) []dns.RR { - var result []dns.RR - - for _, ip := range entry.ipAddresses { - rr := &dns.A{ - Hdr: dns.RR_Header{ - Ttl: uint32(manager.DnsTTL.Seconds()), - Name: manager.BaseName, - Class: dns.ClassINET, - Rrtype: dns.TypeA, - }, - A: ip.To4(), - } - result = append(result, rr) - } - - return result -} - -func (manager *DnsManager) makeAllSRV(entry *appEntry) []dns.RR { - var result []dns.RR - - for _, task := range entry.app.Tasks { - for _, port := range task.Ports { - rr := &dns.SRV{ - Hdr: dns.RR_Header{ - Ttl: uint32(manager.DnsTTL.Seconds()), - Name: manager.BaseName, - Class: dns.ClassINET, - Rrtype: dns.TypeSRV, - }, - Port: uint16(port), - Target: task.Host + ".", - Weight: 1, - Priority: 1, - } - result = append(result, rr) - } - } - - return result -} From 11bb7fae0900e7f5327c2c1a495ffcbecb940aa7 Mon Sep 17 00:00:00 2001 From: Martin Bormeister Date: Thu, 9 Mar 2017 16:40:28 +0100 Subject: [PATCH 39/39] Add logging --- modules/dns.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/dns.go b/modules/dns.go index ad484ad..64caff2 100644 --- a/modules/dns.go +++ b/modules/dns.go @@ -85,6 +85,7 @@ func (module *DNSModule) Shutdown() { // Apply bootstrap the data store from list of AppCluster func (module *DNSModule) Apply(apps []*core.AppCluster) { + log.Printf("DNS Apply : initialize %d apps", len(apps)) module.dbMutex.Lock() module.db = make(map[string]*dbEntry) module.dbMutex.Unlock()