Revamped a lot of backend stuff

Added a basic admin panel (/admin) available to owners only. This panel
can be used to shutdown, update and do some profiling of the running
processes.

Communication between backend components is now standardised through a
very basic service discovery system using redis.

The package bot/rest have now mostly been moved to common/internalapi and each yagpdb process
runs this even if it's not a bot process.

Service discovery is handled by common/service.go and can be used to
find all the other running processes and components.

Also cleaned up a lot of ugly parts.

admin/README.md

@@ -0,0 +1 @@
+The admin plugin is the plugin providing the internal admin interface for controlling and interacting with how the bot is running.

admin/assets/bot_admin_panel.html

@@ -0,0 +1,100 @@
+{{define "bot_admin_panel"}}
+
+{{template "cp_head" .}}
+<header class="page-header">
+    <h2>YAGPDB internal admin panel</h2>
+</header>
+
+{{template "cp_alerts" .}}
+
+<div class="row">
+    {{range .ServiceHosts}}
+    <div class="col-lg-4">
+        <section class="card card-featured card-featured-success mb-4">
+            <header class="card-header">
+                <h2 class="card-title">{{.Host}} - {{.PID}}</h2>
+            </header>
+            <div class="card-body host-status">
+                <div class="row">
+                    <div class="col-lg-12">
+                        {{$sh := .}}
+                        <p>V:<code>{{.Version}}</code> - <code>{{.InternalAPIAddress}}</code></p>
+                        {{range .Services}}
+                        <h4>{{.Type}}</h4>
+                        <ul>
+                            <li>{{.Type}} - {{.Name}}</li>
+                            {{if .Details}}<li>{{.Details}}</li>{{end}}
+                            {{if .BotDetails}}{{if .BotDetails.OrchestratorMode}}
+                            <li>NodeID: {{.BotDetails.NodeID}}</li>
+                            <li>Shards: {{.BotDetails.RunningShards}}</li>
+                            <li>Total Shards: {{.BotDetails.TotalShards}}</li>
+                            {{end}}{{end}}
+                        </ul>
+                        {{if eq .Type "orchestrator"}}
+                        <form method="POST" action="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/updateversion">
+                            <button type="submit" class="btn btn-primary" value="Update version">Update version</button>
+                        </form>
+                        <form method="POST" action="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/migratenodes">
+                            <button type="submit" class="btn btn-primary" value="Update version">Migrate nodes</button>
+                        </form>
+                        <a href="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/deployedversion" class="btn btn-primary"
+                            target="_blank">New node version</a>
+                        {{end}}
+                        {{end}}
+
+                        <a href="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/goroutines?debug=1" class="btn btn-primary"
+                            target="_blank">Goroutines</a>
+                        <a href="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/goroutines?debug=2" class="btn btn-primary"
+                            target="_blank">Goroutines Full</a>
+                        <a href="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/heap?debug=1" class="btn btn-primary"
+                            target="_blank">Heap</a>
+                        <a href="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/allocs?debug=1" class="btn btn-primary" tar
+                            get="_blank">Allocs</a>
+                        <form method="POST" action="/admin/host/{{$sh.Host}}/pid/{{$sh.PID}}/shutdown">
+                            <button type="submit" class="btn btn-danger" value="Update version">Shutdown</button>
+                        </form>
+                    </div>
+                </div>
+            </div>
+        </section>
+    </div>
+    {{end}}
+</div>
+
+<!-- Modal -->
+<div class="modal fade" id="remote-modal" tabindex="-1" role="dialog" aria-hidden="true">
+    <div class="modal-dialog modal-lg" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title" id="remote-modal-title">Modal title</h5>
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <div class="modal-body" id="remote-modal-body">
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+
+    function openRemoteModal(url) {
+        $("#remote-modal").modal('show')
+
+        $("#remote-modal-title").text(url)
+        $("#remote-modal-body").text("Loading...")
+
+        createRequest("GET", url + "?partial=1", null, function () {
+            $("#remote-modal-body").html(this.responseText);
+        })
+    }
+
+</script>
+
+{{template "cp_footer" .}}
+
+{{end}}

admin/plugin_admin.go

@@ -0,0 +1,22 @@
+package admin
+
+import (
+	"github.com/jonas747/yagpdb/common"
+)
+
+var logger = common.GetPluginLogger(&Plugin{})
+
+type Plugin struct {
+}
+
+func (p *Plugin) PluginInfo() *common.PluginInfo {
+	return &common.PluginInfo{
+		Name:     "Admin",
+		SysName:  "admin",
+		Category: common.PluginCategoryCore,
+	}
+}
+
+func RegisterPlugin() {
+	common.RegisterPlugin(&Plugin{})
+}

admin/web.go

@@ -0,0 +1,197 @@
+package admin
+
+import (
+	"github.com/jonas747/dshardorchestrator/orchestrator/rest"
+	"github.com/jonas747/yagpdb/common/internalapi"
+	"io"
+	"net/http"
+	"strconv"
+
+	"emperror.dev/errors"
+	"github.com/jonas747/yagpdb/common"
+	"github.com/jonas747/yagpdb/web"
+	"goji.io"
+	"goji.io/pat"
+)
+
+// InitWeb implements web.Plugin
+func (p *Plugin) InitWeb() {
+	web.LoadHTMLTemplate("../../admin/assets/bot_admin_panel.html", "templates/plugins/bot_admin_panel.html")
+
+	mux := goji.SubMux()
+	web.RootMux.Handle(pat.New("/admin/*"), mux)
+	web.RootMux.Handle(pat.New("/admin"), mux)
+
+	mux.Use(web.RequireSessionMiddleware)
+	mux.Use(web.RequireBotOwnerMW)
+
+	panelHandler := web.ControllerHandler(p.handleGetPanel, "bot_admin_panel")
+
+	mux.Handle(pat.Get(""), panelHandler)
+	mux.Handle(pat.Get("/"), panelHandler)
+
+	// Debug routes
+	mux.Handle(pat.Get("/host/:host/pid/:pid/goroutines"), p.ProxyGetInternalAPI("/debug/pprof/goroutine"))
+	mux.Handle(pat.Get("/host/:host/pid/:pid/trace"), p.ProxyGetInternalAPI("/debug/pprof/trace"))
+	mux.Handle(pat.Get("/host/:host/pid/:pid/profile"), p.ProxyGetInternalAPI("/debug/pprof/profile"))
+	mux.Handle(pat.Get("/host/:host/pid/:pid/heap"), p.ProxyGetInternalAPI("/debug/pprof/heap"))
+	mux.Handle(pat.Get("/host/:host/pid/:pid/allocs"), p.ProxyGetInternalAPI("/debug/pprof/allocs"))
+
+	// Control routes
+	mux.Handle(pat.Post("/host/:host/pid/:pid/shutdown"), web.ControllerPostHandler(p.handleShutdown, panelHandler, nil, ""))
+
+	// Orhcestrator controls
+	mux.Handle(pat.Post("/host/:host/pid/:pid/updateversion"), web.ControllerPostHandler(p.handleUpgrade, panelHandler, nil, ""))
+	mux.Handle(pat.Post("/host/:host/pid/:pid/migratenodes"), web.ControllerPostHandler(p.handleMigrateNodes, panelHandler, nil, ""))
+	mux.Handle(pat.Get("/host/:host/pid/:pid/deployedversion"), http.HandlerFunc(p.handleLaunchNodeVersion))
+}
+
+func (p *Plugin) handleGetPanel(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) {
+	_, tmpl := web.GetBaseCPContextData(r.Context())
+
+	hosts, err := common.ServicePoller.GetActiveServiceHosts()
+	if err != nil {
+		return tmpl, errors.WithStackIf(err)
+	}
+
+	tmpl["ServiceHosts"] = hosts
+
+	return tmpl, nil
+}
+
+func (p *Plugin) ProxyGetInternalAPI(path string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		debug := r.URL.Query().Get("debug")
+		debugStr := ""
+		if debug != "" {
+			debugStr = "?debug=" + debug
+		}
+
+		sh, err := findServicehost(r)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("Error querying service hosts: " + err.Error()))
+			return
+		}
+
+		resp, err := http.Get("http://" + sh.InternalAPIAddress + path + debugStr)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte("Error querying internal api: " + err.Error()))
+			return
+		}
+
+		io.Copy(w, resp.Body)
+	})
+}
+
+func (p *Plugin) handleShutdown(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) {
+	_, tmpl := web.GetBaseCPContextData(r.Context())
+
+	sh, err := findServicehost(r)
+	if err != nil {
+		return tmpl, err
+	}
+
+	var resp string
+	err = internalapi.PostWithAddress(sh.InternalAPIAddress, "shutdown", nil, &resp)
+	if err != nil {
+		return tmpl, err
+	}
+
+	tmpl = tmpl.AddAlerts(web.SucessAlert(resp))
+	return tmpl, nil
+}
+
+func (p *Plugin) handleUpgrade(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) {
+	_, tmpl := web.GetBaseCPContextData(r.Context())
+
+	client, err := createOrhcestatorRESTClient(r)
+	if err != nil {
+		return tmpl, err
+	}
+
+	logger.Println("Upgrading version...")
+
+	newVer, err := client.PullNewVersion()
+	if err != nil {
+		tmpl.AddAlerts(web.ErrorAlert(err.Error()))
+		return tmpl, err
+	}
+
+	tmpl = tmpl.AddAlerts(web.SucessAlert("Upgraded to ", newVer))
+	return tmpl, nil
+}
+
+func (p *Plugin) handleMigrateNodes(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) {
+	_, tmpl := web.GetBaseCPContextData(r.Context())
+
+	client, err := createOrhcestatorRESTClient(r)
+	if err != nil {
+		return tmpl, err
+	}
+
+	logger.Println("Upgrading version...")
+
+	response, err := client.MigrateAllNodesToNewNodes()
+	if err != nil {
+		tmpl.AddAlerts(web.ErrorAlert(err.Error()))
+		return tmpl, err
+	}
+
+	tmpl = tmpl.AddAlerts(web.SucessAlert(response))
+	return tmpl, nil
+}
+
+func (p *Plugin) handleLaunchNodeVersion(w http.ResponseWriter, r *http.Request) {
+	logger.Println("ahahha")
+
+	client, err := createOrhcestatorRESTClient(r)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Error querying service hosts: " + err.Error()))
+		return
+	}
+
+	ver, err := client.GetDeployedVersion()
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Error getting deployed version: " + err.Error()))
+		return
+	}
+
+	w.Write([]byte(ver))
+}
+
+func createOrhcestatorRESTClient(r *http.Request) (*rest.Client, error) {
+	sh, err := findServicehost(r)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range sh.Services {
+		if v.Type == common.ServiceTypeOrchestator {
+			return rest.NewClient("http://" + sh.InternalAPIAddress), nil
+		}
+	}
+
+	return nil, common.ErrNotFound
+}
+
+func findServicehost(r *http.Request) (*common.ServiceHost, error) {
+	host := pat.Param(r, "host")
+	pid := pat.Param(r, "pid")
+
+	serviceHosts, err := common.ServicePoller.GetActiveServiceHosts()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, v := range serviceHosts {
+		if v.Host == host && pid == strconv.Itoa(v.PID) {
+			return v, nil
+		}
+	}
+
+	return nil, common.ErrNotFound
+}

autorole/plugin_botrest.go

@@ -7,16 +7,16 @@ import (
 	"emperror.dev/errors"
 	"github.com/jonas747/retryableredis"
 	"github.com/jonas747/yagpdb/bot"
-	"github.com/jonas747/yagpdb/bot/botrest"
 	"github.com/jonas747/yagpdb/common"
+	"github.com/jonas747/yagpdb/common/internalapi"
 	"goji.io"
 	"goji.io/pat"
 )
 
-var _ botrest.BotRestPlugin = (*Plugin)(nil)
+var _ internalapi.InternalAPIPlugin = (*Plugin)(nil)
 var ErrAlreadyProcessingFullGuild = errors.New("Already processing users on this guild")
 
-func (p *Plugin) InitBotRestServer(mux *goji.Mux) {
+func (p *Plugin) InitInternalAPIRoutes(mux *goji.Mux) {
 	mux.Handle(pat.Post("/:guild/autorole/fullscan"), http.HandlerFunc(botRestHandleScanFullServer))
 }
 
@@ -25,15 +25,15 @@ func botRestHandleScanFullServer(w http.ResponseWriter, r *http.Request) {
 	parsedGID, _ := strconv.ParseInt(guildID, 10, 64)
 
 	if parsedGID == 0 {
-		botrest.ServerError(w, r, errors.New("unknown server"))
+		internalapi.ServerError(w, r, errors.New("unknown server"))
 		return
 	}
 
 	logger.WithField("guild", parsedGID).Info("autorole doing a full scan")
 	session := bot.ShardManager.SessionForGuild(parsedGID)
 	session.GatewayManager.RequestGuildMembers(parsedGID, "", 0)
 
-	botrest.ServeJson(w, r, "ok")
+	internalapi.ServeJson(w, r, "ok")
 }
 
 func botRestPostFullScan(guildID int64) error {
@@ -47,6 +47,6 @@ func botRestPostFullScan(guildID int64) error {
 		return ErrAlreadyProcessingFullGuild
 	}
 
-	err = botrest.Post(bot.GuildShardID(guildID), strconv.FormatInt(guildID, 10)+"/autorole/fullscan", nil, nil)
+	err = internalapi.PostWithGuild(guildID, strconv.FormatInt(guildID, 10)+"/autorole/fullscan", nil, nil)
 	return err
 }