From 9e8d27932fc3ee17e84845e75a275ad15f47a4cc Mon Sep 17 00:00:00 2001 From: jonas747 Date: Thu, 21 Nov 2019 10:22:37 +0100 Subject: [PATCH] Revamped a lot of backend stuff Added a basic admin panel (/admin) available to owners only. This panel can be used to shutdown, update and do some profiling of the running processes. Communication between backend components is now standardised through a very basic service discovery system using redis. The package bot/rest have now mostly been moved to common/internalapi and each yagpdb process runs this even if it's not a bot process. Service discovery is handled by common/service.go and can be used to find all the other running processes and components. Also cleaned up a lot of ugly parts. --- admin/README.md | 1 + admin/assets/bot_admin_panel.html | 100 ++++++ admin/plugin_admin.go | 22 ++ admin/web.go | 197 ++++++++++++ autorole/plugin_botrest.go | 12 +- bot/bot.go | 44 ++- bot/botrest/botrest.go | 12 - bot/botrest/client.go | 158 +--------- bot/botrest/server.go | 218 +++---------- bot/nodeimpl.go | 4 +- bot/util.go | 62 +--- cah/plugin_bot.go | 2 +- cmd/shardorchestrator/main.go | 49 ++- cmd/yagpdb/main.go | 19 +- cmd/yagpdb/templates/status.html | 173 +++++------ common/backgroundworkers/backgroundworkers.go | 2 + common/common.go | 29 ++ common/internalapi/clientutil.go | 113 +++++++ common/internalapi/server.go | 156 ++++++++++ common/plugins.go | 15 + common/service.go | 292 ++++++++++++++++++ feeds/plugin.go | 3 + go.mod | 7 +- go.sum | 32 ++ serverstats/schema.go | 1 - serverstats/serverstats.go | 4 +- stdcommands/currentshard/currentshard.go | 4 +- web/handlers_general.go | 5 - web/middleware.go | 16 +- web/web.go | 8 +- 30 files changed, 1246 insertions(+), 514 deletions(-) create mode 100644 admin/README.md create mode 100644 admin/assets/bot_admin_panel.html create mode 100644 admin/plugin_admin.go create mode 100644 admin/web.go create mode 100644 common/internalapi/clientutil.go create mode 100644 common/internalapi/server.go create mode 100644 common/service.go diff --git a/admin/README.md b/admin/README.md new file mode 100644 index 0000000000..5fbd2621ed --- /dev/null +++ b/admin/README.md @@ -0,0 +1 @@ +The admin plugin is the plugin providing the internal admin interface for controlling and interacting with how the bot is running. \ No newline at end of file diff --git a/admin/assets/bot_admin_panel.html b/admin/assets/bot_admin_panel.html new file mode 100644 index 0000000000..debce51ed5 --- /dev/null +++ b/admin/assets/bot_admin_panel.html @@ -0,0 +1,100 @@ +{{define "bot_admin_panel"}} + +{{template "cp_head" .}} +
+

YAGPDB internal admin panel

+
+ +{{template "cp_alerts" .}} + +
+ {{range .ServiceHosts}} +
+
+
+

{{.Host}} - {{.PID}}

+
+
+
+
+ {{$sh := .}} +

V:{{.Version}} - {{.InternalAPIAddress}}

+ {{range .Services}} +

{{.Type}}

+
    +
  • {{.Type}} - {{.Name}}
    • + {{if .Details}}
  • {{.Details}}
    • {{end}} + {{if .BotDetails}}{{if .BotDetails.OrchestratorMode}} +
  • NodeID: {{.BotDetails.NodeID}}
    • +
  • Shards: {{.BotDetails.RunningShards}}
    • +
  • Total Shards: {{.BotDetails.TotalShards}}
    • + {{end}}{{end}} +
+ {{if eq .Type "orchestrator"}} +
+ +
+
+ +
+ New node version + {{end}} + {{end}} + + Goroutines + Goroutines Full + Heap + Allocs +
+ +
+
+
+
+
+
+ {{end}} +
+ + + + + + +{{template "cp_footer" .}} + +{{end}} \ No newline at end of file diff --git a/admin/plugin_admin.go b/admin/plugin_admin.go new file mode 100644 index 0000000000..d94bc903ee --- /dev/null +++ b/admin/plugin_admin.go @@ -0,0 +1,22 @@ +package admin + +import ( + "github.com/jonas747/yagpdb/common" +) + +var logger = common.GetPluginLogger(&Plugin{}) + +type Plugin struct { +} + +func (p *Plugin) PluginInfo() *common.PluginInfo { + return &common.PluginInfo{ + Name: "Admin", + SysName: "admin", + Category: common.PluginCategoryCore, + } +} + +func RegisterPlugin() { + common.RegisterPlugin(&Plugin{}) +} diff --git a/admin/web.go b/admin/web.go new file mode 100644 index 0000000000..bf4bc35c6e --- /dev/null +++ b/admin/web.go @@ -0,0 +1,197 @@ +package admin + +import ( + "github.com/jonas747/dshardorchestrator/orchestrator/rest" + "github.com/jonas747/yagpdb/common/internalapi" + "io" + "net/http" + "strconv" + + "emperror.dev/errors" + "github.com/jonas747/yagpdb/common" + "github.com/jonas747/yagpdb/web" + "goji.io" + "goji.io/pat" +) + +// InitWeb implements web.Plugin +func (p *Plugin) InitWeb() { + web.LoadHTMLTemplate("../../admin/assets/bot_admin_panel.html", "templates/plugins/bot_admin_panel.html") + + mux := goji.SubMux() + web.RootMux.Handle(pat.New("/admin/*"), mux) + web.RootMux.Handle(pat.New("/admin"), mux) + + mux.Use(web.RequireSessionMiddleware) + mux.Use(web.RequireBotOwnerMW) + + panelHandler := web.ControllerHandler(p.handleGetPanel, "bot_admin_panel") + + mux.Handle(pat.Get(""), panelHandler) + mux.Handle(pat.Get("/"), panelHandler) + + // Debug routes + mux.Handle(pat.Get("/host/:host/pid/:pid/goroutines"), p.ProxyGetInternalAPI("/debug/pprof/goroutine")) + mux.Handle(pat.Get("/host/:host/pid/:pid/trace"), p.ProxyGetInternalAPI("/debug/pprof/trace")) + mux.Handle(pat.Get("/host/:host/pid/:pid/profile"), p.ProxyGetInternalAPI("/debug/pprof/profile")) + mux.Handle(pat.Get("/host/:host/pid/:pid/heap"), p.ProxyGetInternalAPI("/debug/pprof/heap")) + mux.Handle(pat.Get("/host/:host/pid/:pid/allocs"), p.ProxyGetInternalAPI("/debug/pprof/allocs")) + + // Control routes + mux.Handle(pat.Post("/host/:host/pid/:pid/shutdown"), web.ControllerPostHandler(p.handleShutdown, panelHandler, nil, "")) + + // Orhcestrator controls + mux.Handle(pat.Post("/host/:host/pid/:pid/updateversion"), web.ControllerPostHandler(p.handleUpgrade, panelHandler, nil, "")) + mux.Handle(pat.Post("/host/:host/pid/:pid/migratenodes"), web.ControllerPostHandler(p.handleMigrateNodes, panelHandler, nil, "")) + mux.Handle(pat.Get("/host/:host/pid/:pid/deployedversion"), http.HandlerFunc(p.handleLaunchNodeVersion)) +} + +func (p *Plugin) handleGetPanel(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) { + _, tmpl := web.GetBaseCPContextData(r.Context()) + + hosts, err := common.ServicePoller.GetActiveServiceHosts() + if err != nil { + return tmpl, errors.WithStackIf(err) + } + + tmpl["ServiceHosts"] = hosts + + return tmpl, nil +} + +func (p *Plugin) ProxyGetInternalAPI(path string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + debug := r.URL.Query().Get("debug") + debugStr := "" + if debug != "" { + debugStr = "?debug=" + debug + } + + sh, err := findServicehost(r) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Error querying service hosts: " + err.Error())) + return + } + + resp, err := http.Get("http://" + sh.InternalAPIAddress + path + debugStr) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Error querying internal api: " + err.Error())) + return + } + + io.Copy(w, resp.Body) + }) +} + +func (p *Plugin) handleShutdown(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) { + _, tmpl := web.GetBaseCPContextData(r.Context()) + + sh, err := findServicehost(r) + if err != nil { + return tmpl, err + } + + var resp string + err = internalapi.PostWithAddress(sh.InternalAPIAddress, "shutdown", nil, &resp) + if err != nil { + return tmpl, err + } + + tmpl = tmpl.AddAlerts(web.SucessAlert(resp)) + return tmpl, nil +} + +func (p *Plugin) handleUpgrade(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) { + _, tmpl := web.GetBaseCPContextData(r.Context()) + + client, err := createOrhcestatorRESTClient(r) + if err != nil { + return tmpl, err + } + + logger.Println("Upgrading version...") + + newVer, err := client.PullNewVersion() + if err != nil { + tmpl.AddAlerts(web.ErrorAlert(err.Error())) + return tmpl, err + } + + tmpl = tmpl.AddAlerts(web.SucessAlert("Upgraded to ", newVer)) + return tmpl, nil +} + +func (p *Plugin) handleMigrateNodes(w http.ResponseWriter, r *http.Request) (web.TemplateData, error) { + _, tmpl := web.GetBaseCPContextData(r.Context()) + + client, err := createOrhcestatorRESTClient(r) + if err != nil { + return tmpl, err + } + + logger.Println("Upgrading version...") + + response, err := client.MigrateAllNodesToNewNodes() + if err != nil { + tmpl.AddAlerts(web.ErrorAlert(err.Error())) + return tmpl, err + } + + tmpl = tmpl.AddAlerts(web.SucessAlert(response)) + return tmpl, nil +} + +func (p *Plugin) handleLaunchNodeVersion(w http.ResponseWriter, r *http.Request) { + logger.Println("ahahha") + + client, err := createOrhcestatorRESTClient(r) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Error querying service hosts: " + err.Error())) + return + } + + ver, err := client.GetDeployedVersion() + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Error getting deployed version: " + err.Error())) + return + } + + w.Write([]byte(ver)) +} + +func createOrhcestatorRESTClient(r *http.Request) (*rest.Client, error) { + sh, err := findServicehost(r) + if err != nil { + return nil, err + } + + for _, v := range sh.Services { + if v.Type == common.ServiceTypeOrchestator { + return rest.NewClient("http://" + sh.InternalAPIAddress), nil + } + } + + return nil, common.ErrNotFound +} + +func findServicehost(r *http.Request) (*common.ServiceHost, error) { + host := pat.Param(r, "host") + pid := pat.Param(r, "pid") + + serviceHosts, err := common.ServicePoller.GetActiveServiceHosts() + if err != nil { + return nil, err + } + + for _, v := range serviceHosts { + if v.Host == host && pid == strconv.Itoa(v.PID) { + return v, nil + } + } + + return nil, common.ErrNotFound +} diff --git a/autorole/plugin_botrest.go b/autorole/plugin_botrest.go index c0ba231a91..be2c8d033d 100644 --- a/autorole/plugin_botrest.go +++ b/autorole/plugin_botrest.go @@ -7,16 +7,16 @@ import ( "emperror.dev/errors" "github.com/jonas747/retryableredis" "github.com/jonas747/yagpdb/bot" - "github.com/jonas747/yagpdb/bot/botrest" "github.com/jonas747/yagpdb/common" + "github.com/jonas747/yagpdb/common/internalapi" "goji.io" "goji.io/pat" ) -var _ botrest.BotRestPlugin = (*Plugin)(nil) +var _ internalapi.InternalAPIPlugin = (*Plugin)(nil) var ErrAlreadyProcessingFullGuild = errors.New("Already processing users on this guild") -func (p *Plugin) InitBotRestServer(mux *goji.Mux) { +func (p *Plugin) InitInternalAPIRoutes(mux *goji.Mux) { mux.Handle(pat.Post("/:guild/autorole/fullscan"), http.HandlerFunc(botRestHandleScanFullServer)) } @@ -25,7 +25,7 @@ func botRestHandleScanFullServer(w http.ResponseWriter, r *http.Request) { parsedGID, _ := strconv.ParseInt(guildID, 10, 64) if parsedGID == 0 { - botrest.ServerError(w, r, errors.New("unknown server")) + internalapi.ServerError(w, r, errors.New("unknown server")) return } @@ -33,7 +33,7 @@ func botRestHandleScanFullServer(w http.ResponseWriter, r *http.Request) { session := bot.ShardManager.SessionForGuild(parsedGID) session.GatewayManager.RequestGuildMembers(parsedGID, "", 0) - botrest.ServeJson(w, r, "ok") + internalapi.ServeJson(w, r, "ok") } func botRestPostFullScan(guildID int64) error { @@ -47,6 +47,6 @@ func botRestPostFullScan(guildID int64) error { return ErrAlreadyProcessingFullGuild } - err = botrest.Post(bot.GuildShardID(guildID), strconv.FormatInt(guildID, 10)+"/autorole/fullscan", nil, nil) + err = internalapi.PostWithGuild(guildID, strconv.FormatInt(guildID, 10)+"/autorole/fullscan", nil, nil) return err } diff --git a/bot/bot.go b/bot/bot.go index 4d572e842b..9064c1e9bc 100644 --- a/bot/bot.go +++ b/bot/bot.go @@ -51,6 +51,7 @@ var ( processShardsLock sync.RWMutex ) +// Run intializes and starts the discord bot component of yagpdb func Run(nodeID string) { setup() @@ -63,7 +64,7 @@ func Run(nodeID string) { logger.Infof("Set to use orchestrator at address: %s", orcheStratorAddress) } else { logger.Info("Running standalone without any orchestrator") - SetupStandalone() + setupStandalone() } go MemberFetcher.Run() @@ -76,7 +77,7 @@ func Run(nodeID string) { NodeConn.Run() } else { go ShardManager.Start() - InitPlugins() + botReady() } } @@ -92,7 +93,7 @@ func setup() { common.BotSession.AddHandler(eventsystem.HandleEvent) } -func SetupStandalone() { +func setupStandalone() { shardCount, err := ShardManager.GetRecommendedCount() if err != nil { panic("Failed getting shard count: " + err.Error()) @@ -114,7 +115,8 @@ func SetupStandalone() { } } -func InitPlugins() { +// called when the bot is ready and the shards have started connecting +func botReady() { pubsub.AddHandler("bot_status_changed", func(evt *pubsub.Event) { updateAllShardStatuses() }, nil) @@ -122,6 +124,14 @@ func InitPlugins() { pubsub.AddHandler("global_ratelimit", handleGlobalRatelimtPusub, GlobalRatelimitTriggeredEventData{}) pubsub.AddHandler("bot_core_evict_gs_cache", handleEvictCachePubsub, "") + serviceDetails := "Not using orchestrator" + if UsingOrchestrator { + serviceDetails = "Using orchestrator, NodeID: " + common.NodeID + } + + // register us with the service discovery + common.ServiceTracker.RegisterService(common.ServiceTypeBot, "Bot", serviceDetails, botServiceDetailsF) + // Initialize all plugins for _, plugin := range common.Plugins { if initBot, ok := plugin.(BotInitHandler); ok { @@ -299,3 +309,29 @@ func setupShardManager() { // Only handler ShardManager.AddHandler(eventsystem.HandleEvent) } + +func botServiceDetailsF() (details *common.BotServiceDetails, err error) { + if !UsingOrchestrator { + totalShards := ShardManager.GetNumShards() + shards := make([]int, totalShards) + for i := 0; i < totalShards; i++ { + shards[i] = i + } + + return &common.BotServiceDetails{ + OrchestratorMode: false, + TotalShards: totalShards, + RunningShards: shards, + }, nil + } + + totalShards := getTotalShards() + running := GetProcessShards() + + return &common.BotServiceDetails{ + TotalShards: int(totalShards), + RunningShards: running, + NodeID: common.NodeID, + OrchestratorMode: true, + }, nil +} diff --git a/bot/botrest/botrest.go b/bot/botrest/botrest.go index d37134213b..c8b3103e81 100644 --- a/bot/botrest/botrest.go +++ b/bot/botrest/botrest.go @@ -1,13 +1 @@ package botrest - -import ( - "strconv" -) - -func RedisKeyShardAddressMapping(shardID int) string { - return "botrest_shard_mapping:" + strconv.Itoa(shardID) -} - -func RedisKeyNodeAddressMapping(nodeID string) string { - return "botrest_node_mapping:" + nodeID -} diff --git a/bot/botrest/client.go b/bot/botrest/client.go index a67d1d3e5e..7f095695dd 100644 --- a/bot/botrest/client.go +++ b/bot/botrest/client.go @@ -1,124 +1,31 @@ package botrest import ( - "bytes" - "encoding/json" "fmt" - "net/http" "net/url" "strconv" - "sync" "time" - "emperror.dev/errors" "github.com/jonas747/discordgo" "github.com/jonas747/retryableredis" - "github.com/jonas747/yagpdb/bot" "github.com/jonas747/yagpdb/common" + "github.com/jonas747/yagpdb/common/internalapi" ) -var ( - ErrServerError = errors.New("botrest server is having issues") - ErrCantFindServer = errors.New("can't find botrest server for provided shard") -) - -var ( - clientLogger = common.GetFixedPrefixLogger("botrest_client") -) - -func GetServerAddrForGuild(guildID int64) string { - shard := bot.GuildShardID(guildID) - return GetServerAddrForShard(shard) -} - -func GetServerAddrForShard(shard int) string { - resp := "" - err := common.RedisPool.Do(retryableredis.Cmd(&resp, "GET", RedisKeyShardAddressMapping(shard))) - if err != nil { - clientLogger.WithError(err).Error("failed retrieving shard server addr") - } - - return resp -} - -func Get(shard int, url string, dest interface{}) error { - serverAddr := GetServerAddrForShard(shard) - if serverAddr == "" { - return ErrCantFindServer - } - - return GetWithAddress(serverAddr, url, dest) -} - -func GetWithAddress(addr string, url string, dest interface{}) error { - resp, err := http.Get("http://" + addr + "/" + url) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - var errDest string - err := json.NewDecoder(resp.Body).Decode(&errDest) - if err != nil { - return ErrServerError - } - - return errors.New(errDest) - } - - return errors.WithMessage(json.NewDecoder(resp.Body).Decode(dest), "json.Decode") -} - -func Post(shard int, url string, bodyData interface{}, dest interface{}) error { - serverAddr := GetServerAddrForShard(shard) - if serverAddr == "" { - return ErrCantFindServer - } - - var bodyBuf bytes.Buffer - if bodyData != nil { - encoder := json.NewEncoder(&bodyBuf) - err := encoder.Encode(bodyData) - if err != nil { - return err - } - } - resp, err := http.Post("http://"+serverAddr+"/"+url, "application/json", &bodyBuf) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - var errDest string - err := json.NewDecoder(resp.Body).Decode(&errDest) - if err != nil { - return ErrServerError - } - - return errors.New(errDest) - } - - if dest == nil { - return nil - } - - return errors.WithMessage(json.NewDecoder(resp.Body).Decode(dest), "json.Decode") -} +var clientLogger = common.GetFixedPrefixLogger("botrest_client") func GetGuild(guildID int64) (g *discordgo.Guild, err error) { - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/guild", &g) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/guild", &g) return } func GetBotMember(guildID int64) (m *discordgo.Member, err error) { - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/botmember", &m) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/botmember", &m) return } func GetOnlineCount(guildID int64) (c int64, err error) { - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/onlinecount", &c) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/onlinecount", &c) return } @@ -131,7 +38,7 @@ func GetMembers(guildID int64, members ...int64) (m []*discordgo.Member, err err query := url.Values{"users": stringed} encoded := query.Encode() - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/members?"+encoded, &m) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/members?"+encoded, &m) return } @@ -146,7 +53,7 @@ func GetMemberColors(guildID int64, members ...int64) (m map[string]int, err err query := url.Values{"users": stringed} encoded := query.Encode() - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/membercolors?"+encoded, &m) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/membercolors?"+encoded, &m) return } @@ -165,7 +72,7 @@ func GetMemberMultiGuild(userID int64, guilds ...int64) (members []*discordgo.Me } func GetChannelPermissions(guildID, channelID int64) (perms int64, err error) { - err = Get(bot.GuildShardID(guildID), discordgo.StrID(guildID)+"/channelperms/"+discordgo.StrID(channelID), &perms) + err = internalapi.GetWithGuild(guildID, discordgo.StrID(guildID)+"/channelperms/"+discordgo.StrID(channelID), &perms) return } @@ -197,7 +104,7 @@ func GetNodeStatuses() (st *NodeStatusesResponse, err error) { } var status *NodeStatus - err = Get(0, "node_status", &status) + err = internalapi.GetWithShard(0, "node_status", &status) if err != nil { return nil, err } @@ -215,7 +122,7 @@ func getNodeStatusesClustered() (st *NodeStatusesResponse, err error) { return nil, err } - totalShards := bot.GetTotalShards() + totalShards, _ := common.ServicePoller.GetShardCount() st = &NodeStatusesResponse{ TotalShards: int(totalShards), } @@ -264,8 +171,7 @@ OUTER: func getNodeStatus(nodeID string, retCh chan interface{}) { // retrieve the REST address for this node - var addr string - err := common.RedisPool.Do(retryableredis.Cmd(&addr, "GET", RedisKeyNodeAddressMapping(nodeID))) + addr, err := common.ServicePoller.GetNodeAddress(nodeID) if err != nil { clientLogger.WithError(err).Error("failed retrieving rest address for bot for node id: ", nodeID) retCh <- err @@ -273,7 +179,7 @@ func getNodeStatus(nodeID string, retCh chan interface{}) { } var status *NodeStatus - err = GetWithAddress(addr, "node_status", &status) + err = internalapi.GetWithAddress(addr, "node_status", &status) if err != nil { clientLogger.WithError(err).Error("failed retrieving shard status for node ", nodeID) retCh <- err @@ -289,7 +195,7 @@ func SendReconnectShard(shardID int, reidentify bool) (err error) { queryParams = "?reidentify=1" } - err = Post(shardID, fmt.Sprintf("shard/%d/reconnect"+queryParams, shardID), nil, nil) + err = internalapi.PostWithShard(shardID, fmt.Sprintf("shard/%d/reconnect"+queryParams, shardID), nil, nil) return } @@ -299,42 +205,6 @@ func SendReconnectAll(reidentify bool) (err error) { queryParams = "?reidentify=1" } - err = Post(0, "shard/*/reconnect"+queryParams, nil, nil) + err = internalapi.PostWithShard(0, "shard/*/reconnect"+queryParams, nil, nil) return } - -var ( - lastPing time.Time - lastPingMutex sync.RWMutex -) - -func RunPinger() { - lastFailed := false - for { - time.Sleep(time.Second) - - var dest string - err := Get(0, "ping", &dest) - if err != nil { - if !lastFailed { - clientLogger.Warn("Ping to bot failed: ", err) - lastFailed = true - } - continue - } else if lastFailed { - clientLogger.Info("Ping to bot succeeded again after failing!") - } - - lastPingMutex.Lock() - lastPing = time.Now() - lastPingMutex.Unlock() - lastFailed = false - } -} - -// Returns wether the bot is running or not, (time since last sucessfull ping was less than 5 seconds) -func BotIsRunning() bool { - lastPingMutex.RLock() - defer lastPingMutex.RUnlock() - return time.Since(lastPing) < time.Second*5 -} diff --git a/bot/botrest/server.go b/bot/botrest/server.go index 391bf17d7a..aae3525b09 100644 --- a/bot/botrest/server.go +++ b/bot/botrest/server.go @@ -1,24 +1,19 @@ package botrest import ( - "context" - "encoding/json" "net/http" - "net/http/pprof" "os" "sort" "strconv" - "sync" "time" "emperror.dev/errors" "github.com/jonas747/discordgo" "github.com/jonas747/dstate" "github.com/jonas747/dutil" - "github.com/jonas747/retryableredis" "github.com/jonas747/yagpdb/bot" "github.com/jonas747/yagpdb/common" - "github.com/jonas747/yagpdb/common/config" + "github.com/jonas747/yagpdb/common/internalapi" "goji.io" "goji.io/pat" ) @@ -28,20 +23,17 @@ func RegisterPlugin() { } var ( - _ bot.BotInitHandler = (*Plugin)(nil) - _ bot.BotStopperHandler = (*Plugin)(nil) + // _ bot.BotInitHandler = (*Plugin)(nil) + _ internalapi.InternalAPIPlugin = (*Plugin)(nil) ) -var confBotrestListenAddr = config.RegisterOption("yagpdb.botrest.listen_address", "botrest listening address, it will use the first port available above 5010", "127.0.0.1") var serverLogger = common.GetFixedPrefixLogger("botrest_server") -type BotRestPlugin interface { - InitBotRestServer(mux *goji.Mux) -} +// type BotRestPlugin interface { +// InitBotRestServer(mux *goji.Mux) +// } type Plugin struct { - srv *http.Server - srvMU sync.Mutex } func (p *Plugin) PluginInfo() *common.PluginInfo { @@ -52,9 +44,13 @@ func (p *Plugin) PluginInfo() *common.PluginInfo { } } -func (p *Plugin) BotInit() { +func (p *Plugin) InitInternalAPIRoutes(muxer *goji.Mux) { + if !bot.Enabled { + // bot only routes + return + } - muxer := goji.NewMux() + // muxer := goji.NewMux() muxer.HandleFunc(pat.Get("/:guild/guild"), HandleGuild) muxer.HandleFunc(pat.Get("/:guild/botmember"), HandleBotMember) @@ -64,143 +60,13 @@ func (p *Plugin) BotInit() { muxer.HandleFunc(pat.Get("/:guild/channelperms/:channel"), HandleChannelPermissions) muxer.HandleFunc(pat.Get("/node_status"), HandleNodeStatus) muxer.HandleFunc(pat.Post("/shard/:shard/reconnect"), HandleReconnectShard) - muxer.HandleFunc(pat.Get("/ping"), HandlePing) - - // Debug stuff - muxer.HandleFunc(pat.Get("/debug/pprof/*"), pprof.Index) - muxer.HandleFunc(pat.Get("/debug/pprof"), pprof.Index) - muxer.HandleFunc(pat.Get("/debug2/pproff/cmdline"), pprof.Cmdline) - muxer.HandleFunc(pat.Get("/debug2/pproff/profile"), pprof.Profile) - muxer.HandleFunc(pat.Get("/debug2/pproff/symbol"), pprof.Symbol) - muxer.HandleFunc(pat.Get("/debug2/pproff/trace"), pprof.Trace) - - for _, p := range common.Plugins { - if botRestPlugin, ok := p.(BotRestPlugin); ok { - botRestPlugin.InitBotRestServer(muxer) - } - } - - p.srv = &http.Server{ - Handler: muxer, - } - - currentPort := 5010 - - go func() { - // listen address excluding port - listenAddr := confBotrestListenAddr.GetString() - if listenAddr == "" { - // default to safe loopback interface - listenAddr = "127.0.0.1" - } - - for { - address := listenAddr + ":" + strconv.Itoa(currentPort) - - serverLogger.Println("starting botrest on ", address) - - p.srvMU.Lock() - p.srv.Addr = address - p.srvMU.Unlock() - - err := p.srv.ListenAndServe() - if err != nil { - // Shutdown was called for graceful shutdown - if err == http.ErrServerClosed { - serverLogger.Info("server closed, shutting down...") - return - } - - // Retry with a higher port until we succeed - serverLogger.WithError(err).Error("failed starting botrest http server on ", address, " trying again on next port") - currentPort++ - time.Sleep(time.Millisecond) - continue - } - - serverLogger.Println("botrest returned without any error") - break - } - }() - - // Wait for the server address to stop changing - go func() { - lastAddr := "" - lastChange := time.Now() - for { - p.srvMU.Lock() - addr := p.srv.Addr - p.srvMU.Unlock() - - if lastAddr != addr { - lastAddr = addr - time.Sleep(time.Second) - lastChange = time.Now() - continue - } - - if time.Since(lastChange) > time.Second*5 { - // found avaiable port - go p.mapper(lastAddr) - return - } - - time.Sleep(time.Second) - } - }() -} - -func (p *Plugin) mapper(address string) { - t := time.NewTicker(time.Second * 10) - for { - p.mapAddressToShards(address) - <-t.C - } -} - -func (p *Plugin) mapAddressToShards(address string) { - - processShards := bot.GetProcessShards() - - // serverLogger.Debug("mapping ", address, " to current process shards") - for _, shard := range processShards { - err := common.RedisPool.Do(retryableredis.Cmd(nil, "SET", RedisKeyShardAddressMapping(shard), address)) - if err != nil { - serverLogger.WithError(err).Error("failed mapping botrest") - } - } - - if bot.UsingOrchestrator { - err := common.RedisPool.Do(retryableredis.Cmd(nil, "SET", RedisKeyNodeAddressMapping(bot.NodeConn.GetIDLock()), address)) - if err != nil { - serverLogger.WithError(err).Error("failed mapping node") - } - } -} - -func (p *Plugin) StopBot(wg *sync.WaitGroup) { - p.srv.Shutdown(context.TODO()) - wg.Done() -} - -func ServeJson(w http.ResponseWriter, r *http.Request, data interface{}) { - err := json.NewEncoder(w).Encode(data) - if err != nil { - serverLogger.WithError(err).Error("Failed sending json") - } -} - -// Returns true if an error occured -func ServerError(w http.ResponseWriter, r *http.Request, err error) bool { - if err == nil { - return false - } - encodedErr, _ := json.Marshal(err.Error()) + // for _, p := range common.Plugins { + // if botRestPlugin, ok := p.(BotRestPlugin); ok { + // botRestPlugin.InitBotRestServer(muxer) + // } + // } - w.WriteHeader(http.StatusInternalServerError) - w.Write(encodedErr) - return true } func HandleGuild(w http.ResponseWriter, r *http.Request) { @@ -208,13 +74,13 @@ func HandleGuild(w http.ResponseWriter, r *http.Request) { guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } gCopy := guild.DeepCopy(true, true, false, true) - ServeJson(w, r, gCopy) + internalapi.ServeJson(w, r, gCopy) } func HandleBotMember(w http.ResponseWriter, r *http.Request) { @@ -222,17 +88,17 @@ func HandleBotMember(w http.ResponseWriter, r *http.Request) { guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } member := guild.MemberDGoCopy(true, common.BotUser.ID) if member == nil { - ServerError(w, r, errors.New("Bot Member not found")) + internalapi.ServerError(w, r, errors.New("Bot Member not found")) return } - ServeJson(w, r, member) + internalapi.ServeJson(w, r, member) } func HandleGetMembers(w http.ResponseWriter, r *http.Request) { @@ -240,18 +106,18 @@ func HandleGetMembers(w http.ResponseWriter, r *http.Request) { uIDs, ok := r.URL.Query()["users"] if !ok || len(uIDs) < 1 { - ServerError(w, r, errors.New("No id's provided")) + internalapi.ServerError(w, r, errors.New("No id's provided")) return } if len(uIDs) > 100 { - ServerError(w, r, errors.New("Too many ids provided")) + internalapi.ServerError(w, r, errors.New("Too many ids provided")) return } guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } @@ -267,7 +133,7 @@ func HandleGetMembers(w http.ResponseWriter, r *http.Request) { members[i] = v.DGoCopy() } - ServeJson(w, r, members) + internalapi.ServeJson(w, r, members) } func HandleGetMemberColors(w http.ResponseWriter, r *http.Request) { @@ -275,18 +141,18 @@ func HandleGetMemberColors(w http.ResponseWriter, r *http.Request) { uIDs, ok := r.URL.Query()["users"] if !ok || len(uIDs) < 1 { - ServerError(w, r, errors.New("No id's provided")) + internalapi.ServerError(w, r, errors.New("No id's provided")) return } if len(uIDs) > 100 { - ServerError(w, r, errors.New("Too many ids provided")) + internalapi.ServerError(w, r, errors.New("Too many ids provided")) return } guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } @@ -322,7 +188,7 @@ func HandleGetMemberColors(w http.ResponseWriter, r *http.Request) { } } - ServeJson(w, r, colors) + internalapi.ServeJson(w, r, colors) } func HandleGetOnlineCount(w http.ResponseWriter, r *http.Request) { @@ -330,7 +196,7 @@ func HandleGetOnlineCount(w http.ResponseWriter, r *http.Request) { guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } @@ -343,7 +209,7 @@ func HandleGetOnlineCount(w http.ResponseWriter, r *http.Request) { } guild.RUnlock() - ServeJson(w, r, count) + internalapi.ServeJson(w, r, count) } func HandleChannelPermissions(w http.ResponseWriter, r *http.Request) { @@ -352,22 +218,22 @@ func HandleChannelPermissions(w http.ResponseWriter, r *http.Request) { guild := bot.State.Guild(true, gId) if guild == nil { - ServerError(w, r, errors.New("Guild not found")) + internalapi.ServerError(w, r, errors.New("Guild not found")) return } perms, err := guild.MemberPermissions(true, cId, common.BotUser.ID) if err != nil { - ServerError(w, r, errors.WithMessage(err, "Error calculating perms")) + internalapi.ServerError(w, r, errors.WithMessage(err, "Error calculating perms")) return } - ServeJson(w, r, perms) + internalapi.ServeJson(w, r, perms) } func HandlePing(w http.ResponseWriter, r *http.Request) { - ServeJson(w, r, "pong") + internalapi.ServeJson(w, r, "pong") } type ShardStatus struct { @@ -425,7 +291,7 @@ func HandleNodeStatus(w http.ResponseWriter, r *http.Request) { // Guild guild stats gSlice := bot.State.GuildsSlice(true) for _, g := range gSlice { - shardID := bot.GuildShardID(g.ID) + shardID := bot.GuildShardID(int64(numShards), g.ID) available := g.IsAvailable(true) for _, v := range result { if v.ShardID == shardID { @@ -440,7 +306,7 @@ func HandleNodeStatus(w http.ResponseWriter, r *http.Request) { hostname, _ := os.Hostname() - ServeJson(w, r, &NodeStatus{ + internalapi.ServeJson(w, r, &NodeStatus{ Host: hostname, Shards: result, ID: common.NodeID, @@ -453,24 +319,24 @@ func HandleReconnectShard(w http.ResponseWriter, r *http.Request) { forceReidentify := r.FormValue("reidentify") == "1" if sID == "*" { go RestartAll(forceReidentify) - ServeJson(w, r, "ok") + internalapi.ServeJson(w, r, "ok") return } parsed, _ := strconv.ParseInt(sID, 10, 32) shardcount := bot.ShardManager.GetNumShards() if parsed < 0 || int(parsed) >= shardcount { - ServerError(w, r, errors.New("Unknown shard")) + internalapi.ServerError(w, r, errors.New("Unknown shard")) return } err := bot.ShardManager.Sessions[parsed].GatewayManager.Reconnect(forceReidentify) if err != nil { - ServerError(w, r, errors.WithMessage(err, "Reconnect")) + internalapi.ServerError(w, r, errors.WithMessage(err, "Reconnect")) return } - ServeJson(w, r, "ok") + internalapi.ServeJson(w, r, "ok") } func RestartAll(reidentify bool) { diff --git a/bot/nodeimpl.go b/bot/nodeimpl.go index 9c07dd4ab1..a1e31d43fe 100644 --- a/bot/nodeimpl.go +++ b/bot/nodeimpl.go @@ -50,7 +50,7 @@ func (n *NodeImpl) SessionEstablished(info node.SessionInfo) { if err != nil { panic("failed initializing discord sessions: " + err.Error()) } - InitPlugins() + botReady() } } @@ -160,7 +160,7 @@ func (n *NodeImpl) SendGuilds(shard int) int { guildsToSend := make([]*dstate.GuildState, 0) State.RLock() for _, v := range State.Guilds { - shardID := GuildShardID(v.ID) + shardID := guildShardID(v.ID) if int(shardID) == shard { guildsToSend = append(guildsToSend, v) } diff --git a/bot/util.go b/bot/util.go index f0ec94d4ae..51796f662c 100644 --- a/bot/util.go +++ b/bot/util.go @@ -4,8 +4,6 @@ import ( "context" "strconv" "strings" - "sync" - "sync/atomic" "time" "emperror.dev/errors" @@ -241,62 +239,18 @@ func IsGuildOnCurrentProcess(guildID int64) bool { } // GuildShardID returns the shard id for the provided guild id -func GuildShardID(guildID int64) int { - totShards := GetTotalShards() - - shardID := int((guildID >> 22) % totShards) - return shardID -} - -var runShardPollerOnce sync.Once - -// GetTotalShards either retrieves the total shards from passed command line if the bot is set to run in the same process -// or it starts a background poller to poll redis for it every second -func GetTotalShards() int64 { - // if the bot is running on this process, then we know the number of total shards - if Enabled && totalShardCount != 0 { - return int64(totalShardCount) - } - - // otherwise we poll it from redis every second - runShardPollerOnce.Do(func() { - err := fetchTotalShardsFromRedis() - if err != nil { - panic("failed retrieving shards") - } - - go runNumShardsUpdater() - }) - - return atomic.LoadInt64(redisSetTotalShards) +func guildShardID(guildID int64) int { + return GuildShardID(getTotalShards(), guildID) } -var redisSetTotalShards = new(int64) - -func runNumShardsUpdater() { - t := time.NewTicker(time.Second) - for { - err := fetchTotalShardsFromRedis() - if err != nil { - logger.WithError(err).Error("[botrest] failed retrieving total shards") - } - <-t.C - } +// GuildShardID returns the shard id for the provided guild id +func GuildShardID(totalShards, guildID int64) int { + shardID := int((guildID >> 22) % totalShards) + return shardID } -func fetchTotalShardsFromRedis() error { - var result int64 - err := common.RedisPool.Do(retryableredis.Cmd(&result, "GET", "yagpdb_total_shards")) - if err != nil { - return err - } - - old := atomic.SwapInt64(redisSetTotalShards, result) - if old != result { - logger.Info("[botrest] new shard count received: ", old, " -> ", result) - } - - return nil +func getTotalShards() int64 { + return int64(totalShardCount) } func GetProcessShards() []int { diff --git a/cah/plugin_bot.go b/cah/plugin_bot.go index bd0c7f293f..2026f788b6 100644 --- a/cah/plugin_bot.go +++ b/cah/plugin_bot.go @@ -118,7 +118,7 @@ func (p *Plugin) ShardMigrationSend(shard int) int { games := make([]*cardsagainstdiscord.Game, 0) OUTER: for k, v := range p.Manager.ActiveGames { - if bot.GuildShardID(v.GuildID) != shard { + if bot.GuildShardID(int64(bot.ShardManager.GetNumShards()), v.GuildID) != shard { continue } diff --git a/cmd/shardorchestrator/main.go b/cmd/shardorchestrator/main.go index d8c4547c7c..ebef8e9b53 100644 --- a/cmd/shardorchestrator/main.go +++ b/cmd/shardorchestrator/main.go @@ -2,6 +2,8 @@ package main import ( "log" + "os" + "os/exec" "strconv" "strings" "time" @@ -45,13 +47,22 @@ func main() { orch.FixedTotalShardCount = totalShards orch.ResponsibleForShards = activeShards orch.NodeLauncher = &orchestrator.StdNodeLauncher{ - CmdName: "./capturepanics", - Args: []string{"./yagpdb", "-bot", "-syslog"}, + LaunchCmdName: "./capturepanics", + LaunchArgs: []string{"./yagpdb", "-bot", "-syslog"}, + + VersionCmdName: "./yagpdb", + VersionArgs: []string{"-version"}, } orch.Logger = &dshardorchestrator.StdLogger{ Level: dshardorchestrator.LogWarning, } + updateScript := "updateversion.sh" + orch.VersionUpdater = &Updater{ + ScriptLocation: updateScript, + orchestrator: orch, + } + orch.MaxShardsPerNode = 10 orch.MaxNodeDowntimeBeforeRestart = time.Second * 10 orch.EnsureAllShardsRunning = true @@ -63,7 +74,15 @@ func main() { go UpdateRedisNodes(orch) - api := rest.NewRESTAPI(orch, "127.0.0.1:7448") + restAPIAddr := os.Getenv("YAGPDB_BOTREST_LISTEN_ADDRESS") + if restAPIAddr == "" { + restAPIAddr = "127.0.0.1" + } + + api := rest.NewRESTAPI(orch, restAPIAddr+":7448") + common.ServiceTracker.SetAPIAddress(restAPIAddr + ":7448") + common.ServiceTracker.RegisterService(common.ServiceTypeOrchestator, "Shard orchestrator", "", nil) + err = api.Run() if err != nil { log.Fatal("failed starting rest api: ", err) @@ -132,3 +151,27 @@ func ReadActiveShards() []int { return shards } + +type Updater struct { + ScriptLocation string + orchestrator *orchestrator.Orchestrator +} + +func (u *Updater) PullNewVersion() (string, error) { + err := os.Mkdir("updating", 770) + if err != nil && err != os.ErrExist { + return "", err + } + + logrus.Println("Updatig version") + cmd := exec.Command("/bin/bash", u.ScriptLocation) + cmd.Dir = "updating/" + + output, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + + logrus.Println("Update output: ", output) + return u.orchestrator.NodeLauncher.LaunchVersion() +} diff --git a/cmd/yagpdb/main.go b/cmd/yagpdb/main.go index 5d70cfce2f..ce9b5ec907 100644 --- a/cmd/yagpdb/main.go +++ b/cmd/yagpdb/main.go @@ -12,11 +12,10 @@ import ( "time" "github.com/evalphobia/logrus_sentry" - "github.com/jonas747/yagpdb/automod" - "github.com/jonas747/yagpdb/safebrowsing" log "github.com/sirupsen/logrus" // Core yagpdb packages + "github.com/jonas747/yagpdb/admin" "github.com/jonas747/yagpdb/bot" "github.com/jonas747/yagpdb/bot/botrest" "github.com/jonas747/yagpdb/bot/paginatedmessages" @@ -24,6 +23,7 @@ import ( "github.com/jonas747/yagpdb/common/backgroundworkers" "github.com/jonas747/yagpdb/common/config" "github.com/jonas747/yagpdb/common/configstore" + "github.com/jonas747/yagpdb/common/internalapi" "github.com/jonas747/yagpdb/common/mqueue" "github.com/jonas747/yagpdb/common/pubsub" "github.com/jonas747/yagpdb/common/scheduledevents2" @@ -31,6 +31,7 @@ import ( "github.com/jonas747/yagpdb/web" // Plugin imports + "github.com/jonas747/yagpdb/automod" "github.com/jonas747/yagpdb/automod_legacy" "github.com/jonas747/yagpdb/autorole" "github.com/jonas747/yagpdb/aylien" @@ -48,6 +49,7 @@ import ( "github.com/jonas747/yagpdb/reputation" "github.com/jonas747/yagpdb/rolecommands" "github.com/jonas747/yagpdb/rsvp" + "github.com/jonas747/yagpdb/safebrowsing" "github.com/jonas747/yagpdb/serverstats" "github.com/jonas747/yagpdb/soundboard" "github.com/jonas747/yagpdb/stdcommands" @@ -183,6 +185,8 @@ func main() { twitter.RegisterPlugin() rsvp.RegisterPlugin() timezonecompanion.RegisterPlugin() + admin.RegisterPlugin() + internalapi.RegisterPlugin() if flagDryRun { log.Println("This is a dry run, exiting") @@ -225,6 +229,9 @@ func main() { go pubsub.PollEvents() + common.RunCommonRunPlugins() + + common.SetShutdownFunc(shutdown) listenSignal() } @@ -235,8 +242,12 @@ func listenSignal() { c := make(chan os.Signal, 2) signal.Notify(c, os.Interrupt, syscall.SIGTERM) - sig := <-c - log.Info("SHUTTING DOWN... ", sig.String()) + <-c + common.Shutdown() +} + +func shutdown() { + log.Info("SHUTTING DOWN... ") shouldWait := false wg := new(sync.WaitGroup) diff --git a/cmd/yagpdb/templates/status.html b/cmd/yagpdb/templates/status.html index 9274cd3d1c..7363a314a3 100644 --- a/cmd/yagpdb/templates/status.html +++ b/cmd/yagpdb/templates/status.html @@ -8,14 +8,6 @@

Bot status

{{template "cp_alerts" .}} -{{if not .BotRunning}} -
-
-

Bot is currently not running

-
- -
-{{else}}

Summary

@@ -26,7 +18,9 @@

Summary

  • Nodes: {{.BotStatus.NumNodes}}
  • Shards: {{.BotStatus.TotalShards}}
  • Offline shards: {{len .BotStatus.OfflineShards}}
    • -
  • Min, Max, Avg Events/s: {{printf "%.1f" .BotStatus.EventsPerSecondMin}}/{{printf "%.1f" .BotStatus.EventsPerSecondMax}}/{{printf "%.1f" .BotStatus.EventsPerSecondAverage}}
    • +
  • Min, Max, Avg Events/s: + {{printf "%.1f" .BotStatus.EventsPerSecondMin}}/{{printf "%.1f" .BotStatus.EventsPerSecondMax}}/{{printf "%.1f" .BotStatus.EventsPerSecondAverage}} +
  • Unavailable guilds: {{.BotStatus.UnavailableGuilds}}
  • Min Uptime: {{.BotStatus.UptimeMin}}
  • Max Uptime: {{.BotStatus.UptimeMax}}
    • @@ -35,77 +29,77 @@

    Summary

    {{range .BotStatus.HostStatuses}} @@ -119,10 +113,11 @@

    Host {{.Name}}

    Node {{.ID}} - {{humanizeDurationMinutes .Uptime}}
    -
    +
    {{range $i, $v := .Shards}} -
    +

    {{.ShardID}}

    @@ -138,16 +133,16 @@

    Host {{.Name}}

    -{{end}}{{end}} +{{end}} {{template "cp_footer" .}} {{end}} {{define "cp_status_tooltip"}} -Events/s: {{printf "%.1f" .EventsPerSecond}}
    -TotalEvents: {{.TotalEvents}}
    -Last Heartbeat: {{humanizeDurationSeconds (currentTime.Sub .LastHeartbeatSend)}} ago
    -Last Heartbeat Ack: {{humanizeDurationSeconds (currentTime.Sub .LastHeartbeatAck)}} ago
    -Unavailable Guilds: {{.UnavailableGuilds}}/{{.NumGuilds}}
    -{{end}} +Events/s: {{printf "%.1f" .EventsPerSecond}}
    +TotalEvents: {{.TotalEvents}}
    +Last Heartbeat: {{humanizeDurationSeconds (currentTime.Sub .LastHeartbeatSend)}} ago
    +Last Heartbeat Ack: {{humanizeDurationSeconds (currentTime.Sub .LastHeartbeatAck)}} ago
    +Unavailable Guilds: {{.UnavailableGuilds}}/{{.NumGuilds}}
    +{{end}} \ No newline at end of file diff --git a/common/backgroundworkers/backgroundworkers.go b/common/backgroundworkers/backgroundworkers.go index 98ec3872c6..cd129208fc 100644 --- a/common/backgroundworkers/backgroundworkers.go +++ b/common/backgroundworkers/backgroundworkers.go @@ -23,6 +23,8 @@ type BackgroundWorkerPlugin interface { } func RunWorkers() { + common.ServiceTracker.RegisterService(common.ServiceTypeBGWorker, "Background worker", "", nil) + RESTServerMuxer = goji.NewMux() for _, p := range common.Plugins { diff --git a/common/common.go b/common/common.go index 60e352a35f..111293e3ca 100644 --- a/common/common.go +++ b/common/common.go @@ -11,6 +11,7 @@ import ( "net/http" "os" "strings" + "sync" "time" "github.com/DataDog/datadog-go/statsd" @@ -230,3 +231,31 @@ func connectDB(host, user, pass, dbName string) error { return err } + +var ( + shutdownFunc func() + shutdownCalled bool + shutdownMU sync.Mutex +) + +func Shutdown() { + shutdownMU.Lock() + f := shutdownFunc + if f == nil || shutdownCalled { + shutdownMU.Unlock() + return + } + + shutdownCalled = true + shutdownMU.Unlock() + + if f != nil { + f() + } +} + +func SetShutdownFunc(f func()) { + shutdownMU.Lock() + shutdownFunc = f + shutdownMU.Unlock() +} diff --git a/common/internalapi/clientutil.go b/common/internalapi/clientutil.go new file mode 100644 index 0000000000..a196857f6a --- /dev/null +++ b/common/internalapi/clientutil.go @@ -0,0 +1,113 @@ +package internalapi + +import ( + "bytes" + "encoding/json" + "net/http" + + "emperror.dev/errors" + "github.com/jonas747/yagpdb/common" +) + +var ( + ErrServerError = errors.New("internal api server is having issues") + ErrCantFindAddress = errors.New("can't find address for provided shard") +) + +func GetServerAddrForGuild(guildID int64) string { + addr, _ := common.ServicePoller.GetGuildAddress(guildID) + return addr +} + +func GetServerAddrForShard(shard int) string { + addr, _ := common.ServicePoller.GetShardAddress(shard) + return addr +} + +func GetWithGuild(guildID int64, url string, dest interface{}) error { + serverAddr := GetServerAddrForGuild(guildID) + if serverAddr == "" { + return ErrCantFindAddress + } + + return GetWithAddress(serverAddr, url, dest) +} + +func GetWithShard(shard int, url string, dest interface{}) error { + serverAddr := GetServerAddrForShard(shard) + if serverAddr == "" { + return ErrCantFindAddress + } + + return GetWithAddress(serverAddr, url, dest) +} + +func GetWithAddress(addr string, url string, dest interface{}) error { + resp, err := http.Get("http://" + addr + "/" + url) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var errDest string + err := json.NewDecoder(resp.Body).Decode(&errDest) + if err != nil { + return ErrServerError + } + + return errors.New(errDest) + } + + return errors.WithMessage(json.NewDecoder(resp.Body).Decode(dest), "json.Decode") +} + +func PostWithGuild(guildID int64, url string, bodyData interface{}, dest interface{}) error { + serverAddr := GetServerAddrForGuild(guildID) + if serverAddr == "" { + return ErrCantFindAddress + } + + return PostWithAddress(serverAddr, url, bodyData, dest) +} + +func PostWithShard(shard int, url string, bodyData interface{}, dest interface{}) error { + serverAddr := GetServerAddrForShard(shard) + if serverAddr == "" { + return ErrCantFindAddress + } + + return PostWithAddress(serverAddr, url, bodyData, dest) +} + +func PostWithAddress(serverAddr string, url string, bodyData interface{}, dest interface{}) error { + var bodyBuf bytes.Buffer + if bodyData != nil { + encoder := json.NewEncoder(&bodyBuf) + err := encoder.Encode(bodyData) + if err != nil { + return err + } + } + resp, err := http.Post("http://"+serverAddr+"/"+url, "application/json", &bodyBuf) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var errDest string + err := json.NewDecoder(resp.Body).Decode(&errDest) + if err != nil { + return errors.Errorf("Resp code:%d, Error: %v", resp.StatusCode, err) + } + + return errors.New(errDest) + } + + if dest == nil { + return nil + } + + return errors.WithMessage(json.NewDecoder(resp.Body).Decode(dest), "json.Decode") +} diff --git a/common/internalapi/server.go b/common/internalapi/server.go new file mode 100644 index 0000000000..bf705379dd --- /dev/null +++ b/common/internalapi/server.go @@ -0,0 +1,156 @@ +package internalapi + +import ( + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/pprof" + "sync" + "time" + + "github.com/jonas747/yagpdb/common" + "github.com/jonas747/yagpdb/common/config" + "goji.io" + "goji.io/pat" +) + +var _ common.PluginWithCommonRun = (*Plugin)(nil) + +func RegisterPlugin() { + common.RegisterPlugin(&Plugin{}) +} + +var confBotrestListenAddr = config.RegisterOption("yagpdb.botrest.listen_address", "botrest listening address, it will use any available port and make which port used avialable using service discovery (see service.go)", "127.0.0.1") +var serverLogger = common.GetFixedPrefixLogger("internalapi_server") + +// InternalAPIPlugin represents a plugin that provides interactions with the internal apis +type InternalAPIPlugin interface { + InitInternalAPIRoutes(mux *goji.Mux) +} + +type Plugin struct { + srv *http.Server + srvMU sync.Mutex +} + +func (p *Plugin) PluginInfo() *common.PluginInfo { + return &common.PluginInfo{ + Name: "internalapi", + SysName: "internalapi", + Category: common.PluginCategoryCore, + } +} + +func (p *Plugin) CommonRun() { + + muxer := goji.NewMux() + + // muxer.HandleFunc(pat.Get("/:guild/guild"), HandleGuild) + // muxer.HandleFunc(pat.Get("/:guild/botmember"), HandleBotMember) + // muxer.HandleFunc(pat.Get("/:guild/members"), HandleGetMembers) + // muxer.HandleFunc(pat.Get("/:guild/membercolors"), HandleGetMemberColors) + // muxer.HandleFunc(pat.Get("/:guild/onlinecount"), HandleGetOnlineCount) + // muxer.HandleFunc(pat.Get("/:guild/channelperms/:channel"), HandleChannelPermissions) + // muxer.HandleFunc(pat.Get("/node_status"), HandleNodeStatus) + // muxer.HandleFunc(pat.Post("/shard/:shard/reconnect"), HandleReconnectShard) + muxer.HandleFunc(pat.Get("/ping"), handlePing) + muxer.HandleFunc(pat.Post("/shutdown"), handleShutdown) + + // Debug stuff + muxer.HandleFunc(pat.Get("/debug/pprof/cmdline"), pprof.Cmdline) + muxer.HandleFunc(pat.Get("/debug/pprof/profile"), pprof.Profile) + muxer.HandleFunc(pat.Get("/debug/pprof/symbol"), pprof.Symbol) + muxer.HandleFunc(pat.Get("/debug/pprof/trace"), pprof.Trace) + muxer.HandleFunc(pat.Get("/debug/pprof/"), pprof.Index) + muxer.HandleFunc(pat.Get("/debug/pprof/:profile"), pprof.Index) + + // http.HandleFunc("/debug/pprof/", Index) + // http.HandleFunc("/debug/pprof/cmdline", Cmdline) + // http.HandleFunc("/debug/pprof/profile", Profile) + // http.HandleFunc("/debug/pprof/symbol", Symbol) + // http.HandleFunc("/debug/pprof/trace", Trace) + + for _, p := range common.Plugins { + if botRestPlugin, ok := p.(InternalAPIPlugin); ok { + botRestPlugin.InitInternalAPIRoutes(muxer) + } + } + + p.srv = &http.Server{ + Handler: muxer, + } + + // listen address excluding port + listenAddr := confBotrestListenAddr.GetString() + if listenAddr == "" { + // default to safe loopback interface + listenAddr = "127.0.0.1" + } + + l, port, err := p.createListener(listenAddr) + if err != nil { + serverLogger.WithError(err).Panicf("failed starting internal http server on %s:%d", listenAddr, port) + return + } + + common.ServiceTracker.SetAPIAddress(fmt.Sprintf("%s:%d", listenAddr, port)) + + go func() { + + err := p.srv.Serve(l) + if err != nil { + if err == http.ErrServerClosed { + serverLogger.Info("server closed, shutting down...") + return + } + + serverLogger.WithError(err).Error("failed serving internal api") + return + } + }() +} + +func (p *Plugin) createListener(addr string) (net.Listener, int, error) { + listener, err := net.Listen("tcp", addr+":0") + if err != nil { + return nil, 0, err + } + + port := listener.Addr().(*net.TCPAddr).Port + serverLogger.Infof("internalapi using port %d", port) + return listener, port, nil +} + +func ServeJson(w http.ResponseWriter, r *http.Request, data interface{}) { + err := json.NewEncoder(w).Encode(data) + if err != nil { + serverLogger.WithError(err).Error("Failed sending json") + } +} + +// Returns true if an error occured +func ServerError(w http.ResponseWriter, r *http.Request, err error) bool { + if err == nil { + return false + } + + encodedErr, _ := json.Marshal(err.Error()) + + w.WriteHeader(http.StatusInternalServerError) + w.Write(encodedErr) + return true +} + +func handlePing(w http.ResponseWriter, r *http.Request) { + ServeJson(w, r, "pong") +} + +func handleShutdown(w http.ResponseWriter, r *http.Request) { + go func() { + time.Sleep(time.Second * 3) + common.Shutdown() + }() + + ServeJson(w, r, "shutting down in 3 seconds") +} diff --git a/common/plugins.go b/common/plugins.go index 3db5e8ddca..fbad9e20f7 100644 --- a/common/plugins.go +++ b/common/plugins.go @@ -16,6 +16,7 @@ var ( PluginCategoryFeeds = &PluginCategory{Name: "Feeds", Order: 30} ) +// PluginInfo represents basic plugin information type PluginInfo struct { Name string // Human readable name of the plugin SysName string // snake_case version of the name in lower case @@ -32,3 +33,17 @@ func RegisterPlugin(plugin Plugin) { Plugins = append(Plugins, plugin) logger.Info("Registered plugin: " + plugin.PluginInfo().Name) } + +// PluginWithCommonRun is for plugins that include a function that's always run, no matter if its the webserver frontend, bot or whatever +type PluginWithCommonRun interface { + CommonRun() +} + +// RunCommonRunPlugins runs plugins that implement PluginWithCommonRun +func RunCommonRunPlugins() { + for _, v := range Plugins { + if cast, ok := v.(PluginWithCommonRun); ok { + go cast.CommonRun() + } + } +} diff --git a/common/service.go b/common/service.go new file mode 100644 index 0000000000..bbf1a67646 --- /dev/null +++ b/common/service.go @@ -0,0 +1,292 @@ +package common + +import ( + "bytes" + "encoding/json" + "os" + "sync" + "time" + + "emperror.dev/errors" + + "github.com/jonas747/retryableredis" +) + +const ServicesRedisKey = "yag_services" + +type serviceTracker struct { + host *ServiceHost + + lastUpdate []byte + + mu sync.Mutex +} + +// ServiceType represents the type of the component +type ServiceType string + +const ( + ServiceTypeBot ServiceType = "bot" + ServiceTypeFrontend ServiceType = "frontend" + ServiceTypeBGWorker ServiceType = "bgworker" + ServiceTypeFeed ServiceType = "feed" + ServiceTypeOrchestator ServiceType = "orchestrator" +) + +// Service represents a service or component of yagpdb +type Service struct { + Type ServiceType `json:"type"` + Name string `json:"name"` + Details string `json:"details"` + + botDetailsF func() (*BotServiceDetails, error) + BotDetails *BotServiceDetails `json:"bot_details"` +} + +// BotServiceDetails is bot service specific details +type BotServiceDetails struct { + RunningShards []int `json:"running_shards"` + TotalShards int `json:"total_shards"` + NodeID string `json:"node_id"` + OrchestratorMode bool `json:"orchestrator_mode"` +} + +// ServiceHost represents a process that holds oen or more bot components +type ServiceHost struct { + InternalAPIAddress string `json:"internal_api_address"` + Host string `json:"host"` + PID int `json:"pid"` + Version string `json:"version"` + + Services []*Service `json:"services"` +} + +// ServiceTracker keeps track of the various components of yagpdb in a central location for ease of access +var ServiceTracker = newServiceTracker() + +func newServiceTracker() *serviceTracker { + hostname, _ := os.Hostname() + + st := &serviceTracker{ + host: &ServiceHost{ + Host: hostname, + PID: os.Getpid(), + Version: VERSION, + }, + } + + go st.run() + + return st +} + +func (s *serviceTracker) RegisterService(t ServiceType, name string, details string, extrasF func() (*BotServiceDetails, error)) { + s.mu.Lock() + defer s.mu.Unlock() + + s.host.Services = append(s.host.Services, &Service{ + Type: t, + Name: name, + Details: details, + botDetailsF: extrasF, + }) +} + +func (s *serviceTracker) SetAPIAddress(apiAddress string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.host.InternalAPIAddress = apiAddress +} + +func (s *serviceTracker) run() { + t := time.NewTicker(time.Second * 5) + for { + <-t.C + s.update() + } +} + +func (s *serviceTracker) update() { + s.mu.Lock() + defer s.mu.Unlock() + + for _, v := range s.host.Services { + if v.botDetailsF == nil { + continue + } + + botDetails, err := v.botDetailsF() + if err != nil { + logger.WithError(err).Error("failed retrieving extra service details") + v.BotDetails = &BotServiceDetails{} + continue + } + + v.BotDetails = botDetails + } + + serialized, err := json.Marshal(s.host) + if err != nil { + logger.WithError(err).Error("failed marshaling service host") + return + } + + if !bytes.Equal(serialized, s.lastUpdate) { + err = RedisPool.Do(retryableredis.FlatCmd(nil, "ZREM", ServicesRedisKey, s.lastUpdate)) + if err != nil { + logger.WithError(err).Error("failed removing service host") + return + } + } + + err = RedisPool.Do(retryableredis.FlatCmd(nil, "ZADD", ServicesRedisKey, time.Now().Unix(), serialized)) + if err != nil { + logger.WithError(err).Error("failed updating service host") + return + } + + s.lastUpdate = serialized + + err = RedisPool.Do(retryableredis.FlatCmd(nil, "ZREMRANGEBYSCORE", ServicesRedisKey, 0, time.Now().Unix()-30)) + if err != nil { + logger.WithError(err).Error("feailed clearing old service hosts") + return + } +} + +type servicePoller struct { + cachedServiceHosts []*ServiceHost + lastPoll time.Time + mu sync.Mutex +} + +var ServicePoller = &servicePoller{} + +func (sp *servicePoller) getActiveServiceHosts() ([]*ServiceHost, error) { + if time.Since(sp.lastPoll) < time.Second*5 { + return sp.cachedServiceHosts, nil + } + + var hosts []string + + err := RedisPool.Do(retryableredis.FlatCmd(&hosts, "ZRANGE", ServicesRedisKey, 0, -1)) + if err != nil { + return nil, errors.WithStackIf(err) + } + + result := make([]*ServiceHost, 0, len(hosts)) + for _, v := range hosts { + var parsed *ServiceHost + err = json.Unmarshal([]byte(v), &parsed) + if err != nil { + return nil, errors.WithStackIf(err) + } + + result = append(result, parsed) + } + + sp.cachedServiceHosts = result + sp.lastPoll = time.Now() + + return result, nil +} + +// GetActiveServiceHosts returns all of the running service providers of the bot (processes) +func (sp *servicePoller) GetActiveServiceHosts() ([]*ServiceHost, error) { + sp.mu.Lock() + defer sp.mu.Unlock() + + return sp.getActiveServiceHosts() +} + +// GetShardAddress returns the internal api address of the specified shard +func (sp *servicePoller) GetShardAddress(shardID int) (string, error) { + sp.mu.Lock() + defer sp.mu.Unlock() + + hosts, err := sp.getActiveServiceHosts() + if err != nil { + return "", err + } + + for _, h := range hosts { + for _, v := range h.Services { + if v.Type == ServiceTypeBot && ContainsIntSlice(v.BotDetails.RunningShards, shardID) { + return h.InternalAPIAddress, nil + } + } + } + + return "", ErrNotFound +} + +// GetGuildAddress returns the internal api addrress of the specified shard +// This is preferred over GetShardAddress as it also handles cases of different total shard couns (mid upscaling for example) +func (sp *servicePoller) GetGuildAddress(guildID int64) (string, error) { + sp.mu.Lock() + defer sp.mu.Unlock() + + hosts, err := sp.getActiveServiceHosts() + if err != nil { + return "", err + } + + for _, h := range hosts { + for _, v := range h.Services { + if v.Type != ServiceTypeBot { + continue + } + + shardID := int((guildID >> 22) % int64(v.BotDetails.TotalShards)) + + if ContainsIntSlice(v.BotDetails.RunningShards, shardID) { + return h.InternalAPIAddress, nil + } + } + } + + return "", ErrNotFound +} + +// GetNodeAddress returns the internal api address of the specified nodeID +func (sp *servicePoller) GetNodeAddress(nodeID string) (string, error) { + sp.mu.Lock() + defer sp.mu.Unlock() + + hosts, err := sp.getActiveServiceHosts() + if err != nil { + return "", err + } + + for _, h := range hosts { + for _, v := range h.Services { + if v.BotDetails != nil && v.BotDetails.NodeID == nodeID { + return h.InternalAPIAddress, nil + } + } + } + + return "", ErrNotFound +} + +// GetShardCount returns the total shard count from the first node with a bot, or ErrNotFound otherwise +func (sp *servicePoller) GetShardCount() (int, error) { + sp.mu.Lock() + defer sp.mu.Unlock() + + hosts, err := sp.getActiveServiceHosts() + if err != nil { + return 0, err + } + + for _, h := range hosts { + for _, v := range h.Services { + if v.Type == ServiceTypeBot { + return v.BotDetails.TotalShards, nil + } + } + } + + return 0, ErrNotFound +} diff --git a/feeds/plugin.go b/feeds/plugin.go index d8a8e1406a..43e55e64ad 100644 --- a/feeds/plugin.go +++ b/feeds/plugin.go @@ -47,6 +47,9 @@ func Run(which []string) { go fp.StartFeed() runningPlugins = append(runningPlugins, fp) } + + joined := strings.Join(which, ",") + common.ServiceTracker.RegisterService(common.ServiceTypeFeed, "Feeds", joined, nil) } func Stop(wg *sync.WaitGroup) { diff --git a/go.mod b/go.mod index 4e847316d2..d21e8e4c9a 100644 --- a/go.mod +++ b/go.mod @@ -23,9 +23,9 @@ require ( github.com/fatih/structs v1.1.0 github.com/francoispqt/gojay v1.2.13 // indirect github.com/getsentry/raven-go v0.2.0 // indirect - github.com/gin-gonic/gin v1.4.0 // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/gofrs/uuid v3.2.0+incompatible // indirect + github.com/golang/protobuf v1.3.2 // indirect github.com/google/safebrowsing v0.0.0-20190624211811-bbf0d20d26b3 github.com/gorilla/schema v1.1.0 github.com/jarcoal/httpmock v1.0.4 // indirect @@ -35,7 +35,7 @@ require ( github.com/jonas747/dcmd v1.1.0 github.com/jonas747/dice v0.0.0-20170619144252-7735f6ee7b69 github.com/jonas747/discordgo v1.1.7 - github.com/jonas747/dshardorchestrator v0.0.0-20190717204625-752c847fa039 + github.com/jonas747/dshardorchestrator v0.1.0 github.com/jonas747/dstate v1.0.3 github.com/jonas747/dutil v0.0.2 github.com/jonas747/go-reddit v0.0.0-20181219005523-28353ff8e846 @@ -76,12 +76,13 @@ require ( goji.io v2.0.2+incompatible golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 golang.org/x/image v0.0.0-20190902063713-cb417be4ba39 - golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c + golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b // indirect golang.org/x/time v0.0.0-20181108054448-85acf8d2951c golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect google.golang.org/api v0.3.1 + google.golang.org/appengine v1.6.5 // indirect gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect ) diff --git a/go.sum b/go.sum index 5bd9b2a6d2..87a1ccf49a 100644 --- a/go.sum +++ b/go.sum @@ -38,7 +38,10 @@ github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9Pq github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apmckinlay/gsuneido v0.0.0-20190404155041-0b6cd442a18f/go.mod h1:JU2DOj5Fc6rol0yaT79Csr47QR0vONGwJtBNGRD7jmc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwmarrin/discordgo v0.19.0 h1:kMED/DB0NR1QhRcalb85w0Cu3Ep2OrGAqZH1R5awQiY= @@ -77,6 +80,7 @@ github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DP github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ= github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -96,6 +100,8 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -117,10 +123,13 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -128,6 +137,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/safebrowsing v0.0.0-20190624211811-bbf0d20d26b3 h1:4SV2fLwScO6iAgUKNqXwIrz9Fq2ykQxbSV4ObXtNCWY= github.com/google/safebrowsing v0.0.0-20190624211811-bbf0d20d26b3/go.mod h1:hT4r/grkURkgVSWJaWd6PyS4xfAb+vb34DyMDYiOGa8= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= @@ -142,13 +152,16 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jinzhu/gorm v1.9.10 h1:HvrsqdhCW78xpJF67g1hMxS6eCToo9PZH4LDB8WKPac= github.com/jinzhu/gorm v1.9.10/go.mod h1:Kh6hTsSGffh4ui079FHrR5Gg+5D0hgihqDcsDN2BBJY= @@ -176,6 +189,8 @@ github.com/jonas747/discordgo v1.1.7 h1:If1IF6nvL2swzxGznVzsiUwORyu3mlf/UT0HW1Gb github.com/jonas747/discordgo v1.1.7/go.mod h1:Mq9l7/Kn7L8Np5p//vuxKfnZohrX70XYx9cGoa+19J8= github.com/jonas747/dshardorchestrator v0.0.0-20190717204625-752c847fa039 h1:stgj6F7w3gCDnni7qs4chZdeNyo8TNSJJg0gADdpMsE= github.com/jonas747/dshardorchestrator v0.0.0-20190717204625-752c847fa039/go.mod h1:Cs2ax5vvTtpgy2n8wp5zpBhd6A19ThLykp21XXSsrU0= +github.com/jonas747/dshardorchestrator v0.1.0 h1:3/v+lx0gLupm7/VZr7wbfvUGhpxvuNC5JHaS+OnWjCc= +github.com/jonas747/dshardorchestrator v0.1.0/go.mod h1:ZlA6h9kj2H5q2aQkZgBiROp8zpIjIUMJQ3SOpVdWYfQ= github.com/jonas747/dstate v0.0.0-20190905110424-f2ed8c6c957d/go.mod h1:5xeu+zy7jwCfnCA+A9CaWOV4UhQzGHvBVZ381l+jarM= github.com/jonas747/dstate v1.0.3 h1:5uJe8xbHNBJp1Fxqy7hoenmEnxNhIdG19QXlMurK8Ok= github.com/jonas747/dstate v1.0.3/go.mod h1:cs2apI7cGl8py1ibSN0Ha6iqHEw5k62kwPpGBZFgHUE= @@ -224,8 +239,11 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -235,6 +253,8 @@ github.com/microcosm-cc/bluemonday v1.0.1 h1:SIYunPjnlXcW+gVfvm0IlSeR5U3WZUOLfVm github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miolini/datacounter v0.0.0-20190724021726-aa48df3a02c1 h1:vbq5UJhjmsgqYSKuO2osfpQeKAi/rO8kAaCwZoxSamo= github.com/miolini/datacounter v0.0.0-20190724021726-aa48df3a02c1/go.mod h1:P6fDJzlxN+cWYR09KbE9/ta+Y6JofX9tAUhJpWkWPaM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= @@ -259,6 +279,7 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -318,6 +339,7 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -354,6 +376,7 @@ github.com/volatiletech/sqlboiler v3.5.0+incompatible h1:n160O7UQLpZVRnJY6VH5eRN github.com/volatiletech/sqlboiler v3.5.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1 h1:pMEjRZ1M4ebWGikflH7nQpV6+Zr88KBMA2XJD3sbijw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -395,6 +418,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2eP golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914 h1:MlY3mEfbnWGmUi4rtHOtNnnnN4UJRGSyLPx+DXA5Sq4= +golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -406,6 +432,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -423,10 +450,13 @@ golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -447,6 +477,8 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/serverstats/schema.go b/serverstats/schema.go index 5ae9efb39a..0a0b2593fc 100644 --- a/serverstats/schema.go +++ b/serverstats/schema.go @@ -42,7 +42,6 @@ CREATE TABLE IF NOT EXISTS server_stats_member_periods ( UNIQUE(guild_id, created_at) );`, - `CREATE INDEX IF NOT EXISTS server_stats_member_periods_guild_idx on server_stats_member_periods(guild_id);`, `CREATE INDEX IF NOT EXISTS server_stats_member_periods_created_at_idx on server_stats_member_periods(created_at);`, } diff --git a/serverstats/serverstats.go b/serverstats/serverstats.go index 5503d1d41d..0e62b7adba 100644 --- a/serverstats/serverstats.go +++ b/serverstats/serverstats.go @@ -135,9 +135,7 @@ func RetrieveRedisStats(guildID int64) (*DailyStats, error) { online, err := botrest.GetOnlineCount(guildID) if err != nil { - if botrest.BotIsRunning() { - logger.WithError(err).Error("Failed fetching online count") - } + logger.WithError(err).Error("Failed fetching online count") } channelResult, err := parseMessageStats(messageStatsRaw, guildID) diff --git a/stdcommands/currentshard/currentshard.go b/stdcommands/currentshard/currentshard.go index db86445ad0..4dd4bf2b30 100644 --- a/stdcommands/currentshard/currentshard.go +++ b/stdcommands/currentshard/currentshard.go @@ -23,8 +23,8 @@ var Command = &commands.YAGCommand{ gID = data.Args[0].Int64() } - shard := bot.GuildShardID(gID) - totalShards := bot.GetTotalShards() + totalShards := bot.ShardManager.GetNumShards() + shard := bot.GuildShardID(int64(totalShards), gID) status := "" if bot.IsGuildOnCurrentProcess(gID) { diff --git a/web/handlers_general.go b/web/handlers_general.go index a7bff94c25..0e26d5661d 100644 --- a/web/handlers_general.go +++ b/web/handlers_general.go @@ -15,7 +15,6 @@ import ( "sync/atomic" "time" - "emperror.dev/errors" "github.com/jonas747/discordgo" "github.com/jonas747/retryableredis" "github.com/jonas747/yagpdb/bot/botrest" @@ -368,10 +367,6 @@ func HandleReconnectShard(w http.ResponseWriter, r *http.Request) (TemplateData, } func HandleChanenlPermissions(w http.ResponseWriter, r *http.Request) interface{} { - if !botrest.BotIsRunning() { - return errors.New("Bot is not responding") - } - g := r.Context().Value(common.ContextKeyCurrentGuild).(*discordgo.Guild) c, _ := strconv.ParseInt(pat.Param(r, "channel"), 10, 64) perms, err := botrest.GetChannelPermissions(g.ID, c) diff --git a/web/middleware.go b/web/middleware.go index 7ea4359874..b0e9cde212 100644 --- a/web/middleware.go +++ b/web/middleware.go @@ -80,7 +80,6 @@ func BaseTemplateDataMiddleware(inner http.Handler) http.Handler { // set up the base template data baseData := map[string]interface{}{ - "BotRunning": botrest.BotIsRunning(), "RequestURI": r.RequestURI, "StartedAtUnix": StartedAt.Unix(), "CurrentAd": CurrentAd, @@ -863,3 +862,18 @@ func SkipStaticMW(maybeSkip func(http.Handler) http.Handler, alwaysRunSuffixes . return http.HandlerFunc(mw) } } + +// RequireBotOwnerMW requires the user to be logged in and that they're a bot owner +func RequireBotOwnerMW(inner http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if user := r.Context().Value(common.ContextKeyUser); user != nil { + cast := user.(*discordgo.User) + if common.IsOwner(cast.ID) { + inner.ServeHTTP(w, r) + return + } + } + + w.WriteHeader(http.StatusUnauthorized) + }) +} diff --git a/web/web.go b/web/web.go index ec337c6105..a335e1b6a7 100644 --- a/web/web.go +++ b/web/web.go @@ -11,7 +11,6 @@ import ( "github.com/NYTimes/gziphandler" "github.com/jonas747/discordgo" - "github.com/jonas747/yagpdb/bot/botrest" "github.com/jonas747/yagpdb/common" "github.com/jonas747/yagpdb/common/config" "github.com/jonas747/yagpdb/common/patreon" @@ -119,6 +118,8 @@ func BaseURL() string { } func Run() { + common.ServiceTracker.RegisterService(common.ServiceTypeFrontend, "Webserver", "", nil) + common.RegisterPlugin(&ControlPanelPlugin{}) loadTemplates() @@ -139,7 +140,6 @@ func Run() { mux := setupRoutes() // Start monitoring the bot - go botrest.RunPinger() go pollCommandsRan() blogChannel := confAnnouncementsChannel.GetInt() @@ -147,13 +147,13 @@ func Run() { go discordblog.RunPoller(common.BotSession, int64(blogChannel), time.Minute) } - LoadAd() + loadAd() logger.Info("Running webservers") runServers(mux) } -func LoadAd() { +func loadAd() { path := confAdPath.GetString() linkurl := confAdLinkurl.GetString()