Skip to content

Commit

Permalink
New state system (botlabs-gg#945)
Browse files Browse the repository at this point in the history 
* initial work on new state system

* state: intial version of new state system and basic embedded tracker

* progress

* more progress

* update go.mod

* add CacheSet replacement for GuildState.Userstate

This is a new package that provides a pretty simple cache
system based on "slots",

each cached type has it's own "slot" and managed all the
cached values inside it on its own

values inside a "slot" are keyed by interface{}
so the user can use whatever key type they want
but the most common one will probably be int64

* more progress moving to the new state and caching sytem

* continued work on migrating to the new state and cache system

* we compiled now bois

* update to latest dcmd and dstate

MemberState no longer contains roles and nickname, they're in the nested member object

this is to increase support for future discord api versions where presences are less usefull

* remove bot.GetMemberJoinedAt as it's not needed

I believe discord has fixed the issues where JoinedAt was not available
atleast im not able to find any such cases anymore

And with the roles and nick fields being moved to the
member fields for dstate.MemberState, all calls to bot.GetMember
requires the member field to be set to use the cached version.

* common: run CacheSet gc loop in goroutine

* bot: fix not setting proper state in setupState

* automodv2: fix panic in CheckTriggers

* tickets: fix build

* premium: fix deadlocking state

* update to latest dstate

* enabled shard migration, added state debug commands

* update to latest dshardorchestrator, go mod tidy

* accidentally included some compiled files

* add a couple more executables to gitignore

* memberfetcher: add results to state, properly set guildid

* bot: run state gc loop

* udpate to latest dstate

* templates: fixes and tweaks

* fix a bunch of warnings

* update to latest dcmd

* update to latest dcmd

* web: fixes for new state system

* botrest: use bot.GetMember to fallback to api

* commands: simplify a section

* templates: use user reference to keep backwards compat

* templates: fix referencing looping var

* pubsub: change EvictCacheSet to not require a guildID

* undelete: fix build
  • Loading branch information
@jogramming
jogramming authored Jun 3, 2021
1 parent 0355286 commit afe4ff8
Show file tree
Hide file tree
Showing 147 changed files with 2,166 additions and 2,232 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ cmd/yagpdb/shutdown_logs
cmd/yagpdb/safebrowsing_db
cmd/yagpdb/static/video/*

cmd/capturepanics/capturepanics
cmd/shardorchestrator/shardorchestrator

# We don't want filthy pre-configured config files here ! >:O
# The graw agent file
*.agent
Expand Down
86 changes: 50 additions & 36 deletions automod/automod_bot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"sort"

"github.com/jonas747/discordgo"
"github.com/jonas747/dstate/v2"
"github.com/jonas747/dstate/v3"
"github.com/jonas747/yagpdb/analytics"
"github.com/jonas747/yagpdb/automod/models"
"github.com/jonas747/yagpdb/bot"
Expand Down Expand Up @@ -50,15 +50,15 @@ func (p *Plugin) checkMessage(evt *eventsystem.EventData, msg *discordgo.Message
return true
}

cs := bot.State.Channel(true, msg.ChannelID)
if cs == nil || cs.Guild == nil {
cs := evt.GS.GetChannel(msg.ChannelID)
if cs == nil {
return true
}

ms := dstate.MSFromDGoMember(cs.Guild, msg.Member)
ms := dstate.MemberStateFromMember(msg.Member)

stripped := ""
return !p.CheckTriggers(nil, ms, msg, cs, func(trig *ParsedPart) (activated bool, err error) {
return !p.CheckTriggers(nil, evt.GS, ms, msg, cs, func(trig *ParsedPart) (activated bool, err error) {
if stripped == "" {
stripped = PrepareMessageForWordCheck(msg.Content)
}
Expand All @@ -68,7 +68,7 @@ func (p *Plugin) checkMessage(evt *eventsystem.EventData, msg *discordgo.Message
return
}

return cast.CheckMessage(ms, cs, msg, stripped, trig.ParsedSettings)
return cast.CheckMessage(&TriggerContext{GS: evt.GS, MS: ms, Data: trig.ParsedSettings}, cs, msg, stripped)
})
}

Expand All @@ -83,7 +83,7 @@ func (p *Plugin) checkViolationTriggers(ctxData *TriggeredRuleData, violationNam
return
}

rulesets, err := p.FetchGuildRulesets(ctxData.GS)
rulesets, err := p.FetchGuildRulesets(ctxData.GS.ID)
if err != nil {
logger.WithError(err).WithField("guild", ctxData.GS.ID).Error("failed fetching guild rulesets")
return
Expand All @@ -94,7 +94,7 @@ func (p *Plugin) checkViolationTriggers(ctxData *TriggeredRuleData, violationNam
}

// retrieve users violations
userViolations, err := models.AutomodViolations(qm.Where("guild_id = ? AND user_id = ? AND name = ?", ctxData.GS.ID, ctxData.MS.ID, violationName)).AllG(context.Background())
userViolations, err := models.AutomodViolations(qm.Where("guild_id = ? AND user_id = ? AND name = ?", ctxData.GS.ID, ctxData.MS.User.ID, violationName)).AllG(context.Background())
if err != nil {
logger.WithError(err).Error("automod failed retrieving user violations")
return
Expand Down Expand Up @@ -195,8 +195,8 @@ func (p *Plugin) checkViolationTriggers(ctxData *TriggeredRuleData, violationNam
func (p *Plugin) handleGuildMemberUpdate(evt *eventsystem.EventData) {
evtData := evt.GuildMemberUpdate()

ms := dstate.MSFromDGoMember(evt.GS, evtData.Member)
if ms.Nick == "" {
ms := dstate.MemberStateFromMember(evtData.Member)
if ms.Member.Nick == "" {
return
}

Expand All @@ -206,51 +206,67 @@ func (p *Plugin) handleGuildMemberUpdate(evt *eventsystem.EventData) {
func (p *Plugin) handleGuildMemberJoin(evt *eventsystem.EventData) {
evtData := evt.GuildMemberAdd()

ms := dstate.MSFromDGoMember(evt.GS, evtData.Member)
ms := dstate.MemberStateFromMember(evtData.Member)

p.checkJoin(ms)
p.checkUsername(ms)
}

func (p *Plugin) checkNickname(ms *dstate.MemberState) {
p.CheckTriggers(nil, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
gs := bot.State.GetGuild(ms.GuildID)
if gs == nil {
return
}

p.CheckTriggers(nil, gs, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
cast, ok := trig.Part.(NicknameListener)
if !ok {
return false, nil
}

return cast.CheckNickname(ms, trig.ParsedSettings)
return cast.CheckNickname(&TriggerContext{GS: gs, MS: ms, Data: trig.ParsedSettings})
})
}

func (p *Plugin) checkUsername(ms *dstate.MemberState) {
p.CheckTriggers(nil, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
gs := bot.State.GetGuild(ms.GuildID)
if gs == nil {
return
}

p.CheckTriggers(nil, gs, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
cast, ok := trig.Part.(UsernameListener)
if !ok {
return false, nil
}

return cast.CheckUsername(ms, trig.ParsedSettings)
return cast.CheckUsername(&TriggerContext{GS: gs, MS: ms, Data: trig.ParsedSettings})
})
}

func (p *Plugin) checkJoin(ms *dstate.MemberState) {
p.CheckTriggers(nil, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
gs := bot.State.GetGuild(ms.GuildID)
if gs == nil {
return
}

p.CheckTriggers(nil, gs, ms, nil, nil, func(trig *ParsedPart) (activated bool, err error) {
cast, ok := trig.Part.(JoinListener)
if !ok {
return false, nil
}

return cast.CheckJoin(ms, trig.ParsedSettings)
return cast.CheckJoin(&TriggerContext{GS: gs, MS: ms, Data: trig.ParsedSettings})
})
}

func (p *Plugin) CheckTriggers(rulesets []*ParsedRuleset, ms *dstate.MemberState, msg *discordgo.Message, cs *dstate.ChannelState, checkF func(trp *ParsedPart) (activated bool, err error)) bool {
func (p *Plugin) CheckTriggers(rulesets []*ParsedRuleset, gs *dstate.GuildSet, ms *dstate.MemberState, msg *discordgo.Message, cs *dstate.ChannelState, checkF func(trp *ParsedPart) (activated bool, err error)) bool {

if rulesets == nil {
var err error
rulesets, err = p.FetchGuildRulesets(ms.Guild)
rulesets, err = p.FetchGuildRulesets(gs.ID)
if err != nil {
logger.WithError(err).WithField("guild", ms.Guild.ID).Error("failed fetching triggers")
logger.WithError(err).WithField("guild", msg.GuildID).Error("failed fetching triggers")
return false
}

Expand All @@ -269,7 +285,7 @@ func (p *Plugin) CheckTriggers(rulesets []*ParsedRuleset, ms *dstate.MemberState
ctxData := &TriggeredRuleData{
MS: ms,
CS: cs,
GS: ms.Guild,
GS: gs,
Plugin: p,
Ruleset: rs,

Expand Down Expand Up @@ -409,9 +425,7 @@ func (p *Plugin) RulesetRulesTriggeredCondsPassed(ruleset *ParsedRuleset, trigge
cid := int64(0)

if ctxData.CS != nil {
ctxData.CS.Owner.RLock()
cname = ctxData.CS.Name
ctxData.CS.Owner.RUnlock()
cid = ctxData.CS.ID
}

Expand Down Expand Up @@ -444,8 +458,8 @@ func (p *Plugin) RulesetRulesTriggeredCondsPassed(ruleset *ParsedRuleset, trigge
RuleID: null.Int64{Int64: rule.Model.ID, Valid: true},
RuleName: rule.Model.Name,
RulesetName: rule.Model.R.Ruleset.Name,
UserID: ctxData.MS.ID,
UserName: ctxData.MS.Username + "#" + ctxData.MS.StrDiscriminator(),
UserID: ctxData.MS.User.ID,
UserName: ctxData.MS.User.Username + "#" + ctxData.MS.User.Discriminator,
Extradata: serializedExtraData,
}
}
Expand All @@ -471,14 +485,14 @@ func (p *Plugin) RulesetRulesTriggeredCondsPassed(ruleset *ParsedRuleset, trigge
}
}

const (
CacheKeyRulesets bot.GSCacheKey = "automod_2_rulesets"
CacheKeyLists bot.GSCacheKey = "automod_2_lists"
var (
cachedRulesets = common.CacheSet.RegisterSlot("amod2_rulesets", nil, int64(0))
cachedLists = common.CacheSet.RegisterSlot("amod2_lists", nil, int64(0))
)

func (p *Plugin) FetchGuildRulesets(gs *dstate.GuildState) ([]*ParsedRuleset, error) {
v, err := gs.UserCacheFetch(CacheKeyRulesets, func() (interface{}, error) {
rulesets, err := models.AutomodRulesets(qm.Where("guild_id=?", gs.ID),
func (p *Plugin) FetchGuildRulesets(guildID int64) ([]*ParsedRuleset, error) {
v, err := cachedRulesets.GetCustomFetch(guildID, func(key interface{}) (interface{}, error) {
rulesets, err := models.AutomodRulesets(qm.Where("guild_id=?", guildID),
qm.Load("RulesetAutomodRules.RuleAutomodRuleData"), qm.Load("RulesetAutomodRulesetConditions")).AllG(context.Background())

if err != nil {
Expand All @@ -505,9 +519,9 @@ func (p *Plugin) FetchGuildRulesets(gs *dstate.GuildState) ([]*ParsedRuleset, er
return cast, nil
}

func FetchGuildLists(gs *dstate.GuildState) ([]*models.AutomodList, error) {
v, err := gs.UserCacheFetch(CacheKeyLists, func() (interface{}, error) {
lists, err := models.AutomodLists(qm.Where("guild_id = ?", gs.ID)).AllG(context.Background())
func FetchGuildLists(guildID int64) ([]*models.AutomodList, error) {
v, err := cachedLists.GetCustomFetch(guildID, func(key interface{}) (interface{}, error) {
lists, err := models.AutomodLists(qm.Where("guild_id = ?", guildID)).AllG(context.Background())
if err != nil {
return nil, err
}
Expand All @@ -525,8 +539,8 @@ func FetchGuildLists(gs *dstate.GuildState) ([]*models.AutomodList, error) {

var ErrListNotFound = errors.New("list not found")

func FindFetchGuildList(gs *dstate.GuildState, listID int64) (*models.AutomodList, error) {
lists, err := FetchGuildLists(gs)
func FindFetchGuildList(guildID int64, listID int64) (*models.AutomodList, error) {
lists, err := FetchGuildLists(guildID)
if err != nil {
return nil, err
}
Expand Down
20 changes: 10 additions & 10 deletions automod/automod_web.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ import (

"github.com/fatih/structs"
"github.com/gorilla/schema"
"github.com/jonas747/discordgo"
"github.com/jonas747/dstate/v3"
"github.com/jonas747/yagpdb/automod/models"
"github.com/jonas747/yagpdb/bot"
"github.com/jonas747/yagpdb/common"
"github.com/jonas747/yagpdb/common/cplogs"
"github.com/jonas747/yagpdb/common/featureflags"
"github.com/jonas747/yagpdb/common/pubsub"
"github.com/jonas747/yagpdb/web"
"github.com/volatiletech/sqlboiler/boil"
"github.com/volatiletech/sqlboiler/queries/qm"
Expand Down Expand Up @@ -205,7 +205,7 @@ func (p *Plugin) handlePostAutomodCreateList(w http.ResponseWriter, r *http.Requ

err = list.InsertG(r.Context(), boil.Infer())
if err == nil {
bot.EvictGSCache(g.ID, CacheKeyLists)
pubsub.EvictCacheSet(cachedLists, g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyNewList))
}
return tmpl, err
Expand All @@ -228,7 +228,7 @@ func (p *Plugin) handlePostAutomodUpdateList(w http.ResponseWriter, r *http.Requ
list.Content = strings.Fields(data.Content)
_, err = list.UpdateG(r.Context(), boil.Whitelist("content"))
if err == nil {
bot.EvictGSCache(g.ID, CacheKeyLists)
pubsub.EvictCacheSet(cachedLists, g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyUpdatedList))
}
return tmpl, err
Expand All @@ -245,7 +245,7 @@ func (p *Plugin) handlePostAutomodDeleteList(w http.ResponseWriter, r *http.Requ

_, err = list.DeleteG(r.Context())
if err == nil {
bot.EvictGSCache(g.ID, CacheKeyLists)
pubsub.EvictCacheSet(cachedLists, g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyRemovedList))
}
return tmpl, err
Expand Down Expand Up @@ -390,7 +390,7 @@ func (p *Plugin) handlePostAutomodUpdateRuleset(w http.ResponseWriter, r *http.R
return tmpl, err
}

bot.EvictGSCache(g.ID, CacheKeyRulesets)
pubsub.EvictCacheSet(cachedRulesets, g.ID)
featureflags.MarkGuildDirty(g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyUpdatedRuleset))

Expand All @@ -413,7 +413,7 @@ func (p *Plugin) handlePostAutomodDeleteRuleset(w http.ResponseWriter, r *http.R
delete(tmpl, "CurrentRuleset")

if rows > 0 {
bot.EvictGSCache(g.ID, CacheKeyRulesets)
pubsub.EvictCacheSet(cachedRulesets, g.ID)
featureflags.MarkGuildDirty(g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyRemovedRuleset))
}
Expand Down Expand Up @@ -523,7 +523,7 @@ func (p *Plugin) handlePostAutomodUpdateRule(w http.ResponseWriter, r *http.Requ

WebLoadRuleSettings(r, tmpl, ruleSet)

bot.EvictGSCache(g.ID, CacheKeyRulesets)
pubsub.EvictCacheSet(cachedRulesets, g.ID)
featureflags.MarkGuildDirty(g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyUpdatedRule))

Expand Down Expand Up @@ -599,7 +599,7 @@ func CheckLimits(exec boil.ContextExecutor, rule *models.AutomodRule, tmpl web.T
return
}

func ReadRuleRowData(guild *discordgo.Guild, tmpl web.TemplateData, rawData []RuleRowData, form url.Values, namePrefix string) (result []*models.AutomodRuleDatum, validationOK bool, err error) {
func ReadRuleRowData(guild *dstate.GuildSet, tmpl web.TemplateData, rawData []RuleRowData, form url.Values, namePrefix string) (result []*models.AutomodRuleDatum, validationOK bool, err error) {
parsedSettings := make([]*ParsedPart, 0, len(rawData))

for i, entry := range rawData {
Expand Down Expand Up @@ -724,7 +724,7 @@ func (p *Plugin) handlePostAutomodDeleteRule(w http.ResponseWriter, r *http.Requ
_, err := v.DeleteG(r.Context())
if err == nil {
ruleset.R.RulesetAutomodRules = append(ruleset.R.RulesetAutomodRules[:k], ruleset.R.RulesetAutomodRules[k+1:]...)
bot.EvictGSCache(g.ID, CacheKeyRulesets)
pubsub.EvictCacheSet(cachedRulesets, g.ID)
featureflags.MarkGuildDirty(g.ID)
go cplogs.RetryAddEntry(web.NewLogEntryFromContext(r.Context(), panelLogKeyRemovedRule))
}
Expand Down
10 changes: 5 additions & 5 deletions automod/commands.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ import (
"strings"
"time"

"github.com/jonas747/dcmd/v2"
"github.com/jonas747/dcmd/v3"
"github.com/jonas747/discordgo"
"github.com/jonas747/dstate/v2"
"github.com/jonas747/dstate/v3"
"github.com/jonas747/yagpdb/automod/models"
"github.com/jonas747/yagpdb/bot/paginatedmessages"
"github.com/jonas747/yagpdb/commands"
Expand Down Expand Up @@ -43,8 +43,8 @@ func (p *Plugin) AddCommands() {
return nil, err
}

data.GuildData.GS.UserCacheDel(CacheKeyRulesets)
data.GuildData.GS.UserCacheDel(CacheKeyLists)
cachedRulesets.Delete(data.GuildData.GS.ID)
cachedLists.Delete(data.GuildData.GS.ID)
featureflags.MarkGuildDirty(data.GuildData.GS.ID)

enabledStr := "enabled"
Expand Down Expand Up @@ -376,7 +376,7 @@ func (p *Plugin) AddCommands() {
container.AddCommand(cmdListVLC, cmdListVLC.GetTrigger())
container.AddCommand(cmdDelV, cmdDelV.GetTrigger())
container.AddCommand(cmdClearV, cmdClearV.GetTrigger())
commands.RegisterSlashCommandsContainer(container, false, func(gs *dstate.GuildState) ([]int64, error) {
commands.RegisterSlashCommandsContainer(container, false, func(gs *dstate.GuildSet) ([]int64, error) {
return nil, nil
})
}
Loading

0 comments on commit afe4ff8

Please sign in to comment.