From b7ad118483bb7f9a9d3601190e35df8b20664504 Mon Sep 17 00:00:00 2001 From: Maxim Dietz Date: Fri, 15 Nov 2024 19:01:29 -0500 Subject: [PATCH] fix: Flakey TestPluginCleanup in CI Acquire lock for all Access List resources before locking individual Access List in `ListAccessListMembers`. What seemed to happen in CI was `ListAccessListMembers` was called, thereby locking the individual list. If IneligibilityReconciler ran simultaneously, it would lock all Access Lists, then fail to acquire the individual Access List's lock, leading to a deadlock. This didn't seem to come up in real-world conditions, but was causing flakey failures in CI when running many times at once. --- lib/services/local/access_list.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/services/local/access_list.go b/lib/services/local/access_list.go index 611d9de23f094..a092cf5e9823a 100644 --- a/lib/services/local/access_list.go +++ b/lib/services/local/access_list.go @@ -423,13 +423,15 @@ func (a *AccessListService) CountAccessListMembers(ctx context.Context, accessLi // ListAccessListMembers returns a paginated list of all access list members. func (a *AccessListService) ListAccessListMembers(ctx context.Context, accessListName string, pageSize int, nextToken string) ([]*accesslist.AccessListMember, string, error) { var members []*accesslist.AccessListMember - err := a.service.RunWhileLocked(ctx, lockName(accessListName), accessListLockTTL, func(ctx context.Context, _ backend.Backend) error { - _, err := a.service.GetResource(ctx, accessListName) - if err != nil { + err := a.service.RunWhileLocked(ctx, []string{accessListResourceLockName}, accessListLockTTL, func(ctx context.Context, _ backend.Backend) error { + return a.service.RunWhileLocked(ctx, lockName(accessListName), accessListLockTTL, func(ctx context.Context, _ backend.Backend) error { + _, err := a.service.GetResource(ctx, accessListName) + if err != nil { + return trace.Wrap(err) + } + members, nextToken, err = a.memberService.WithPrefix(accessListName).ListResources(ctx, pageSize, nextToken) return trace.Wrap(err) - } - members, nextToken, err = a.memberService.WithPrefix(accessListName).ListResources(ctx, pageSize, nextToken) - return trace.Wrap(err) + }) }) if err != nil { return nil, "", trace.Wrap(err)