Skip to content

Commit

Permalink
Merge pull request #94 from underdog-tech/jose/make-gh-linkable
Browse files Browse the repository at this point in the history
feat: Make GH repos linkable and ignore `internal` repos
  • Loading branch information
JoseAngel1196 authored Feb 23, 2024
2 parents 2263c6c + 6af554d commit 1d93495
Show file tree
Hide file tree
Showing 6 changed files with 153 additions and 120 deletions.
155 changes: 52 additions & 103 deletions querying/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,19 @@ package querying

import (
"context"
"strings"
"sync"

"golang.org/x/oauth2"

"github.com/rs/zerolog"
"github.com/shurcooL/githubv4"
"github.com/underdog-tech/vulnbot/configs"
"github.com/underdog-tech/vulnbot/logger"
)

const DisableVulnBotTopicKeyword = "disable-vulnbot"

type githubClient interface {
Query(context.Context, interface{}, map[string]interface{}) error
}
Expand Down Expand Up @@ -38,55 +42,6 @@ func NewGithubDataSource(conf *configs.Config) GithubDataSource {
}
}

type githubVulnerability struct {
SecurityAdvisory struct {
Description string
Identifiers []struct {
Type string
Value string
}
}
SecurityVulnerability struct {
Severity string
Package struct {
Ecosystem string
Name string
}
}
}

type orgRepo struct {
Name string
Url string
VulnerabilityAlerts struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []githubVulnerability
} `graphql:"vulnerabilityAlerts(states: OPEN, first: 100, after: $alertCursor)"`
}

type repositoryQuery struct {
Repository orgRepo `graphql:"repository(name: $repoName, owner: $orgName)"`
}

type orgVulnerabilityQuery struct {
Organization struct {
Name string
Login string
Repositories struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []orgRepo
} `graphql:"repositories(orderBy: {field: NAME, direction: ASC}, isFork: false, isArchived: false, first: 100, after: $repoCursor)"`
} `graphql:"organization(login: $login)"`
}

// Ref: https://docs.github.com/en/graphql/reference/enums#securityadvisoryecosystem
var githubEcosystems = map[string]configs.FindingEcosystemType{
"ACTIONS": configs.FindingEcosystemGHA,
Expand Down Expand Up @@ -150,7 +105,7 @@ func (gh *GithubDataSource) processRepoFindings(projects *ProjectCollection, rep

// Link directly to Dependabot findings.
// There doesn't appear to be a GraphQL property for this link.
project.Links["GitHub"] = repo.Url + "/security/dependabot"
project.Link = repo.Url + "/security/dependabot"

log.Debug().Str("project", project.Name).Msg("Processing findings for project.")

Expand Down Expand Up @@ -198,38 +153,6 @@ func (gh *GithubDataSource) processRepoFindings(projects *ProjectCollection, rep
return nil
}

type orgTeam struct {
Name string
Slug string
Repositories struct {
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Edges []struct {
Permission string
Node struct {
Name string
IsFork bool
IsArchived bool
}
}
} `graphql:"repositories(orderBy: {field: NAME, direction: ASC}, first: 100, after: $repoCursor)"`
}

type orgRepoOwnerQuery struct {
Organization struct {
Teams struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []orgTeam
} `graphql:"teams(orderBy: {field: NAME, direction: ASC}, first: 100, after: $teamCursor)"`
} `graphql:"organization(login: $login)"`
}

func (gh *GithubDataSource) gatherRepoOwners(projects *ProjectCollection) {
var ownerQuery orgRepoOwnerQuery
log := logger.Get()
Expand All @@ -242,33 +165,59 @@ func (gh *GithubDataSource) gatherRepoOwners(projects *ProjectCollection) {

for {
log.Info().Msg("Querying GitHub API for repository ownership information.")
if err := gh.GhClient.Query(gh.ctx, &ownerQuery, queryVars); err != nil {
if err := gh.queryRepoOwners(&ownerQuery, queryVars); err != nil {
log.Fatal().Err(err).Msg("Failed to query GitHub for repository ownership.")
}
for _, team := range ownerQuery.Organization.Teams.Nodes {
teamConfig, err := configs.GetTeamConfigBySlug(team.Slug, gh.conf.Team)
if err != nil {
log.Warn().Err(err).Str("slug", team.Slug).Msg("Failed to load team from configs.")

gh.processRepoOwners(&ownerQuery, projects, log)
if !ownerQuery.Organization.Teams.PageInfo.HasNextPage {
break
}
queryVars["teamCursor"] = githubv4.NewString(ownerQuery.Organization.Teams.PageInfo.EndCursor)
}
}

func (gh *GithubDataSource) queryRepoOwners(ownerQuery *orgRepoOwnerQuery, queryVars map[string]interface{}) error {
if err := gh.GhClient.Query(gh.ctx, ownerQuery, queryVars); err != nil {
return err
}
return nil
}

func (gh *GithubDataSource) processRepoOwners(ownerQuery *orgRepoOwnerQuery, projects *ProjectCollection, log zerolog.Logger) {
for _, team := range ownerQuery.Organization.Teams.Nodes {
teamConfig, err := configs.GetTeamConfigBySlug(team.Slug, gh.conf.Team)
if err != nil {
log.Warn().Err(err).Str("slug", team.Slug).Msg("Failed to load team from configs.")
continue
}
for _, repo := range team.Repositories.Edges {
shouldIgnoreRepo := repo.Node.IsArchived || repo.Node.IsFork || hasDisableVulnbotTopic(repo.Node.RepositoryTopics)
if shouldIgnoreRepo {
log.Debug().
Str("Repo", repo.Node.Name).
Bool("IsFork", repo.Node.IsFork).
Bool("IsArchived", repo.Node.IsArchived).
Msg("Skipping untracked repository.")
continue
}
// TODO: Handle pagination of repositories owned by a team
for _, repo := range team.Repositories.Edges {
if repo.Node.IsArchived || repo.Node.IsFork {
log.Debug().Str("Repo", repo.Node.Name).Bool("IsFork", repo.Node.IsFork).Bool("IsArchived", repo.Node.IsArchived).Msg("Skipping untracked repository.")
continue
}
switch repo.Permission {
case "ADMIN", "MAINTAIN":
project := projects.GetProject(repo.Node.Name)
project.Owners.Add(teamConfig)
default:
continue
}
switch repo.Permission {
case "ADMIN", "MAINTAIN":
project := projects.GetProject(repo.Node.Name)
project.Owners.Add(teamConfig)
default:
continue
}
}
if !ownerQuery.Organization.Teams.PageInfo.HasNextPage {
break
}
}

// Function to check if the repository has "disable-vulnbot" in its topics
func hasDisableVulnbotTopic(repoTopics repositoryTopics) bool {
for _, edge := range repoTopics.Edges {
if strings.Contains(strings.ToLower(edge.Node.Topic.Name), DisableVulnBotTopicKeyword) {
return true
}
queryVars["teamCursor"] = githubv4.NewString(ownerQuery.Organization.Teams.PageInfo.EndCursor)
}
return false
}
4 changes: 1 addition & 3 deletions querying/github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@ func getTestProject() querying.ProjectCollection {
Projects: []*querying.Project{
{
Name: "zaphod",
Links: map[string]string{
"GitHub": "https://heart-of-gold/zaphod/security/dependabot",
},
Link: "https://heart-of-gold/zaphod/security/dependabot",
Findings: []*querying.Finding{
{
Ecosystem: configs.FindingEcosystemGo,
Expand Down
3 changes: 1 addition & 2 deletions querying/project.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type ProjectCollection struct {
type Project struct {
Name string
Findings []*Finding
Links map[string]string
Link string
Owners mapset.Set[configs.TeamConfig]
mu sync.Mutex
}
Expand All @@ -38,7 +38,6 @@ func NewProject(name string) *Project {
return &Project{
Name: name,
Findings: []*Finding{},
Links: map[string]string{},
Owners: mapset.NewSet[configs.TeamConfig](),
}
}
Expand Down
95 changes: 95 additions & 0 deletions querying/queries.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package querying

import "github.com/shurcooL/githubv4"

type githubVulnerability struct {
SecurityAdvisory struct {
Description string
Identifiers []struct {
Type string
Value string
}
}
SecurityVulnerability struct {
Severity string
Package struct {
Ecosystem string
Name string
}
}
}

type repositoryTopics struct {
Edges []struct {
Node struct {
Topic struct {
Name string
}
}
}
}

type orgRepo struct {
Name string
Url string
VulnerabilityAlerts struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []githubVulnerability
} `graphql:"vulnerabilityAlerts(states: OPEN, first: 100, after: $alertCursor)"`
}

type repositoryQuery struct {
Repository orgRepo `graphql:"repository(name: $repoName, owner: $orgName)"`
}

type orgVulnerabilityQuery struct {
Organization struct {
Name string
Login string
Repositories struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []orgRepo
} `graphql:"repositories(orderBy: {field: NAME, direction: ASC}, isFork: false, isArchived: false, first: 100, after: $repoCursor)"`
} `graphql:"organization(login: $login)"`
}

type orgTeam struct {
Name string
Slug string
Repositories struct {
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Edges []struct {
Permission string
Node struct {
Name string
IsFork bool
IsArchived bool
RepositoryTopics repositoryTopics `graphql:"repositoryTopics(first: 10, last: null)"`
}
}
} `graphql:"repositories(orderBy: {field: NAME, direction: ASC}, first: 100, after: $repoCursor)"`
}

type orgRepoOwnerQuery struct {
Organization struct {
Teams struct {
TotalCount int
PageInfo struct {
EndCursor githubv4.String
HasNextPage bool
}
Nodes []orgTeam
} `graphql:"teams(orderBy: {field: NAME, direction: ASC}, first: 100, after: $teamCursor)"`
} `graphql:"organization(login: $login)"`
}
10 changes: 2 additions & 8 deletions reporting/slack.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,8 @@ func (s *SlackReporter) BuildTeamRepositoryReport(
if severityIcon == "" {
severityIcon = configs.GetIconForSeverity(configs.FindingSeverityUndefined, s.Config.Severity)
}
projLinks := make([]string, 0)
for title, link := range repoReport.Project.Links {
projLinks = append(projLinks, fmt.Sprintf("[<%s|%s>]", link, title))
}
projName := fmt.Sprintf("%s *%s*", severityIcon, repoReport.Project.Name)
if len(projLinks) > 0 {
projName = fmt.Sprintf("%s · %s", projName, strings.Join(projLinks, " "))
}

projName := fmt.Sprintf("%s *<%s|%s>*", severityIcon, repoReport.Project.Link, repoReport.Project.Name)
fields := []*slack.TextBlockObject{
slack.NewTextBlockObject(slack.MarkdownType, projName, false, false),
slack.NewTextBlockObject(slack.MarkdownType, strings.Join(vulnCounts, " | "), false, false),
Expand Down
6 changes: 2 additions & 4 deletions reporting/slack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,7 @@ func TestSendSlackSummaryReportSendsSingleMessage(t *testing.T) {
func TestBuildSlackTeamRepositoryReport(t *testing.T) {
reporter := reporting.SlackReporter{Config: &configs.Config{}}
proj := querying.NewProject("foo")
proj.Links = map[string]string{
"GitHub": "https://github.com/bar/foo",
}
proj.Link = "https://github.com/bar/foo"
report := reporting.NewProjectFindingSummary(proj)
report.VulnsByEcosystem[configs.FindingEcosystemPython] = 15
report.VulnsBySeverity[configs.FindingSeverityCritical] = 2
Expand All @@ -319,7 +317,7 @@ func TestBuildSlackTeamRepositoryReport(t *testing.T) {
"fields": []map[string]interface{}{
{
"type": "mrkdwn",
"text": " *foo* · [<https://github.com/bar/foo|GitHub>]",
"text": " *<https://github.com/bar/foo|foo>*",
},
{
"type": "mrkdwn",
Expand Down

0 comments on commit 1d93495

Please sign in to comment.