Skip to content

Commit

Permalink
ID Generation (#428)
Browse files Browse the repository at this point in the history 
Signed-off-by: Frank Jogeleit <frank.jogeleit@lovoo.com>
  • Loading branch information
@fjogeleit
fjogeleit authored May 2, 2024
1 parent 1edff60 commit dd150ee
Show file tree
Hide file tree
Showing 20 changed files with 716 additions and 130 deletions.
6 changes: 6 additions & 0 deletions charts/policy-reporter/config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,12 @@ redis:
{{- toYaml . | nindent 2 }}
{{- end }}

{{- with .Values.sourceConfig }}
sourceConfig:
{{- toYaml . | nindent 2 }}
{{- end }}


logging:
encoding: {{ .Values.logging.encoding }}
logLevel: {{ include "policyreporter.logLevel" . }}
Expand Down
7 changes: 7 additions & 0 deletions charts/policy-reporter/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ reportFilter:
# Disable the processing of ClusterPolicyReports
disabled: false

# customize source specific logic like result ID generation
sourceConfig: {}
# sourcename:
# customID:
# enabled: true
# fields: ["resource", "policy", "rule", "category", "result", "message"]

# enable policy-report-ui
ui:
enabled: false
Expand Down
64 changes: 37 additions & 27 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,34 +347,44 @@ type Database struct {
MountedSecret string `mapstructure:"mountedSecret"`
}

type CustomID struct {
Enabled bool `mapstructure:"enabled"`
Fields []string `mapstructure:"fields"`
}

type SourceConfig struct {
CustomID `mapstructure:"customID"`
}

// Config of the PolicyReporter
type Config struct {
Version string
Namespace string `mapstructure:"namespace"`
Loki *Loki `mapstructure:"loki"`
Elasticsearch *Elasticsearch `mapstructure:"elasticsearch"`
Slack *Slack `mapstructure:"slack"`
Discord *Discord `mapstructure:"discord"`
Teams *Teams `mapstructure:"teams"`
S3 *S3 `mapstructure:"s3"`
Kinesis *Kinesis `mapstructure:"kinesis"`
SecurityHub *SecurityHub `mapstructure:"securityHub"`
GCS *GCS `mapstructure:"gcs"`
UI *UI `mapstructure:"ui"`
Webhook *Webhook `mapstructure:"webhook"`
Telegram *Telegram `mapstructure:"telegram"`
GoogleChat *GoogleChat `mapstructure:"googleChat"`
API API `mapstructure:"api"`
WorkerCount int `mapstructure:"worker"`
DBFile string `mapstructure:"dbfile"`
Metrics Metrics `mapstructure:"metrics"`
REST REST `mapstructure:"rest"`
ReportFilter ReportFilter `mapstructure:"reportFilter"`
Redis Redis `mapstructure:"redis"`
Profiling Profiling `mapstructure:"profiling"`
EmailReports EmailReports `mapstructure:"emailReports"`
LeaderElection LeaderElection `mapstructure:"leaderElection"`
K8sClient K8sClient `mapstructure:"k8sClient"`
Logging Logging `mapstructure:"logging"`
Database Database `mapstructure:"database"`
Namespace string `mapstructure:"namespace"`
Loki *Loki `mapstructure:"loki"`
Elasticsearch *Elasticsearch `mapstructure:"elasticsearch"`
Slack *Slack `mapstructure:"slack"`
Discord *Discord `mapstructure:"discord"`
Teams *Teams `mapstructure:"teams"`
S3 *S3 `mapstructure:"s3"`
Kinesis *Kinesis `mapstructure:"kinesis"`
SecurityHub *SecurityHub `mapstructure:"securityHub"`
GCS *GCS `mapstructure:"gcs"`
UI *UI `mapstructure:"ui"`
Webhook *Webhook `mapstructure:"webhook"`
Telegram *Telegram `mapstructure:"telegram"`
GoogleChat *GoogleChat `mapstructure:"googleChat"`
API API `mapstructure:"api"`
WorkerCount int `mapstructure:"worker"`
DBFile string `mapstructure:"dbfile"`
Metrics Metrics `mapstructure:"metrics"`
REST REST `mapstructure:"rest"`
ReportFilter ReportFilter `mapstructure:"reportFilter"`
Redis Redis `mapstructure:"redis"`
Profiling Profiling `mapstructure:"profiling"`
EmailReports EmailReports `mapstructure:"emailReports"`
LeaderElection LeaderElection `mapstructure:"leaderElection"`
K8sClient K8sClient `mapstructure:"k8sClient"`
Logging Logging `mapstructure:"logging"`
Database Database `mapstructure:"database"`
SourceConfig map[string]SourceConfig `mapstructure:"sourceConfig"`
}
16 changes: 16 additions & 0 deletions pkg/config/resolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"crypto/tls"
"crypto/x509"
"os"
"strings"
"time"

goredis "github.com/go-redis/redis/v8"
Expand Down Expand Up @@ -33,6 +34,7 @@ import (
"github.com/kyverno/policy-reporter/pkg/listener"
"github.com/kyverno/policy-reporter/pkg/listener/metrics"
"github.com/kyverno/policy-reporter/pkg/report"
"github.com/kyverno/policy-reporter/pkg/report/result"
"github.com/kyverno/policy-reporter/pkg/target"
"github.com/kyverno/policy-reporter/pkg/validate"
)
Expand Down Expand Up @@ -173,6 +175,19 @@ func (r *Resolver) EventPublisher() report.EventPublisher {
return r.publisher
}

func (r *Resolver) CustomIDGenerators() map[string]result.IDGenerator {
generators := make(map[string]result.IDGenerator)
for s, c := range r.config.SourceConfig {
if !c.Enabled || len(c.Fields) == 0 {
continue
}

generators[strings.ToLower(s)] = result.NewIDGenerator(c.Fields)
}

return generators
}

// EventPublisher resolver method
func (r *Resolver) Queue() (*kubernetes.Queue, error) {
client, err := r.CRDClient()
Expand All @@ -184,6 +199,7 @@ func (r *Resolver) Queue() (*kubernetes.Queue, error) {
kubernetes.NewDebouncer(1*time.Minute, r.EventPublisher()),
workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "report-queue"),
client,
result.NewReconditioner(r.CustomIDGenerators()),
), nil
}

Expand Down
18 changes: 18 additions & 0 deletions pkg/config/resolver_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,15 @@ var testConfig = &config.Config{
Webhook: "http://localhost:900/webhook",
Channels: []*config.GoogleChat{{}},
},
SourceConfig: map[string]config.SourceConfig{
"test": {
CustomID: config.CustomID{
Enabled: true,
Fields: []string{"resource"},
},
},
"default": {},
},
}

func Test_ResolveTargets(t *testing.T) {
Expand Down Expand Up @@ -580,3 +589,12 @@ func Test_ResolveEnableLeaderElection(t *testing.T) {
}
})
}

func Test_ResolveCustomIDGenerators(t *testing.T) {
resolver := config.NewResolver(testConfig, nil)

generators := resolver.CustomIDGenerators()
if len(generators) != 1 {
t.Error("only enabled custom id config should be mapped")
}
}
28 changes: 0 additions & 28 deletions pkg/crd/api/policyreport/v1alpha2/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ package v1alpha2
import (
"bytes"
"fmt"
"strconv"
"strings"

"github.com/segmentio/fasthash/fnv1a"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
Expand Down Expand Up @@ -243,32 +241,6 @@ func (r *PolicyReportResult) GetKind() string {
}

func (r *PolicyReportResult) GetID() string {
if r.ID != "" {
return r.ID
}

if id, ok := r.Properties[ResultIDKey]; ok {
r.ID = id

return r.ID
}

h1 := fnv1a.Init64

res := r.GetResource()
if res != nil {
h1 = fnv1a.AddString64(h1, res.Name)
h1 = fnv1a.AddString64(h1, string(res.UID))
}

h1 = fnv1a.AddString64(h1, r.Policy)
h1 = fnv1a.AddString64(h1, r.Rule)
h1 = fnv1a.AddString64(h1, string(r.Result))
h1 = fnv1a.AddString64(h1, r.Category)
h1 = fnv1a.AddString64(h1, r.Message)

r.ID = strconv.FormatUint(h1, 10)

return r.ID
}

Expand Down
22 changes: 6 additions & 16 deletions pkg/crd/api/policyreport/v1alpha2/common_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"testing"

"github.com/kyverno/policy-reporter/pkg/crd/api/policyreport/v1alpha2"
"github.com/kyverno/policy-reporter/pkg/report/result"
corev1 "k8s.io/api/core/v1"
)

Expand Down Expand Up @@ -98,7 +99,7 @@ func TestCommon(t *testing.T) {
})
}

func TestPolicyReportResul(t *testing.T) {
func TestPolicyReportResult(t *testing.T) {
t.Run("GetResource Without Resources", func(t *testing.T) {
r := &v1alpha2.PolicyReportResult{}

Expand Down Expand Up @@ -128,31 +129,20 @@ func TestPolicyReportResul(t *testing.T) {
}
})
t.Run("GetID from Result With Resource", func(t *testing.T) {
r := &v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Kind: "Pod"}}}
r := v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Kind: "Pod"}}}
r.ID = result.NewIDGenerator(nil).Generate(&v1alpha2.PolicyReport{}, r)

if r.GetID() != "18007334074686647077" {
t.Errorf("expected result kind to be '18007334074686647077', got :%s", r.GetID())
}
})
t.Run("GetID from Result With ID Property", func(t *testing.T) {
r := &v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Kind: "Pod"}}, Properties: map[string]string{"resultID": "result-id"}}

if r.GetID() != "result-id" {
t.Errorf("expected result kind to be 'result-id', got :%s", r.GetID())
}
})
t.Run("GetID cached", func(t *testing.T) {
r := &v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Kind: "Pod"}}, Properties: map[string]string{"resultID": "result-id"}}
r := v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Kind: "Pod"}}, Properties: map[string]string{"resultID": "result-id"}}
r.ID = result.NewIDGenerator(nil).Generate(&v1alpha2.PolicyReport{}, r)

if r.GetID() != "result-id" {
t.Errorf("expected result kind to be 'result-id', got :%s", r.GetID())
}

r.Properties["resultID"] = "test"

if r.GetID() != "result-id" {
t.Errorf("expected result ID doesn't change, got :%s", r.GetID())
}
})
t.Run("ToResourceString with Namespace and Kind", func(t *testing.T) {
r := &v1alpha2.PolicyReportResult{Resources: []corev1.ObjectReference{{Name: "test", Namespace: "default", Kind: "Pod"}}}
Expand Down
33 changes: 14 additions & 19 deletions pkg/database/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import (

"github.com/segmentio/fasthash/fnv1a"
"github.com/uptrace/bun"
corev1 "k8s.io/api/core/v1"

api "github.com/kyverno/policy-reporter/pkg/api/v1"
"github.com/kyverno/policy-reporter/pkg/crd/api/policyreport/v1alpha2"
"github.com/kyverno/policy-reporter/pkg/report"
"github.com/kyverno/policy-reporter/pkg/report/result"
)

type Config struct {
Expand Down Expand Up @@ -109,21 +109,16 @@ func MapPolicyReport(r v1alpha2.ReportInterface) *PolicyReport {

func MapPolicyReportResults(polr v1alpha2.ReportInterface) []*PolicyReportResult {
list := make([]*PolicyReportResult, 0, len(polr.GetResults()))
for _, result := range polr.GetResults() {
res := result.GetResource()
if res == nil && polr.GetScope() != nil {
res = polr.GetScope()
} else if res == nil {
res = &corev1.ObjectReference{}
}
for _, r := range polr.GetResults() {
res := result.Resource(polr, r)

ns := res.Namespace
if ns == "" {
ns = polr.GetNamespace()
}

list = append(list, &PolicyReportResult{
ID: result.GetID(),
ID: r.GetID(),
PolicyReportID: polr.GetID(),
Resource: Resource{
APIVersion: res.APIVersion,
Expand All @@ -132,16 +127,16 @@ func MapPolicyReportResults(polr v1alpha2.ReportInterface) []*PolicyReportResult
Namespace: ns,
UID: string(res.UID),
},
Policy: result.Policy,
Rule: result.Rule,
Source: result.Source,
Scored: result.Scored,
Message: result.Message,
Result: string(result.Result),
Severity: string(result.Severity),
Category: result.Category,
Properties: result.Properties,
Created: result.Timestamp.Seconds,
Policy: r.Policy,
Rule: r.Rule,
Source: r.Source,
Scored: r.Scored,
Message: r.Message,
Result: string(r.Result),
Severity: string(r.Severity),
Category: r.Category,
Properties: r.Properties,
Created: r.Timestamp.Seconds,
})
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/fixtures/policy_reports.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ var DefaultPolicyReport = &v1alpha2.PolicyReport{
},
Results: []v1alpha2.PolicyReportResult{
{
ID: "12348",
Message: "message",
Result: v1alpha2.StatusFail,
Scored: true,
Expand All @@ -53,13 +54,15 @@ var DefaultPolicyReport = &v1alpha2.PolicyReport{
Properties: map[string]string{"version": "1.2.0"},
},
{
ID: "12346",
Message: "message 2",
Result: v1alpha2.StatusFail,
Scored: true,
Policy: "priority-test",
Timestamp: v1.Timestamp{Seconds: 1614093000},
},
{
ID: "12347",
Message: "message 3",
Result: v1alpha2.StatusFail,
Scored: true,
Expand Down
8 changes: 8 additions & 0 deletions pkg/helper/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,11 @@ func Contains(source string, sources []string) bool {

return false
}

func Defaults(s, f string) string {
if s != "" {
return s
}

return f
}
Loading

0 comments on commit dd150ee

Please sign in to comment.