Update obfuscation

Author: tbauriedel

README.md

@@ -28,9 +28,13 @@ You can also combine your CLI arguments and the wizard. All arguments you pass f
 
 > **WARNING:** Some passwords or secrets are automatically removed, but this no guarantee, so be careful what you share!
 
-The `--hide` flag can be used multiple times to hide sensitive data, it supports regular expressions.
+The `--hide` flag can be used multiple times to hide sensitive data.  
+As these obfuscators are based on regex, you must add a regex pattern that meets your requirements.
 
-`# support-collector --hide "Secret:.*" --hide "Password:.*"`
+`# support-collector --hide "Secret:\s*(.*)"`
+
+This will replace:
+* Values like `Secret: DummyValue`
 
 In addition, files and folders that follow a specific pattern are not collected. This affects all files that correspond to the following filters:  
 `.*`, `*~`, `*.key`, `*.csr`, `*.crt`, and `*.pem`

internal/collection/collection.go

@@ -142,6 +142,7 @@ func (c *Collection) AddFileYAML(fileName string, data interface{}) {
 	_ = c.AddFileToOutput(file)
 }
 
+// AddFileJSON will add raw file data without obfuscation
 func (c *Collection) AddFileJSON(fileName string, data []byte) {
 	var jsonData interface{}
 
@@ -158,7 +159,7 @@ func (c *Collection) AddFileJSON(fileName string, data []byte) {
 	file := NewFile(fileName)
 	file.Data = prettyJSON
 
-	_ = c.AddFileToOutput(file)
+	_ = c.AddFileToOutputRaw(file)
 }
 
 func (c *Collection) AddFiles(prefix, source string) {
@@ -250,16 +251,19 @@ func (c *Collection) AddJournalLog(fileName, service string) {
 	c.AddCommandOutput(fileName, "journalctl", "-u", service, "-S", c.JournalLoggingInterval)
 }
 
+// RegisterObfuscator adds the given Obfuscator to the Obfuscators of Collection
 func (c *Collection) RegisterObfuscator(o *obfuscate.Obfuscator) {
 	c.Obfuscators = append(c.Obfuscators, o)
 }
 
+// RegisterObfuscators adds the given list of Obfuscator to the Obfuscators of Collection
 func (c *Collection) RegisterObfuscators(list ...*obfuscate.Obfuscator) {
 	for _, o := range list {
 		c.RegisterObfuscator(o)
 	}
 }
 
+// ClearObfuscators clears the list of Obfuscators in the Collection
 func (c *Collection) ClearObfuscators() {
 	c.Obfuscators = c.Obfuscators[:0]
 }
@@ -272,14 +276,17 @@ func (c *Collection) callObfuscators(kind obfuscate.Kind, name string, data []by
 
 	for _, o := range c.Obfuscators {
 		if o.IsAccepting(kind, name) {
-			count, out, err = o.Process(data)
+			count, out, err = o.Process(data, name)
 			if err != nil {
 				return
 			}
 
-			if count > 0 {
-				c.Log.Debugf("Obfuscation replaced %d token in %s", count, name)
-			}
+			data = out
+		}
+
+		if count > 0 {
+			c.Log.Debugf("ReplacePattern '%s' replaced %d token in %s", o.ReplacePattern, count, name)
+			count = 0
 		}
 	}
 

internal/obfuscate/obfuscate.go

@@ -29,24 +29,24 @@ const (
 
 // Obfuscator provides the basic functionality of an obfuscation engine.
 //
-// Kind filters the variant of resource we want to work on, while Affecting defines a list of regexp.Regexp, to match
+// Kind filters the variant of resource we want to work on, while ShouldAffect defines a list of regexp.Regexp, to match
 // against for the file names, or command.
 //
-// Replacements will be iterated, so all matches or matched groups will be replaced.
+// Replacement will be iterated, so all matches or matched groups will be replaced.
 type Obfuscator struct {
 	Kind
-	Affecting    []*regexp.Regexp
-	Replacements []*regexp.Regexp
-	Files        uint
-	Replaced     uint
+	ShouldAffect    []*regexp.Regexp
+	ReplacePattern  *regexp.Regexp
+	ObfuscatedFiles []string
+	Replaced        uint
 }
 
 // New returns a basic Obfuscator with provided regexp.Regexp.
 func New(kind Kind, affects, replace *regexp.Regexp) *Obfuscator {
 	return &Obfuscator{
-		Kind:         kind,
-		Affecting:    []*regexp.Regexp{affects},
-		Replacements: []*regexp.Regexp{replace},
+		Kind:           kind,
+		ShouldAffect:   []*regexp.Regexp{affects},
+		ReplacePattern: replace,
 	}
 }
 
@@ -77,27 +77,30 @@ func NewAny(replace string) *Obfuscator {
 	return o
 }
 
-// WithAffecting adds a new element to the list.
+// WithAffecting adds a new pattern to the list where the Obfuscator will be applied
 func (o *Obfuscator) WithAffecting(a *regexp.Regexp) *Obfuscator {
-	o.Affecting = append(o.Affecting, a)
+	o.ShouldAffect = append(o.ShouldAffect, a)
 
 	return o
 }
 
-// WithReplacement adds a new element to the list.
+// WithReplacement adds a new pattern to the list.
 func (o *Obfuscator) WithReplacement(r *regexp.Regexp) *Obfuscator {
-	o.Replacements = append(o.Replacements, r)
+	o.ReplacePattern = r
 
 	return o
 }
 
 // IsAccepting checks if we want to work on the resource.
+//
+//	Checks if the Obfuscator.Kind is matching the given kind. If not returns false.
+//	Checks if the Obfuscator.ShouldAffect is matching the given name.
 func (o Obfuscator) IsAccepting(kind Kind, name string) bool {
 	if o.Kind != KindAny && o.Kind != kind {
 		return false
 	}
 
-	for _, re := range o.Affecting {
+	for _, re := range o.ShouldAffect {
 		if re.MatchString(name) {
 			return true
 		}
@@ -106,16 +109,18 @@ func (o Obfuscator) IsAccepting(kind Kind, name string) bool {
 	return false
 }
 
-// Process takes data and returns it obfuscated.
-func (o *Obfuscator) Process(data []byte) (uint, []byte, error) {
-	count, out, err := o.ProcessReader(bytes.NewReader(data))
+// Process takes data and returns it obfuscated. Also takes name of file that is obfuscated
+//
+// Returns count of replaced values as uint, obfuscated data as []byte, and error
+func (o *Obfuscator) Process(data []byte, name string) (uint, []byte, error) {
+	count, out, err := o.ProcessReader(bytes.NewReader(data), name)
 
 	//goland:noinspection GoNilness
 	return count, out.Bytes(), err
 }
 
 // ProcessReader takes an io.Reader and returns a new one obfuscated.
-func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, err error) {
+func (o *Obfuscator) ProcessReader(r io.Reader, name string) (count uint, out bytes.Buffer, err error) {
 	rd := bufio.NewReader(r)
 
 	var (
@@ -153,7 +158,7 @@ func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, e
 
 		// Replace any matches, but skip empty lines
 		if strings.TrimSpace(line) != "" {
-			line, c = ReplacePatterns(line, o.Replacements)
+			line, c = ReplacePattern(line, o.ReplacePattern)
 		}
 
 		// Add line ending back after replacement
@@ -168,12 +173,13 @@ func (o *Obfuscator) ProcessReader(r io.Reader) (count uint, out bytes.Buffer, e
 	}
 
 	if count > 0 {
-		o.Files++
+		o.ObfuscatedFiles = append(o.ObfuscatedFiles, name)
 	}
 
 	return count, out, err
 }
 
+/*
 // ReplacePatterns replaces all the patterns matches in a line.
 func ReplacePatterns(line string, patterns []*regexp.Regexp) (s string, count uint) {
 	for _, pattern := range patterns {
@@ -185,6 +191,7 @@ func ReplacePatterns(line string, patterns []*regexp.Regexp) (s string, count ui
 
 	return line, count
 }
+*/
 
 // ReplacePattern replaces all matches in a line.
 func ReplacePattern(line string, pattern *regexp.Regexp) (s string, count uint) {

internal/obfuscate/obfuscate_test.go

@@ -24,7 +24,7 @@ func ExampleObfuscator() {
 	content := []byte(`password = "secret"`)
 
 	if o.IsAccepting(KindFile, "test.ini") {
-		count, data, err := o.Process(content)
+		count, data, err := o.Process(content, "")
 		fmt.Println(err)
 		fmt.Println(count)
 		fmt.Println(string(data))
@@ -64,17 +64,15 @@ func TestObfuscator_IsAccepting(t *testing.T) {
 
 func TestObfuscator_Process(t *testing.T) {
 	o := &Obfuscator{
-		Replacements: []*regexp.Regexp{
-			regexp.MustCompile(`^password\s*=\s*(.*)`),
-		},
+		ReplacePattern: regexp.MustCompile(`^password\s*=\s*(.*)`),
 	}
 
-	count, out, err := o.Process([]byte("default content\r\n"))
+	count, out, err := o.Process([]byte("default content\r\n"), "")
 	assert.NoError(t, err)
 	assert.Equal(t, uint(0), count)
 	assert.Equal(t, "default content\r\n", string(out))
 
-	count, out, err = o.Process([]byte(iniExample))
+	count, out, err = o.Process([]byte(iniExample), "")
 	assert.NoError(t, err)
 	assert.Equal(t, uint(1), count)
 	assert.Equal(t, iniResult, string(out))
@@ -83,15 +81,15 @@ func TestObfuscator_Process(t *testing.T) {
 func TestNewFile(t *testing.T) {
 	o := NewFile(`^password\s*=\s*(.*)`, "conf")
 	assert.Equal(t, KindFile, o.Kind)
-	assert.Len(t, o.Affecting, 1)
-	assert.Len(t, o.Replacements, 1)
+	assert.Len(t, o.ShouldAffect, 1)
+	assert.NotEmpty(t, o.ShouldAffect)
 }
 
 func TestNewOutput(t *testing.T) {
 	o := NewOutput(`^password\s*=\s*(.*)`, "icinga2", "daemon", "-C")
 	assert.Equal(t, KindOutput, o.Kind)
-	assert.Len(t, o.Affecting, 1)
-	assert.Len(t, o.Replacements, 1)
+	assert.Len(t, o.ShouldAffect, 1)
+	assert.NotEmpty(t, o.ReplacePattern)
 
 	assert.True(t, o.IsAccepting(KindOutput, "icinga2 daemon -C"))
 	assert.False(t, o.IsAccepting(KindOutput, "icinga2 daemon"))

internal/util/testing.go

@@ -19,7 +19,7 @@ func AssertObfuscation(t *testing.T, obfuscators []*obfuscate.Obfuscator,
 			continue
 		}
 
-		_, out, err := o.Process([]byte(input))
+		_, out, err := o.Process([]byte(input), name)
 		if err != nil {
 			t.Errorf("error during obfuscation: %s", err)
 			return

internal/util/util.go

@@ -20,6 +20,21 @@ func StringInSlice(a string, list []string) bool {
 	return false
 }
 
+// DistinctStringSlice retuns the given slice with unique values
+func DistinctStringSlice(arr []string) []string {
+	seen := make(map[string]bool)
+	var result []string
+
+	for _, val := range arr {
+		if !seen[val] {
+			result = append(result, val)
+			seen[val] = true
+		}
+	}
+
+	return result
+}
+
 // IsPrivilegedUser returns true when the current user is root.
 func IsPrivilegedUser() bool {
 	u, err := user.Current()

main.go

@@ -246,16 +246,19 @@ func main() {
 	c.Log.Infof("Collection complete, took us %.3f seconds", metric.Timings["total"].Seconds())
 
 	// Collect obfuscation info
-	var files, count uint
+	var (
+		count         uint
+		affectedFiles []string
+	)
 
 	for _, o := range c.Obfuscators {
-		files += o.Files
-
 		count += o.Replaced
+
+		affectedFiles = append(affectedFiles, o.ObfuscatedFiles...)
 	}
 
-	if files > 0 {
-		c.Log.Infof("Obfuscation replaced %d token in %d files (%d definitions)", count, files, len(c.Obfuscators))
+	if len(affectedFiles) > 0 {
+		c.Log.Infof("Obfuscation replaced %d token in %d files (%d definitions)", count, len(util.DistinctStringSlice(affectedFiles)), len(c.Obfuscators))
 	}
 
 	// get absolute path of outputFile

modules/icinga2/collector.go

@@ -1,13 +1,13 @@
 package icinga2
 
 import (
+	"github.com/NETWAYS/support-collector/internal/obfuscate"
 	"github.com/NETWAYS/support-collector/internal/util"
 	"os"
 	"path/filepath"
 	"regexp"
 
 	"github.com/NETWAYS/support-collector/internal/collection"
-	"github.com/NETWAYS/support-collector/internal/obfuscate"
 )
 
 const ModuleName = "icinga2"