converted json support to yaml

kaanaktas · Aug 20, 2022 · d86d40e · d86d40e
1 parent 72c88ef
commit d86d40e
Show file tree

Hide file tree

Showing 26 changed files with 594 additions and 765 deletions.
diff --git a/README.md b/README.md
@@ -22,133 +22,94 @@ Configuration
 ## datafilter
 
 Currently, go-slm implements 3 data filters, **owasp-sqli**, **owasp-xss** and **pan-filtering**. The default definitions for these filters are defined in the go-slm package 
-and can be viewed under **datafilter/rules**. At the same time, the definitions of these filters are defined in **datafilter/datafilter_rule_set.json** and are ready to use without any modification.<br/>
+and can be viewed under **datafilter/rules**. At the same time, the definitions of these filters are defined in **datafilter/datafilter_rule_set.yaml** and are ready to use without any modification.<br/>
 If users want to make any changes in the existing filters, or if they want to add new rules to the filters;
 * First, they need to create custom filter files and put them into the project directory. 
-* Second, they need to create a custom **datafilter_rule_set.json** file and put it into the project directory. Users can update existing types/rules in the default datafilter_rule_set.json file 
-or define new types/rules with changes made in their own datafilter_rule_set.json.
-* Finally, custom filter files should be linked in the custom datafilter_rule_set.json. 
+* Second, they need to create a custom **datafilter_rule_set.yaml** file and put it into the project directory. Users can update existing types/rules in the default datafilter_rule_set.yaml file 
+or define new types/rules with changes made in their own datafilter_rule_set.yaml.
+* Finally, custom filter files should be linked in the custom datafilter_rule_set.yaml. 
 
-**custom_owasp_attack_sqli.json**
+**custom_owasp_attack_sqli.yaml**
 
 (As an example, let's assume that we put this file under the **/config** directory in the main application.)
 
 ```
-[
-  {
-    "name": "942110",
-    "disable" : true,
-    "rule": "(?:^\\s*[\\\"'`;]+|[\\\"'`]+\\s*$)",
-    "message": "My custom message: SQL Injection Attack: Common Injection Testing Detected",
-    "sample": "var=''"
-  },
-  {
-    "name": "new_rule_1",
-    "disable" : false,
-    "rule": "<new_rule_regex>",
-    "message": "<new_rule_message>",
-    "sample": "<new_rule_sample>"
-  },
-]
+- name: '942110'
+  disable: true
+  rule: (?:^\s*[\"'`;]+|[\"'`]+\s*$)
+  message: 'My custom message: SQL Injection Attack: Common Injection Testing Detected'
+  sample: var=''
+- name: new_rule_1
+  disable: false
+  rule: <new_rule_regex>
+  message: <new_rule_message>
+  sample: <new_rule_sample>
 ```
 
 In the example file above, 2 rules are defined for owasp_attack_sqli. 
-* The first rule with name=942110 updates and disables the existing rule in the package rule file (**datafilter/rules/owasp_attack_sqli.json**). 
+* The first rule with name=942110 updates and disables the existing rule in the package rule file (**datafilter/rules/owasp_attack_sqli.yaml**). 
 By doing this, we disable the rule which is not required in our rule set. Similarly, we can change the rule message or regex value as needed.
 * The second rule creates a new filter rule and adds it to the rule set which is generated from the package rule file.
 
 
-**custom_datafilter_rule_set.json**
+**custom_datafilter_rule_set.yaml**
 
 ```
-[
-  {
-    "type": "owasp",
-    "rules": [
-      {
-        "name": "sqli",
-        "path": "rules/owasp_attack_sqli.json"
-        "custom_path": "config/custom_owasp_attack_sqli.json"
-      }
-    ]
-  }
-]
+- type: owasp
+  rules:
+    - name: sqli
+      path: rules/owasp_attack_sqli.yaml
+      custom_path: config/custom_owasp_attack_sql.yaml
 ```
 
-In the **custom_datafilter_rule_set.json** file above, we define a single rule which only updates **owasp_sqli** and leaves the other rules as is.
-So, the rules inside **custom_owasp_attack_sqli.json** update the rules defined in the **owasp_attack_sqli.json** file if necessary, or add them to our rule_set as a new rule.</br>
-In order for the newly created **custom_owasp_attack_sqli.json** file to be considered, it should be defined in the **GO_SLM_DATA_FILTER_RULE_SET_PATH** environment variable as in the example below.
+In the **custom_datafilter_rule_set.yaml** file above, we define a single rule which only updates **owasp_sqli** and leaves the other rules as is.
+So, the rules inside **custom_owasp_attack_sqli.yaml** update the rules defined in the **owasp_attack_sqli.yaml** file if necessary, or add them to our rule_set as a new rule.</br>
+In order for the newly created **custom_owasp_attack_sqli.yaml** file to be considered, it should be defined in the **GO_SLM_DATA_FILTER_RULE_SET_PATH** environment variable as in the example below.
 
-`_ = os.Setenv("GO_SLM_DATA_FILTER_RULE_SET_PATH", "/{directory}/custom_datafilter_rule_set.json")
+`_ = os.Setenv("GO_SLM_DATA_FILTER_RULE_SET_PATH", "/{directory}/custom_datafilter_rule_set.yaml")
 `
 ## policy
 
-We can create reusable policies in our common policy rule file (similar to **/testconfig/common_policies.json**) and use them 
-to combine different policies in **policy_rule_set.json**. This file can be named based on requirement and should be defined in the **GO_SLM_COMMON_POLICIES_PATH**
+We can create reusable policies in our common policy rule file (similar to **/testconfig/common_policies.yaml**) and use them 
+to combine different policies in **policy_rule_set.yaml**. This file can be named based on requirement and should be defined in the **GO_SLM_COMMON_POLICIES_PATH**
 environment variable as in the example below. 
 
-`_ = os.Setenv("GO_SLM_COMMON_POLICIES_PATH", "/{directory}/common_policies.json")
+`_ = os.Setenv("GO_SLM_COMMON_POLICIES_PATH", "/{directory}/common_policies.yaml")
 `
 
-**common_policies.json**
+**common_policies.yaml**
 
 ```
-  {
-  "commonPolicies": [
-    {
-      "PolicyName": "combined_policy",
-      "Policy": [
-        {
-          "name": "xss",
-          "active": true
-        },
-        {
-          "name": "pan_process",
-          "active": true
-        },
-        {
-          "name": "sqli",
-          "active": true
-        }
-      ]
-    },
-    {
-      "PolicyName": "pan_only_policy",
-      "Policy": [
-        {
-          "name": "pan_process",
-          "active": true
-        }
-      ]
-    }
-  ]
-}
+- PolicyName: combined_policy
+  Policy:
+    - name: xss
+      active: true
+    - name: pan_process
+      active: true
+    - name: sqli
+      active: true
+- PolicyName: pan_only_policy
+  Policy:
+    - name: pan_process
+      active: true
 ```
 
 Below, you can see how policy definitions are generated for our API services. Simply, our common policies that we defined 
 before are assigned to the services to be triggered for request and response in each API service.
 This file can be named based on requirement and should be defined in the **GO_SLM_COMMON_RULES_PATH**
 environment variable as in the example below.
 
-`_ = os.Setenv("GO_SLM_POLICY_RULE_SET_PATH", "/{directory}/policy_rule_set.json")
+`_ = os.Setenv("GO_SLM_POLICY_RULE_SET_PATH", "/{directory}/policy_rule_set.yaml")
 `
 
-**policy_rule_set.json**
+**policy_rule_set.yaml**
 
 
 ```
-{
-  "policies": [
-    {
-      "serviceName": "test",
-      "request": "pan_only_policy",
-      "response": "combined_policy"
-    },
-    {
-      "serviceName": "test2",
-      "request": "combined_policy",
-      "response": "pan_only_policy"
-    }
-  ]
-}
+- serviceName: test
+  request: combined_policy
+  response: pan_only_policy
+- serviceName: test2
+  request: combined_policy
+  response: pan_only_policy
 ```
diff --git a/common_test.go b/common_test.go
@@ -6,8 +6,8 @@ import (
 )
 
 func TestMain(m *testing.M) {
-	_ = os.Setenv("GO_SLM_POLICY_RULE_SET_PATH", "/testconfig/policy_rule_set.json")
-	_ = os.Setenv("GO_SLM_COMMON_POLICIES_PATH", "/testconfig/common_policies.json")
+	_ = os.Setenv("GO_SLM_POLICY_RULE_SET_PATH", "/testconfig/policy_rule_set.yaml")
+	_ = os.Setenv("GO_SLM_COMMON_POLICIES_PATH", "/testconfig/common_policies.yaml")
 	_ = os.Setenv("GO_SLM_CURRENT_MODULE_NAME", "github.com/kaanaktas/dummy")
 
 	os.Exit(m.Run())

diff --git a/datafilter/datafilter_rule_set.json b/datafilter/datafilter_rule_set.json
diff --git a/datafilter/datafilter_rule_set.yaml b/datafilter/datafilter_rule_set.yaml
@@ -0,0 +1,10 @@
+- type: pan
+  rules:
+      - name: pan_process
+        path: rules/pan_process.yaml
+- type: owasp
+  rules:
+      - name: sqli
+        path: rules/owasp_attack_sqli.yaml
+      - name: xss
+        path: rules/owasp_attack_xss.yaml
diff --git a/datafilter/load.go b/datafilter/load.go
@@ -2,38 +2,38 @@ package datafilter
 
 import (
 	"embed"
-	"encoding/json"
 	"fmt"
 	"github.com/kaanaktas/go-slm/cache"
 	"github.com/kaanaktas/go-slm/config"
+	"gopkg.in/yaml.v3"
 	"log"
 	"path/filepath"
 )
 
 type ruleSet struct {
-	Type  string  `json:"type"`
-	Rules []rules `json:"rules"`
+	Type  string  `yaml:"type"`
+	Rules []rules `yaml:"rules"`
 }
 
 type rules struct {
-	Name       string `json:"name"`
-	Path       string `json:"path"`
-	CustomPath string `json:"custom_path"`
+	Name       string `yaml:"name"`
+	Path       string `yaml:"path"`
+	CustomPath string `yaml:"custom_path"`
 }
 
 var cacheIn = cache.NewInMemory()
 
-//go:embed datafilter_rule_set.json
+//go:embed datafilter_rule_set.yaml
 var dataFilterRuleSet []byte
 
 //go:embed rules/*
 var ruleFs embed.FS
 
 func Load(dataFilterRuleSetPath string) {
 	var ruleSet, customRuleSet []ruleSet
-	err := json.Unmarshal(dataFilterRuleSet, &ruleSet)
+	err := yaml.Unmarshal(dataFilterRuleSet, &ruleSet)
 	if err != nil {
-		msg := fmt.Sprintf("Can't unmarshall the content of datafilter_rule_set.json. Error: %s", err)
+		msg := fmt.Sprintf("Can't unmarshall the content of datafilter_rule_set.yaml. Error: %s", err)
 		panic(msg)
 	}
 
@@ -43,7 +43,7 @@ func Load(dataFilterRuleSetPath string) {
 			msg := fmt.Sprintf("Error while reading %s. Error: %s", dataFilterRuleSetPath, err)
 			panic(msg)
 		}
-		err = json.Unmarshal(content, &customRuleSet)
+		err = yaml.Unmarshal(content, &customRuleSet)
 		if err != nil {
 			msg := fmt.Sprintf("Can't unmarshall the content of %s. Error: %s", dataFilterRuleSetPath, err)
 			panic(msg)
@@ -76,7 +76,7 @@ func Load(dataFilterRuleSetPath string) {
 			}
 
 			var patterns, customPatterns []pattern
-			err = json.Unmarshal(content, &patterns)
+			err = yaml.Unmarshal(content, &patterns)
 			if err != nil {
 				msg := fmt.Sprintf("Can't unmarshall the content of %s. Error: %s", rule.Path, err)
 				panic(msg)
@@ -88,7 +88,7 @@ func Load(dataFilterRuleSetPath string) {
 					msg := fmt.Sprintf("Error while reading %s. Error: %s", rule.CustomPath, err)
 					panic(msg)
 				}
-				err = json.Unmarshal(content, &customPatterns)
+				err = yaml.Unmarshal(content, &customPatterns)
 				if err != nil {
 					msg := fmt.Sprintf("Can't unmarshall the content of %s. Error: %s", rule.CustomPath, err)
 					panic(msg)