Merge pull request #4 from DCSO/stix2

implement STIX2 support

README.md

@@ -28,7 +28,6 @@ Usage of ./tie-threatbus-bridge:
     	configuration file (default "config.yaml")
   -verbose
     	be verbose
-
 ```
 
 Example:
@@ -76,6 +75,8 @@ collectors:
 threatbus:
   host: 1.0.0.1
   port: 13372
+  # legacy or stix2
+  format: legacy
 
 logfile: /var/log/tie-threatbus-bridge.log
 ```

collectorconfig.go

@@ -17,6 +17,7 @@ type Collector interface {
 }
 
 type ThreatBusConfig struct {
-	Host string
-	Port int
+	Host   string
+	Port   int
+	Format string
 }

config.yaml.example

@@ -34,5 +34,7 @@ collectors:
 threatbus:
   host: 127.0.0.1
   port: 13372
+  # legacy or stix2
+  format: legacy
 
 logfile: /var/log/tie-threatbus-bridge.log

go.mod

@@ -0,0 +1,14 @@
+module github.com/DCSO/tie-threatbus-bridge
+
+go 1.14
+
+require (
+	github.com/TcM1911/stix2 v0.6.1-0.20201122154655-049b8a26ae97
+	github.com/google/uuid v1.2.0 // indirect
+	github.com/pebbe/zmq4 v1.2.5
+	github.com/sirupsen/logrus v1.8.1
+	github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9
+	github.com/ugorji/go v1.2.4 // indirect
+	gopkg.in/yaml.v2 v2.4.0
+	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
+)

go.sum

@@ -0,0 +1,48 @@
+github.com/TcM1911/stix2 v0.6.1-0.20201122154655-049b8a26ae97 h1:gBYTlrBylpXIWw/zIlGVbaVMHPb20uwXU+Os4AqQNB8=
+github.com/TcM1911/stix2 v0.6.1-0.20201122154655-049b8a26ae97/go.mod h1:8WeIBIsco0zRt5UZZq0t+91n3k4crIREWZyCjCcM9Yc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
+github.com/pebbe/zmq4 v1.2.5 h1:ygTu6F/sMp7TIo7JN/ObpotHudy7+Rnun1LLSybyCFs=
+github.com/pebbe/zmq4 v1.2.5/go.mod h1:3+LG+02U+ToKtxF9avLo17NGTVDhWtRhsdU3spikK8o=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9 h1:/Bsw4C+DEdqPjt8vAqaC9LAqpAQnaCQQqmolqq3S1T4=
+github.com/tent/http-link-go v0.0.0-20130702225549-ac974c61c2f9/go.mod h1:RHkNRtSLfOK7qBTHaeSX1D6BNpI3qw7NTxsmNr4RvN8=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go v1.2.4 h1:cTciPbZ/VSOzCLKclmssnfQ/jyoVyOcJ3aoJyUV1Urc=
+github.com/ugorji/go v1.2.4/go.mod h1:EuaSCk8iZMdIspsu6HXH7X2UGKw1ezO4wCfGszGmmo4=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ugorji/go/codec v1.2.4 h1:C5VurWRRCKjuENsbM6GYVw8W++WVW9rSxoACKIvxzz8=
+github.com/ugorji/go/codec v1.2.4/go.mod h1:bWBu1+kIRWcF8uMklKaJrR6fTWQOwAlrIzX22pHwryA=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
+launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=

helpers_test.go

@@ -0,0 +1,11 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+func MakeIOC(indicator, datatype string) IOC {
+	return IOC{
+		Value:    indicator,
+		DataType: datatype,
+	}
+}

ioc_converter.go

@@ -0,0 +1,22 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+import "fmt"
+
+type IOCConverter interface {
+	Topic() string
+	FromIOC(*IOC) ([]byte, error)
+}
+
+func MakeIOCConverter(format string) (IOCConverter, error) {
+	switch format {
+	case "legacy":
+		return MakeIOCConverterLegacy(), nil
+	case "stix2":
+		return MakeIOCConverterSTIX2(), nil
+	default:
+		return nil, fmt.Errorf("unknown format: %s", format)
+	}
+}

data.go → ioc_converter_legacy.go

@@ -1,8 +1,15 @@
 // tie-threatbus-bridge
-// Copyright (c) 2020, DCSO GmbH
+// Copyright (c) 2021, DCSO GmbH
 
 package main
 
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
 // taken from ThreatBus code, starts with 1
 const (
 	_ = iota
@@ -74,8 +81,42 @@ func mapTIEtoThreatBus(iocType string) int {
 	case "FileName":
 		return FILENAME
 	case "EMail":
-		return EMAILSRC // TODO what to do with this? IPDST?
+		return EMAILSRC // TODO what to do with this? EMAILDST?
 	default:
 		return -1
 	}
 }
+
+type IOCConverterLegacy struct{}
+
+func MakeIOCConverterLegacy() *IOCConverterLegacy {
+	return &IOCConverterLegacy{}
+}
+
+func (c *IOCConverterLegacy) Topic() string {
+	return "threatbus/intel"
+}
+
+func (c *IOCConverterLegacy) FromIOC(ioc *IOC) ([]byte, error) {
+	tbIOCType := mapTIEtoThreatBus(ioc.DataType)
+	if tbIOCType < 0 {
+		return nil, fmt.Errorf("unsupported data type: %s", ioc.DataType)
+	}
+	iocJSON := IOCJSON{
+		TS: time.Now(),
+		ID: fmt.Sprintf("intel_%x", sha256.Sum256([]byte(ioc.Value))),
+		Data: struct {
+			Indicator []string `json:"indicator"`
+			IntelType int      `json:"intel_type"`
+		}{
+			Indicator: []string{ioc.Value},
+			IntelType: tbIOCType,
+		},
+		Operation: "ADD",
+	}
+	data, err := json.Marshal(iocJSON)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}

ioc_converter_legacy_test.go

@@ -0,0 +1,46 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestIOCConverterLegacyDomain(t *testing.T) {
+	// {"ts":"2021-03-18T19:29:31.555292569+01:00","id":"intel_c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2","data":{"indicator":["foobar"],"intel_type":11},"operation":"ADD"}
+	c := MakeIOCConverterLegacy()
+	ioc := MakeIOC("foobar", "DomainName")
+	v, err := c.FromIOC(&ioc)
+	if err != nil {
+		t.Fatalf("error creating json: %s", err.Error())
+	}
+	var i interface{}
+	err = json.Unmarshal(v, &i)
+	if err != nil {
+		t.Fatalf("error parsing json: %s", err.Error())
+	}
+	if !strings.Contains(string(v), `"data":{"indicator":["foobar"],"intel_type":11},"operation":"ADD"`) {
+		t.Fatalf("invalid result: %s", string(v))
+	}
+}
+
+func TestIOCConverterLegacyURL(t *testing.T) {
+	// {"ts":"2021-03-18T19:31:59.751183469+01:00","id":"intel_c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2","data":{"indicator":["foobar"],"intel_type":13},"operation":"ADD"}
+	c := MakeIOCConverterLegacy()
+	ioc := MakeIOC("foobar", "URLVerbatim")
+	v, err := c.FromIOC(&ioc)
+	if err != nil {
+		t.Fatalf("error creating json: %s", err.Error())
+	}
+	var i interface{}
+	err = json.Unmarshal(v, &i)
+	if err != nil {
+		t.Fatalf("error parsing json: %s", err.Error())
+	}
+	if !strings.Contains(string(v), `"data":{"indicator":["foobar"],"intel_type":13},"operation":"ADD"`) {
+		t.Fatalf("invalid result: %s", string(v))
+	}
+}

ioc_converter_stix.go

@@ -0,0 +1,62 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/TcM1911/stix2"
+)
+
+func mapTIEtoSTIX2(iocType string) (string, error) {
+	switch iocType {
+	case "DomainName":
+		return "[domain-name:value = '%s']", nil
+	case "URLVerbatim":
+		return "[url:value = '%s']", nil
+	case "IPv4":
+		return "[ipv4-addr:value = '%s']", nil
+	case "IPv6":
+		return "[ipv6-addr:value = '%s']", nil
+	case "FileName":
+		return "[file:name = '%s']", nil
+	case "EMail":
+		return "[email-addr:value = '%s']", nil
+	default:
+		return "", fmt.Errorf("unsupported data type: %s", iocType)
+	}
+}
+
+type IOCConverterSTIX2 struct{}
+
+func MakeIOCConverterSTIX2() *IOCConverterSTIX2 {
+	return &IOCConverterSTIX2{}
+}
+
+func (c *IOCConverterSTIX2) Topic() string {
+	return "stix2/indicator"
+}
+
+func (c *IOCConverterSTIX2) FromIOC(ioc *IOC) ([]byte, error) {
+	t, err := mapTIEtoSTIX2(ioc.DataType)
+	if err != nil {
+		return nil, err
+	}
+	tsNow := &stix2.Timestamp{
+		Time: time.Now().UTC(),
+	}
+	i, err := stix2.NewIndicator(fmt.Sprintf(t, ioc.Value), "stix", tsNow,
+		stix2.OptionCreated(tsNow), stix2.OptionModified(tsNow))
+	if err != nil {
+		return nil, err
+	}
+	i.Types = append(i.Types, stix2.IndicatorTypeMaliciousActivity)
+	data, err := json.Marshal(i)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}

ioc_converter_stix_test.go

@@ -0,0 +1,46 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestIOCConverterSTIX2Domain(t *testing.T) {
+	// {"type":"indicator","id":"indicator--42992b91-3ca8-40f6-9f0b-a8c644288663","spec_version":"2.1","created":"2021-03-18T18:32:42.275Z","modified":"2021-03-18T18:32:42.275Z","indicator_types":["malicious-activity"],"pattern":"[domain-name:value = 'foobar']","pattern_type":"stix","valid_from":"2021-03-18T18:32:42.275Z"}
+	c := MakeIOCConverterSTIX2()
+	ioc := MakeIOC("foobar", "DomainName")
+	v, err := c.FromIOC(&ioc)
+	if err != nil {
+		t.Fatalf("error creating json: %s", err.Error())
+	}
+	var i interface{}
+	err = json.Unmarshal(v, &i)
+	if err != nil {
+		t.Fatalf("error parsing json: %s", err.Error())
+	}
+	if !strings.Contains(string(v), `[domain-name:value = 'foobar']`) {
+		t.Fatalf("invalid result: %s", string(v))
+	}
+}
+
+func TestIOCConverterSTIX2URL(t *testing.T) {
+	// {"type":"indicator","id":"indicator--b152903a-35e7-412e-8a2e-47fb78d773ba","spec_version":"2.1","created":"2021-03-18T18:32:42.275Z","modified":"2021-03-18T18:32:42.275Z","indicator_types":["malicious-activity"],"pattern":"[url:value = 'foobar']","pattern_type":"stix","valid_from":"2021-03-18T18:32:42.275Z"}
+	c := MakeIOCConverterSTIX2()
+	ioc := MakeIOC("foobar", "URLVerbatim")
+	v, err := c.FromIOC(&ioc)
+	if err != nil {
+		t.Fatalf("error creating json: %s", err.Error())
+	}
+	var i interface{}
+	err = json.Unmarshal(v, &i)
+	if err != nil {
+		t.Fatalf("error parsing json: %s", err.Error())
+	}
+	if !strings.Contains(string(v), `[url:value = 'foobar']`) {
+		t.Fatalf("invalid result: %s", string(v))
+	}
+}

ioc_converter_test.go

@@ -0,0 +1,31 @@
+// tie-threatbus-bridge
+// Copyright (c) 2021, DCSO GmbH
+
+package main
+
+import "testing"
+
+func TestIOCConverterFactory(t *testing.T) {
+	c, err := MakeIOCConverter("legacy")
+	if err != nil {
+		t.Fatalf("error creating converter for legacy: %s", err.Error())
+	}
+	if c.Topic() != "threatbus/intel" {
+		t.Fatalf("wrong topic: %s", c.Topic())
+	}
+
+	c, err = MakeIOCConverter("stix2")
+	if err != nil {
+		t.Fatalf("error creating converter for stix2: %s", err.Error())
+	}
+	if c.Topic() != "stix2/indicator" {
+		t.Fatalf("wrong topic: %s", c.Topic())
+	}
+}
+
+func TestIOCConverterFactoryFail(t *testing.T) {
+	_, err := MakeIOCConverter("foo")
+	if err == nil {
+		t.Fatal("no error creating converter for foo")
+	}
+}