diff --git a/README.md b/README.md index 4d20ba8..497fa81 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ application without any arguments. rickybobby [global options] command [command options] [arguments...] VERSION: - 1.0.0 + 1.0.3 AUTHOR: Chaz Lever @@ -86,6 +86,7 @@ application without any arguments. --profile toggle performance profiler --sensor value name of sensor DNS traffic was collected from --source value name of source DNS traffic was collected from + --format value specify the output formatter to use ["avro" "json"] (default: "json") --help, -h show help --version, -v print the version diff --git a/go.mod b/go.mod index 6b09d10..2829926 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.12 require ( github.com/google/gopacket v1.1.17 + github.com/hamba/avro v1.5.4 github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/miekg/dns v1.1.15 github.com/pkg/profile v1.3.0 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.3.0 // indirect golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect diff --git a/go.sum b/go.sum index 299296b..6f68bea 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,26 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/hamba/avro v1.5.4 h1:4S1QSzzGU7vMrDmZo4aFN/OkhnV7UTKqRG0yUAZdljo= +github.com/hamba/avro v1.5.4/go.mod h1:sq9qfIRLiKNXCXDNo52SPwJ2euqeiWGQIE4Nc2RW1pg= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI= github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -17,8 +30,9 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -34,5 +48,9 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/parser/schema.go b/iohandlers/schema.go similarity index 80% rename from parser/schema.go rename to iohandlers/schema.go index 930d534..de86c50 100644 --- a/parser/schema.go +++ b/iohandlers/schema.go @@ -1,11 +1,15 @@ -package parser +package iohandlers import ( - "encoding/json" - "fmt" - "github.com/miekg/dns" - log "github.com/sirupsen/logrus" "strings" + + "github.com/miekg/dns" +) + +const ( + DnsAnswer = iota + DnsAuthority = iota + DnsAdditional = iota ) // JSON serialization only supports nullifying types that can accept nil. @@ -40,7 +44,25 @@ type DnsSchema struct { Sensor string `json:"sensor,omitempty"` } -func (d DnsSchema) ToJson(rr *dns.RR, section int) { +var ( + Initializers = make(map[string]func()) + Marshalers = make(map[string]func(*DnsSchema)) + Closers = make(map[string]func()) +) + +func Initialize(format string) { + if init, ok := Initializers[format]; ok { + init() + } +} + +func Close(format string) { + if closer, ok := Closers[format]; ok { + closer() + } +} + +func (d DnsSchema) Marshal(rr *dns.RR, section int, format string) { if rr != nil { // This works because RR.Header().String() prefixes the RDATA // in the RR.String() representation. @@ -63,9 +85,5 @@ func (d DnsSchema) ToJson(rr *dns.RR, section int) { } } - jsonData, err := json.Marshal(&d) - if err != nil { - log.Warnf("Error converting to JSON: %v", err) - } - fmt.Printf("%s\n", jsonData) + Marshalers[format](&d) } diff --git a/iohandlers/toAvro.go b/iohandlers/toAvro.go new file mode 100644 index 0000000..baa4b53 --- /dev/null +++ b/iohandlers/toAvro.go @@ -0,0 +1,254 @@ +package iohandlers + +import ( + "os" + + "github.com/hamba/avro/ocf" + log "github.com/sirupsen/logrus" +) + +func init() { + Initializers["avro"] = toAvroInitializer + Marshalers["avro"] = toAvro + Closers["avro"] = toAvroCloser +} + +var ( + avroEncoder *ocf.Encoder + avroData avroDnsSchema + avroSchema string + ttl int + rtype int + ecsSource int + ecsScope int +) + +// Avro doesn't support unsigned integers so we create a new type for serialization. +type avroDnsSchema struct { + Timestamp int64 `avro:"timestamp"` + Sha256 string `avro:"sha256"` + Udp bool `avro:"udp"` + Ipv4 bool `avro:"ipv4"` + SourceAddress string `avro:"src_address"` + SourcePort int `avro:"src_port"` + DestinationAddress string `avro:"dst_address"` + DestinationPort int `avro:"dst_port"` + Id int `avro:"id"` + Rcode int `avro:"rcode"` + Truncated bool `avro:"truncated"` + Response bool `avro:"response"` + RecursionDesired bool `avro:"recursion_desired"` + Answer bool `avro:"answer"` + Authority bool `avro:"authority"` + Additional bool `avro:"additional"` + Qname string `avro:"qname"` + Qtype int `avro:"qtype"` + Ttl *int `avro:"ttl"` + Rname *string `avro:"rname"` + Rtype *int `avro:"rtype"` + Rdata *string `avro:"rdata"` + EcsClient *string `avro:"ecs_client"` + EcsSource *int `avro:"ecs_source"` + EcsScope *int `avro:"ecs_scope"` + Source *string `avro:"source"` + Sensor *string `avro:"sensor"` +} + +func toAvroInitializer() { + avroSchema = `{ + "type": "record", + "name": "DnsSchema", + "namespace": "org.hamba.avro", + "fields": [ + { + "name": "timestamp", + "type": "long" + }, + { + "name": "sha256", + "type": "string" + }, + { + "name": "udp", + "type": "boolean" + }, + { + "name": "ipv4", + "type": "boolean" + }, + { + "name": "src_address", + "type": "string" + }, + { + "name": "src_port", + "type": "int" + }, + { + "name": "dst_address", + "type": "string" + }, + { + "name": "dst_port", + "type": "int" + }, + { + "name": "id", + "type": "int" + }, + { + "name": "rcode", + "type": "int" + }, + { + "name": "truncated", + "type": "boolean" + }, + { + "name": "response", + "type": "boolean" + }, + { + "name": "recursion_desired", + "type": "boolean" + }, + { + "name": "answer", + "type": "boolean" + }, + { + "name": "authority", + "type": "boolean" + }, + { + "name": "additional", + "type": "boolean" + }, + { + "name": "qname", + "type": "string" + }, + { + "name": "qtype", + "type": "int" + }, + { + "name": "ttl", + "type": ["null", "int"], + "default": null + }, + { + "name": "rname", + "type": ["null", "string"], + "default": null + }, + { + "name": "rtype", + "type": ["null", "int"], + "default": null + }, + { + "name": "rdata", + "type": ["null", "string"], + "default": null + }, + { + "name": "ecs_client", + "type": ["null", "string"], + "default": null + }, + { + "name": "ecs_source", + "type": ["null", "int"], + "default": null + }, + { + "name": "ecs_scope", + "type": ["null", "int"], + "default": null + }, + { + "name": "source", + "type": ["null", "string"], + "default": null + }, + { + "name": "sensor", + "type": ["null", "string"], + "default": null + } + ] + }` + + var err error + avroEncoder, err = ocf.NewEncoder(avroSchema, os.Stdout, ocf.WithCodec(ocf.Snappy)) + if err != nil { + log.Fatalf("Error creating Avro Encoder: %v", err) + } +} + +func toAvro(d *DnsSchema) { + avroData.Timestamp = d.Timestamp + avroData.Sha256 = d.Sha256 + avroData.Udp = d.Udp + avroData.Ipv4 = d.Ipv4 + avroData.SourceAddress = d.SourceAddress + avroData.SourcePort = int(d.SourcePort) + avroData.DestinationAddress = d.DestinationAddress + avroData.DestinationPort = int(d.DestinationPort) + avroData.Id = int(d.Id) + avroData.Rcode = d.Rcode + avroData.Truncated = d.Truncated + avroData.Response = d.Response + avroData.RecursionDesired = d.RecursionDesired + avroData.Answer = d.Answer + avroData.Authority = d.Authority + avroData.Additional = d.Additional + avroData.Qname = d.Qname + avroData.Qtype = int(d.Qtype) + avroData.Ttl = nil + avroData.Rname = d.Rname + avroData.Rdata = d.Rdata + avroData.Rtype = nil + avroData.EcsClient = d.EcsClient + avroData.EcsSource = nil + avroData.EcsScope = nil + avroData.Source = nil + avroData.Sensor = nil + + // Handle source and sensor + if len(d.Source) > 0 { + avroData.Source = &d.Source + } + if len(d.Sensor) > 0 { + avroData.Sensor = &d.Sensor + } + + // Handle pointers requiring type conversion + if d.Ttl != nil { + ttl = int(*d.Ttl) + avroData.Ttl = &ttl + } + if d.Rtype != nil { + rtype = int(*d.Rtype) + avroData.Rtype = &rtype + } + if d.EcsSource != nil { + ecsSource = int(*d.EcsSource) + avroData.EcsSource = &ecsSource + } + if d.EcsScope != nil { + ecsScope = int(*d.EcsScope) + avroData.EcsScope = &ecsScope + } + + err := avroEncoder.Encode(&avroData) + if err != nil { + log.Warnf("Error encoding Avro: %v", err) + } +} + +func toAvroCloser() { + avroEncoder.Flush() + avroEncoder.Close() +} diff --git a/iohandlers/toJson.go b/iohandlers/toJson.go new file mode 100644 index 0000000..44412c8 --- /dev/null +++ b/iohandlers/toJson.go @@ -0,0 +1,20 @@ +package iohandlers + +import ( + "encoding/json" + "fmt" + + log "github.com/sirupsen/logrus" +) + +func init() { + Marshalers["json"] = toJson +} + +func toJson(d *DnsSchema) { + jsonData, err := json.Marshal(d) + if err != nil { + log.Warnf("Error converting to JSON: %v", err) + } + fmt.Printf("%s\n", jsonData) +} diff --git a/main.go b/main.go index 325df90..88f4cdc 100644 --- a/main.go +++ b/main.go @@ -1,21 +1,49 @@ package main import ( - log "github.com/sirupsen/logrus" - "gopkg.in/urfave/cli.v1" + "fmt" "os" "time" + "github.com/chazlever/rickybobby/iohandlers" "github.com/chazlever/rickybobby/parser" "github.com/pkg/profile" + log "github.com/sirupsen/logrus" + "gopkg.in/urfave/cli.v1" ) -func loadGlobalOptions(c *cli.Context) { +func getOutputFormats() []string { + marshalers := make([]string, 0, len(iohandlers.Marshalers)) + for m := range iohandlers.Marshalers { + marshalers = append(marshalers, m) + } + + return marshalers +} + +func loadGlobalOptions(c *cli.Context) error { parser.DoParseTcp = c.GlobalBool("tcp") parser.DoParseQuestions = c.GlobalBool("questions") parser.DoParseQuestionsEcs = c.GlobalBool("questions-ecs") parser.Source = c.GlobalString("source") parser.Sensor = c.GlobalString("sensor") + outputFormat := c.GlobalString("format") + parser.OutputFormat = outputFormat + + outputFormats := make(map[string]bool) + for _, format := range getOutputFormats() { + outputFormats[format] = true + } + + if _, ok := outputFormats[outputFormat]; !ok { + return cli.NewExitError( + fmt.Sprintf("ERROR: Invalid output format: \"%s\" not in %v", + outputFormat, + getOutputFormats()), + 1) + } + + return nil } func pcapCommand(c *cli.Context) error { @@ -27,7 +55,9 @@ func pcapCommand(c *cli.Context) error { defer profile.Start().Stop() } - loadGlobalOptions(c) + if err := loadGlobalOptions(c); err != nil { + return err + } for _, f := range c.Args() { parser.ParseFile(f) @@ -44,7 +74,9 @@ func liveCommand(c *cli.Context) error { defer profile.Start().Stop() } - loadGlobalOptions(c) + if err := loadGlobalOptions(c); err != nil { + return err + } // Load command specific flags snapshotLen := int32(c.Int("snaplen")) @@ -125,6 +157,11 @@ func main() { Name: "source", Usage: "name of source DNS traffic was collected from", }, + cli.StringFlag{ + Name: "format", + Usage: fmt.Sprintf("specify the output formatter to use %+q", getOutputFormats()), + Value: "json", + }, } app.Action = cli.ShowAppHelp diff --git a/parser/parse.go b/parser/parse.go index cf594ca..ae97d82 100644 --- a/parser/parse.go +++ b/parser/parse.go @@ -7,6 +7,7 @@ import ( "os" "time" + "github.com/chazlever/rickybobby/iohandlers" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" @@ -14,18 +15,13 @@ import ( log "github.com/sirupsen/logrus" ) -const ( - DnsAnswer = iota - DnsAuthority = iota - DnsAdditional = iota -) - var ( DoParseTcp = true DoParseQuestions = false DoParseQuestionsEcs = true Source = "" Sensor = "" + OutputFormat = "" ) func ParseFile(fname string) { @@ -80,7 +76,7 @@ func ParseDevice(device string, snapshotLen int32, promiscuous bool, timeout tim func ParseDns(handle *pcap.Handle) { var ( - schema DnsSchema + schema iohandlers.DnsSchema stats Statistics ip4 *layers.IPv4 ip6 *layers.IPv6 @@ -98,6 +94,9 @@ func ParseDns(handle *pcap.Handle) { packetSource.NoCopy = true packetSource.Lazy = true + // Initialize IO handler for output format + iohandlers.Initialize(OutputFormat) + PACKETLOOP: for { packet, err := packetSource.NextPacket() @@ -239,25 +238,28 @@ PACKETLOOP: if (DoParseQuestions && !schema.Response) || (DoParseQuestionsEcs && schema.EcsClient != nil && !schema.Response) || (schema.Rcode == 3 && len(msg.Ns) < 1) { - schema.ToJson(nil, -1) + schema.Marshal(nil, -1, OutputFormat) } // Let's get ANSWERS for _, rr := range msg.Answer { - schema.ToJson(&rr, DnsAnswer) + schema.Marshal(&rr, iohandlers.DnsAnswer, OutputFormat) } // Let's get AUTHORITATIVE information for _, rr := range msg.Ns { - schema.ToJson(&rr, DnsAuthority) + schema.Marshal(&rr, iohandlers.DnsAuthority, OutputFormat) } // Let's get ADDITIONAL information for _, rr := range msg.Extra { - schema.ToJson(&rr, DnsAdditional) + schema.Marshal(&rr, iohandlers.DnsAdditional, OutputFormat) } } + // Cleanup IO handler for output format + iohandlers.Close(OutputFormat) + log.Infof("Number of TOTAL packets: %v", stats.PacketTotal) log.Infof("Number of IPv4 packets: %v", stats.PacketIPv4) log.Infof("Number of IPv6 packets: %v", stats.PacketIPv6) diff --git a/parser/stats.go b/parser/stats.go index 01fe377..84611b3 100644 --- a/parser/stats.go +++ b/parser/stats.go @@ -3,6 +3,7 @@ package parser import ( "encoding/json" "fmt" + log "github.com/sirupsen/logrus" )