Allow for custom MSDNS time format

- Add inputs->msdns_time_format field to config
- Fix parsing of default timestamp with 12h clock (AM/PM)

Author: tg

config.yml

@@ -71,6 +71,10 @@ inputs:
       # Default: (none)
       file:
 
+  # Time format used for parsing MSDNS log files.
+  # Format layout documented at https://golang.org/pkg/time/#Parse
+  msdns_time_format:
+
 ################################################################################
 # The outputs section describes where NFR should send the alerts generated by
 # the Analytics Engine (e.g. Graylog, a local file, or terminal)

config/config.go

@@ -76,6 +76,9 @@ type Config struct {
 
 		// Monitors keeps list of log files to monitor.
 		Monitors []Monitor `yaml:"monitor"`
+
+		// MSDNS time format
+		MSDNSTimeFormat string `yaml:"msdns_time_format"`
 	} `yaml:"inputs"`
 
 	// Outputs describes where should send the alerts generated by the Analytics Engine.

executor/executor.go

@@ -567,7 +567,12 @@ func (e *Executor) openFileParser(file, fileFomrat string) (err error) {
 	case "suricata":
 		e.lr, err = suricata.NewFileParser(file)
 	case "msdns":
-		e.lr, err = msdns.NewFileParser(file)
+		var p *msdns.Parser
+		p, err = msdns.NewFileParser(file)
+		if p != nil {
+			p.TimeFormat = e.cfg.Inputs.MSDNSTimeFormat
+		}
+		e.lr = p
 	case "syslog-named":
 		e.lr, err = syslognamed.NewFileParser(file)
 	default:

logs/msdns/parser.go

@@ -16,6 +16,14 @@ import (
 
 // A Parser parses and reads network events from msdns logs.
 type Parser struct {
+	// TimeFormat used for parsing timestamp.
+	// If not set, will accept the following format:
+	//   2006-01-02 3:04:05 PM
+	//   2006/01/02 3:04:05 PM
+	//   2006-01-02 15:04:05
+	//   2006/01/02 15:04:05
+	TimeFormat string
+
 	r io.ReadCloser
 }
 
@@ -66,34 +74,81 @@ func (*Parser) ReadIP() ([]*packet.IPPacket, error) {
 
 var numToDotRe = regexp.MustCompile(`\(\d+\)`)
 
+func guessTimeFormat(s string) string {
+	if len(s) < 10 {
+		return ""
+	}
+
+	sep := s[4]                  // date separator
+	ampm := (s[len(s)-1] == 'M') // use 12h clock (AM/PM)
+
+	switch true {
+	case sep == '-' && ampm:
+		return "2006-01-02 3:04:05 PM"
+	case sep == '-' && !ampm:
+		return "2006-01-02 15:04:05"
+	case sep == '/' && ampm:
+		return "2006/01/02 3:04:05 PM"
+	case sep == '/' && !ampm:
+		return "2006/01/02 15:04:05"
+	}
+
+	return ""
+}
+
 // ParseLineDNS parse single log line with dns data.
-func (*Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) {
+func (p *Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) {
 	s := strings.Fields(line)
-	if len(s) == 16 && (s[2] == "AM" || s[2] == "PM") {
-		s = append(s[:3], s[4:]...)
+
+	if len(s) < 15 {
+		return nil, nil
+	}
+
+	// find Context field, which can be 4th or 5th field,
+	// depends if timestamp consists of 2 or 3 fields.
+	contextIdx := 0
+	switch "PACKET" {
+	case s[3]:
+		contextIdx = 3
+	case s[4]:
+		contextIdx = 4
+	default:
+		return nil, nil
 	}
 
-	if len(s) != 15 || s[3] != "PACKET" || s[6] != "Rcv" || s[9] != "Q" {
+	ts := strings.Join(s[:contextIdx-1], " ")
+	s = s[contextIdx:]
+
+	// fields are now starting with Context, expect 12 fields
+	if len(s) != 12 || s[3] != "Rcv" || s[6] != "Q" {
 		return nil, nil
 	}
 
-	timestamp, err := time.Parse("2006-01-02 15:04:05", s[0]+" "+s[1])
+	timeFormat := p.TimeFormat
+	if timeFormat == "" {
+		timeFormat = guessTimeFormat(ts)
+	}
+	if timeFormat == "" {
+		return nil, fmt.Errorf("Unknown time format for timestamp: %s", ts)
+	}
+
+	timestamp, err := time.Parse(timeFormat, ts)
 	if err != nil {
 		return nil, err
 	}
 
-	srcIP := net.ParseIP(s[7])
+	srcIP := net.ParseIP(s[4])
 	if err != nil {
 		return nil, fmt.Errorf("invalid source ip at line %s", line)
 	}
 
 	return &packet.DNSPacket{
 		DstPort:    0,
-		Protocol:   strings.ToLower(s[5]),
+		Protocol:   strings.ToLower(s[2]),
 		Timestamp:  timestamp,
 		SrcIP:      srcIP,
-		RecordType: s[13],
-		FQDN:       strings.Trim(numToDotRe.ReplaceAllString(s[14], "."), "."),
+		RecordType: s[10],
+		FQDN:       strings.Trim(numToDotRe.ReplaceAllString(s[11], "."), "."),
 	}, nil
 }
 

logs/msdns/parser_test.go

@@ -45,11 +45,11 @@ Message logging key (for packets - other items use a subset of these fields):
       14     ResponseCode ]
       15     Question Type
       16     Question Name
-2017-01-01 00:00:00 01A0 EVENT   The DNS server did not detect any zones of either primary or secondary type during initialization. It will not be authoritative for any zones, and it will run as a caching-only server until a zone is loaded manually or by Active Directory replication. For more information, see the online Help.
-2017-01-01 00:00:00 01A0 EVENT   The DNS server has started.
-2017-01-01 00:00:00 0DB8 PACKET  0000000001962BB0 UDP Rcv 10.0.0.1   0030   Q [0001   D   NOERROR] A      (8)alphasoc(3)com(0)
-2017-01-01 00:00:00 0DB8 PACKET  0000000001962BB0 UDP Snd 127.0.0.1  0030   Q [0001   D   NOERROR] A      (8)alphasoc(3)com(0)
-2017-01-01 00:00:00 AM 0DB8 PACKET  0000000001962BB0 TCP Rcv 10.0.0.2   0030   Q [0001   D   NOERROR] AAAA   (8)alphasoc(3)net(0)
+2017-01-02 00:00:00 01A0 EVENT   The DNS server did not detect any zones of either primary or secondary type during initialization. It will not be authoritative for any zones, and it will run as a caching-only server until a zone is loaded manually or by Active Directory replication. For more information, see the online Help.
+2017-01-02 00:00:00 01A0 EVENT   The DNS server has started.
+2017-01-02 21:00:00 0DB8 PACKET  0000000001962BB0 UDP Rcv 10.0.0.1   0030   Q [0001   D   NOERROR] A      (8)alphasoc(3)com(0)
+2017-01-02 00:00:00 0DB8 PACKET  0000000001962BB0 UDP Snd 127.0.0.1  0030   Q [0001   D   NOERROR] A      (8)alphasoc(3)com(0)
+2017/01/02 09:00:00 PM 0DB8 PACKET  0000000001962BB0 TCP Rcv 10.0.0.2   0030   Q [0001   D   NOERROR] AAAA   (8)alphasoc(3)net(0)
 `
 	)
 
@@ -73,7 +73,7 @@ Message logging key (for packets - other items use a subset of these fields):
 		t.Fatalf("reading msdns dns package failed - want: 2, got: %d", len(packets))
 	}
 
-	tc := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC)
+	tc := time.Date(2017, 1, 2, 21, 0, 0, 0, time.UTC)
 	if !(packets[0].DstPort == 0 &&
 		packets[0].Protocol == "udp" &&
 		packets[0].Timestamp.Equal(tc) &&