diff --git a/Makefile b/Makefile index 10333211..a48e3ab1 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,11 @@ test: $(GO_VARS) $(GO) test -v ./response $(GO_VARS) $(GO) test -v ./backends $(GO_VARS) $(GO) test -v ./mail + $(GO_VARS) $(GO) test -v ./mail/mimeparse $(GO_VARS) $(GO) test -v ./mail/encoding - $(GO_VARS) $(GO) test -v ./mail/rfc5321 + $(GO_VARS) $(GO) test -v ./mail/iconv + $(GO_VARS) $(GO) test -v ./mail/smtp + $(GO_VARS) $(GO) test -v ./chunk testrace: $(GO_VARS) $(GO) test -v . -race diff --git a/api.go b/api.go index 14cfe0d4..a68b92d5 100644 --- a/api.go +++ b/api.go @@ -13,9 +13,9 @@ import ( // Daemon provides a convenient API when using go-guerrilla as a package in your Go project. // Is's facade for Guerrilla, AppConfig, backends.Backend and log.Logger type Daemon struct { - Config *AppConfig - Logger log.Logger - Backend backends.Backend + Config *AppConfig + Logger log.Logger + Backends []backends.Backend // Guerrilla will be managed through the API g Guerrilla @@ -51,16 +51,11 @@ func (d *Daemon) Start() (err error) { return err } } - if d.Backend == nil { - d.Backend, err = backends.New(d.Config.BackendConfig, d.Logger) - if err != nil { - return err - } - } - d.g, err = New(d.Config, d.Backend, d.Logger) + d.g, err = New(d.Config, d.Logger, d.Backends...) if err != nil { return err } + for i := range d.subs { _ = d.Subscribe(d.subs[i].topic, d.subs[i].fn) @@ -70,9 +65,8 @@ func (d *Daemon) Start() (err error) { err = d.g.Start() if err == nil { if err := d.resetLogger(); err == nil { - d.Log().Infof("main log configured to %s", d.Config.LogFile) + d.Log().Fields("file", d.Config.LogFile).Info("main log configured") } - } return err } @@ -128,7 +122,7 @@ func (d *Daemon) ReloadConfig(c AppConfig) error { d.Log().WithError(err).Error("Error while reloading config") return err } - d.Log().Infof("Configuration was reloaded at %s", d.configLoadTime) + d.Log().Info("configuration was reloaded") d.Config.EmitChangeEvents(&oldConfig, d.g) return nil @@ -143,7 +137,7 @@ func (d *Daemon) ReloadConfigFile(path string) error { } else if d.Config != nil { oldConfig := *d.Config d.Config = &ac - d.Log().Infof("Configuration was reloaded at %s", d.configLoadTime) + d.Log().Info("configuration was reloaded") d.Config.EmitChangeEvents(&oldConfig, d.g) } return nil @@ -155,7 +149,7 @@ func (d *Daemon) ReopenLogs() error { if d.Config == nil { return errors.New("d.Config nil") } - d.Config.EmitLogReopenEvents(d.g) + d.Config.emitLogReopenEvents(d.g) return nil } @@ -217,8 +211,10 @@ func (d *Daemon) configureDefaults() error { if err != nil { return err } - if d.Backend == nil { - err = d.Config.setBackendDefaults() + if d.Backends == nil { + d.Backends = make([]backends.Backend, 0) + // the config will be used to make backends + err = d.Config.BackendConfig.ConfigureDefaults() if err != nil { return err } diff --git a/api_test.go b/api_test.go index 570c47aa..833ad5b9 100644 --- a/api_test.go +++ b/api_test.go @@ -4,16 +4,18 @@ import ( "bufio" "errors" "fmt" - "github.com/flashmob/go-guerrilla/backends" - "github.com/flashmob/go-guerrilla/log" - "github.com/flashmob/go-guerrilla/mail" - "github.com/flashmob/go-guerrilla/response" "io/ioutil" "net" "os" "strings" "testing" "time" + + "github.com/flashmob/go-guerrilla/backends" + _ "github.com/flashmob/go-guerrilla/chunk" + "github.com/flashmob/go-guerrilla/log" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/response" ) // Test Starting smtp without setting up logger / backend @@ -100,11 +102,22 @@ func TestSMTPCustomBackend(t *testing.T) { } cfg.Servers = append(cfg.Servers, sc) bcfg := backends.BackendConfig{ - "save_workers_size": 3, - "save_process": "HeadersParser|Header|Hasher|Debugger", - "log_received_mails": true, - "primary_mail_host": "example.com", + backends.ConfigProcessors: { + "debugger": { + "log_received_mails": true, + }, + "header": { + "primary_mail_host": "example.com", + }, + }, + backends.ConfigGateways: { + "default": { + "save_workers_size": 3, + "save_process": "HeadersParser|Header|Hasher|Debugger", + }, + }, } + cfg.BackendConfig = bcfg d := Daemon{Config: cfg} @@ -124,12 +137,20 @@ func TestSMTPLoadFile(t *testing.T) { "log_level" : "debug", "pid_file" : "tests/go-guerrilla.pid", "allowed_hosts": ["spam4.me","grr.la"], - "backend_config" : - { - "log_received_mails" : true, - "save_process": "HeadersParser|Header|Hasher|Debugger", - "save_workers_size": 3 - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 3, + "save_process": "HeadersParser|Header|Hasher|Debugger" + } + } + }, + "servers" : [ { "is_enabled" : true, @@ -154,12 +175,19 @@ func TestSMTPLoadFile(t *testing.T) { "log_level" : "debug", "pid_file" : "tests/go-guerrilla2.pid", "allowed_hosts": ["spam4.me","grr.la"], - "backend_config" : - { - "log_received_mails" : true, - "save_process": "HeadersParser|Header|Hasher|Debugger", - "save_workers_size": 3 - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 3, + "save_process": "HeadersParser|Header|Hasher|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -295,14 +323,14 @@ func TestReopenServerLog(t *testing.T) { if err != nil { t.Error("start error", err) } else { - if err := talkToServer("127.0.0.1:2526"); err != nil { + if err := talkToServer("127.0.0.1:2526", ""); err != nil { t.Error(err) } if err = d.ReopenLogs(); err != nil { t.Error(err) } time.Sleep(time.Second * 2) - if err := talkToServer("127.0.0.1:2526"); err != nil { + if err := talkToServer("127.0.0.1:2526", ""); err != nil { t.Error(err) } d.Shutdown() @@ -326,7 +354,7 @@ func TestReopenServerLog(t *testing.T) { return } - if !strings.Contains(string(b), "Handle client") { + if !strings.Contains(string(b), "handle client") { t.Error("server log does not contain \"handle client\"") } @@ -401,7 +429,7 @@ func TestSetConfigError(t *testing.T) { err := d.SetConfig(cfg) if err == nil { - t.Error("SetConfig should have returned an error compalning about bad tls settings") + t.Error("SetConfig should have returned an error complaining about bad tls settings") return } } @@ -429,13 +457,10 @@ var funkyLogger = func() backends.Decorator { func(e *mail.Envelope, task backends.SelectTask) (backends.Result, error) { if task == backends.TaskValidateRcpt { // log the last recipient appended to e.Rcpt - backends.Log().Infof( - "another funky recipient [%s]", - e.RcptTo[len(e.RcptTo)-1]) + backends.Log().Fields("recipient", e.RcptTo[len(e.RcptTo)-1]).Info( + "another funky recipient") // if valid then forward call to the next processor in the chain return p.Process(e, task) - // if invalid, return a backend result - //return backends.NewResult(response.Canned.FailRcptCmd), nil } else if task == backends.TaskSaveMail { backends.Log().Info("Another funky email!") } @@ -453,8 +478,12 @@ func TestSetAddProcessor(t *testing.T) { LogFile: "tests/testlog", AllowedHosts: []string{"grr.la"}, BackendConfig: backends.BackendConfig{ - "save_process": "HeadersParser|Debugger|FunkyLogger", - "validate_process": "FunkyLogger", + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger|FunkyLogger", + "validate_process": "FunkyLogger", + }, + }, }, } d := Daemon{Config: cfg} @@ -464,7 +493,7 @@ func TestSetAddProcessor(t *testing.T) { t.Error(err) } // lets have a talk with the server - if err := talkToServer("127.0.0.1:2525"); err != nil { + if err := talkToServer("127.0.0.1:2525", ""); err != nil { t.Error(err) } @@ -493,7 +522,7 @@ func TestSetAddProcessor(t *testing.T) { } -func talkToServer(address string) (err error) { +func talkToServer(address string, body string) (err error) { conn, err := net.Dial("tcp", address) if err != nil { @@ -512,7 +541,7 @@ func talkToServer(address string) (err error) { if err != nil { return err } - _, err = fmt.Fprint(conn, "MAIL FROM:r\r\n") + _, err = fmt.Fprint(conn, "MAIL FROM: BODY=8BITMIME\r\n") if err != nil { return err } @@ -536,26 +565,44 @@ func talkToServer(address string) (err error) { if err != nil { return err } - _, err = fmt.Fprint(conn, "Subject: Test subject\r\n") - if err != nil { - return err - } - _, err = fmt.Fprint(conn, "\r\n") - if err != nil { - return err - } - _, err = fmt.Fprint(conn, "A an email body\r\n") - if err != nil { - return err + if body == "" { + _, err = fmt.Fprint(conn, "Subject: Test subject\r\n") + if err != nil { + return err + } + _, err = fmt.Fprint(conn, "\r\n") + if err != nil { + return err + } + _, err = fmt.Fprint(conn, "A an email body\r\n") + if err != nil { + return err + } + _, err = fmt.Fprint(conn, ".\r\n") + if err != nil { + return err + } + } else { + _, err = fmt.Fprint(conn, body) + if err != nil { + return err + } + _, err = fmt.Fprint(conn, ".\r\n") + if err != nil { + return err + } } - _, err = fmt.Fprint(conn, ".\r\n") + + str, err = in.ReadString('\n') if err != nil { return err } - str, err = in.ReadString('\n') + + _, err = fmt.Fprint(conn, "QUIT\r\n") if err != nil { return err } + _ = str return nil } @@ -577,8 +624,12 @@ func TestReloadConfig(t *testing.T) { LogFile: "tests/testlog", AllowedHosts: []string{"grr.la"}, BackendConfig: backends.BackendConfig{ - "save_process": "HeadersParser|Debugger|FunkyLogger", - "validate_process": "FunkyLogger", + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger|FunkyLogger", + "validate_process": "FunkyLogger", + }, + }, }, } // Look mom, reloading the config without shutting down! @@ -589,8 +640,11 @@ func TestReloadConfig(t *testing.T) { } func TestPubSubAPI(t *testing.T) { + if err := truncateIfExists("tests/testlog"); err != nil { + t.Error(err) + } - if err := os.Truncate("tests/testlog", 0); err != nil { + if err := truncateIfExists("tests/pidfilex.pid"); err != nil { t.Error(err) } @@ -605,8 +659,12 @@ func TestPubSubAPI(t *testing.T) { LogFile: "tests/testlog", AllowedHosts: []string{"grr.la"}, BackendConfig: backends.BackendConfig{ - "save_process": "HeadersParser|Debugger|FunkyLogger", - "validate_process": "FunkyLogger", + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger|FunkyLogger", + "validate_process": "FunkyLogger", + }, + }, }, } @@ -742,18 +800,24 @@ func TestCustomBackendResult(t *testing.T) { LogFile: "tests/testlog", AllowedHosts: []string{"grr.la"}, BackendConfig: backends.BackendConfig{ - "save_process": "HeadersParser|Debugger|Custom", - "validate_process": "Custom", + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger|Custom", + "validate_process": "Custom", + }, + }, }, } + d := Daemon{Config: cfg} d.AddProcessor("Custom", customBackend2) if err := d.Start(); err != nil { t.Error(err) + return } // lets have a talk with the server - if err := talkToServer("127.0.0.1:2525"); err != nil { + if err := talkToServer("127.0.0.1:2525", ""); err != nil { t.Error(err) } @@ -774,3 +838,761 @@ func TestCustomBackendResult(t *testing.T) { } } + +// Test a backends removed, 2 new backends added added +func TestBackendAddRemove(t *testing.T) { + + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + + servers := []ServerConfig{ + 0: { + IsEnabled: true, + Hostname: "mail.guerrillamail.com", + MaxSize: 100017, + Timeout: 160, + ListenInterface: "127.0.0.1:2526", + MaxClients: 2, + TLS: ServerTLSConfig{ + PrivateKeyFile: "", + PublicKeyFile: "", + StartTLSOn: false, + AlwaysOn: false, + }, + }, + } + + cfg := &AppConfig{ + LogFile: "tests/testlog", + PidFile: "tests/go-guerrilla.pid", + AllowedHosts: []string{"grr.la", "spam4.me"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger|Custom", + "validate_process": "Custom", + }, + "temp": { + "save_process": "HeadersParser|Debugger|Custom", + "validate_process": "Custom", + }, + }, + }, + Servers: servers, + } + + d := Daemon{Config: cfg} + d.AddProcessor("Custom", customBackend2) + + if err := d.Start(); err != nil { + t.Error(err) + return + } + + cfg2 := *cfg + cfg2.BackendConfig = backends.BackendConfig{ + backends.ConfigGateways: { + "client1": { + "save_process": "HeadersParser|Debugger|Custom", + "validate_process": "Custom", + }, + "client2": { + "save_process": "HeadersParser|Debugger", + "validate_process": "Custom", + }, + }, + } + + eventFiredAdded := false + _ = d.Subscribe(EventConfigBackendConfigAdded, backendEvent(func(appConfig *AppConfig, name string) { + eventFiredAdded = true + })) + + eventFiredRemoved := false + _ = d.Subscribe(EventConfigBackendConfigRemoved, backendEvent(func(appConfig *AppConfig, name string) { + eventFiredRemoved = true + })) + + // default changed, temp removed, client1 and client2 added + + if err := d.ReloadConfig(cfg2); err != nil { + t.Error(err) + return + } + + d.Shutdown() + + if eventFiredAdded == false { + t.Error("EventConfigBackendConfigAdded did not fired") + } + if eventFiredRemoved == false { + t.Error("EventConfigBackendConfigRemoved did not get fired") + } + +} + +func TestStreamProcessorConfig(t *testing.T) { + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + + servers := []ServerConfig{ + 0: { + IsEnabled: true, + Hostname: "mail.guerrillamail.com", + MaxSize: 100017, + Timeout: 160, + ListenInterface: "127.0.0.1:2526", + MaxClients: 2, + TLS: ServerTLSConfig{ + PrivateKeyFile: "", + PublicKeyFile: "", + StartTLSOn: false, + AlwaysOn: false, + }, + }, + } + + cfg := &AppConfig{ + LogFile: "tests/testlog", + PidFile: "tests/go-guerrilla.pid", + AllowedHosts: []string{"grr.la", "spam4.me"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "chunkSaver": { // note mixed case + "chunk_size": 8000, + "storage_engine": "memory", + "compress_level": 0, + }, + "test:chunksaver": { + "chunk_size": 8000, + "storage_engine": "memory", + "compress_level": 0, + }, + "debug": { + "sleep_seconds": 2, + "log_reads": true, + }, + }, + backends.ConfigGateways: { + "default": { + "save_stream": "mimeanalyzer|chunksaver|test|debug", + }, + }, + }, + Servers: servers, + } + + d := Daemon{Config: cfg} + if err := d.Start(); err != nil { + t.Error(err) + return + } + + d.Shutdown() + +} + +func TestStreamProcessor(t *testing.T) { + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + cfg := &AppConfig{ + LogFile: "tests/testlog", + AllowedHosts: []string{"grr.la"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "debug": { + "log_reads": true, + }, + }, + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger", + "save_stream": "Header|headersparser|compress|Decompress|debug", + "post_process": "Header|headersparser|compress|Decompress|debug", + }, + }, + }, + } + d := Daemon{Config: cfg} + + if err := d.Start(); err != nil { + t.Error(err) + } + body := "Subject: Test subject\r\n" + + //"\r\n" + + "A an email body.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + // lets have a talk with the server + if err := talkToServer("127.0.0.1:2525", body); err != nil { + t.Error(err) + } + + d.Shutdown() + + b, err := ioutil.ReadFile("tests/testlog") + if err != nil { + t.Error("could not read logfile") + return + } + + // lets check for fingerprints + if strings.Index(string(b), "debug stream") < 0 { + t.Error("did not log: Debug stream") + } + + if strings.Index(string(b), "Error") != -1 { + t.Error("There was an error", string(b)) + } + +} + +func TestStreamProcessorBackground(t *testing.T) { + + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + cfg := &AppConfig{ + LogFile: "tests/testlog", + AllowedHosts: []string{"grr.la"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "debug": { + "log_reads": true, + }, + "moo:chunksaver": { + "chunk_size": 8000, + "storage_engine": "memory", + "compress_level": 0, + }, + }, + backends.ConfigGateways: { + "default": { + "save_process": "", + "save_stream": "mimeanalyzer|moo", + "post_process_consumer": "Header|headersparser|compress|Decompress|debug", + "post_process_producer": "moo", + "post_process_size": 100, + "stream_buffer_size": 1024, + "save_workers_size": 8, + "save_timeout": "20s", + "val_rcpt_timeout": "2s", + }, + }, + }, + } + d := Daemon{Config: cfg} + + if err := d.Start(); err != nil { + t.Error(err) + } + body := "Subject: Test subject\r\n" + + "\r\n" + + "A an email body.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + "Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" + + // lets have a talk with the server + if err := talkToServer("127.0.0.1:2525", body); err != nil { + t.Error(err) + } + + d.Shutdown() + + b, err := ioutil.ReadFile("tests/testlog") + if err != nil { + t.Error("could not read logfile") + return + } + time.Sleep(time.Second * 2) + // lets check for fingerprints + if strings.Index(string(b), "debug stream") < 0 { + t.Error("did not log: Debug stream") + } + + if strings.Index(string(b), "background process done") < 0 { + t.Error("did not log: background process done") + } + + if strings.Index(string(b), "Error") != -1 { + t.Error("There was an error", string(b)) + } + +} + +var mime0 = `MIME-Version: 1.0 +X-Mailer: MailBee.NET 8.0.4.428 +Subject: test + subject +To: kevinm@datamotion.com +Content-Type: multipart/mixed; + boundary="XXXXboundary text" + +--XXXXboundary text +Content-Type: multipart/alternative; + boundary="XXXXboundary text" + +--XXXXboundary text +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +This is the body text of a sample message. +--XXXXboundary text +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +
This is the body text of a sample message.
+ +--XXXXboundary text +Content-Type: text/plain; + name="log_attachment.txt" +Content-Disposition: attachment; + filename="log_attachment.txt" +Content-Transfer-Encoding: base64 + +TUlNRS1WZXJzaW9uOiAxLjANClgtTWFpbGVyOiBNYWlsQmVlLk5FVCA4LjAuNC40MjgNClN1Ympl +Y3Q6IHRlc3Qgc3ViamVjdA0KVG86IGtldmlubUBkYXRhbW90aW9uLmNvbQ0KQ29udGVudC1UeXBl +OiBtdWx0aXBhcnQvYWx0ZXJuYXRpdmU7DQoJYm91bmRhcnk9Ii0tLS09X05leHRQYXJ0XzAwMF9B +RTZCXzcyNUUwOUFGLjg4QjdGOTM0Ig0KDQoNCi0tLS0tLT1fTmV4dFBhcnRfMDAwX0FFNkJfNzI1 +RTA5QUYuODhCN0Y5MzQNCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsNCgljaGFyc2V0PSJ1dGYt +OCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KdGVzdCBi +b2R5DQotLS0tLS09X05leHRQYXJ0XzAwMF9BRTZCXzcyNUUwOUFGLjg4QjdGOTM0DQpDb250ZW50 +LVR5cGU6IHRleHQvaHRtbDsNCgljaGFyc2V0PSJ1dGYtOCINCkNvbnRlbnQtVHJhbnNmZXItRW5j +b2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KPHByZT50ZXN0IGJvZHk8L3ByZT4NCi0tLS0tLT1f +TmV4dFBhcnRfMDAwX0FFNkJfNzI1RTA5QUYuODhCN0Y5MzQtLQ0K +--XXXXboundary text-- +` + +var mime2 = `From: abc@def.de +Content-Type: multipart/mixed; + boundary="----_=_NextPart_001_01CBE273.65A0E7AA" +To: ghi@def.de + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01CBE273.65A0E7AA +Content-Type: multipart/alternative; + boundary="----_=_NextPart_002_01CBE273.65A0E7AA" + + +------_=_NextPart_002_01CBE273.65A0E7AA +Content-Type: text/plain; + charset="UTF-8" +Content-Transfer-Encoding: base64 + +[base64-content] +------_=_NextPart_002_01CBE273.65A0E7AA +Content-Type: text/html; + charset="UTF-8" +Content-Transfer-Encoding: base64 + +[base64-content] +------_=_NextPart_002_01CBE273.65A0E7AA-- +------_=_NextPart_001_01CBE273.65A0E7AA +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +X-MimeOLE: Produced By Microsoft Exchange V6.5 +Content-class: urn:content-classes:message +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----_=_NextPart_003_01CBE272.13692C80" +From: bla@bla.de +To: xxx@xxx.de + +This is a multi-part message in MIME format. + +------_=_NextPart_003_01CBE272.13692C80 +Content-Type: multipart/alternative; + boundary="----_=_NextPart_004_01CBE272.13692C80" + + +------_=_NextPart_004_01CBE272.13692C80 +Content-Type: text/plain; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +=20 + +Viele Gr=FC=DFe + +------_=_NextPart_004_01CBE272.13692C80 +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +... +------_=_NextPart_004_01CBE272.13692C80-- +------_=_NextPart_003_01CBE272.13692C80 +Content-Type: application/x-zip-compressed; + name="abc.zip" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="abc.zip" + +[base64-content] + +------_=_NextPart_003_01CBE272.13692C80-- +------_=_NextPart_001_01CBE273.65A0E7AA-- +` + +var mime3 = `From lara_lars@hotmail.com Mon Feb 19 22:24:21 2001 +Received: from [137.154.210.66] by hotmail.com (3.2) with ESMTP id MHotMailBC5B58230039400431D5899AD24289FA0; Mon Feb 19 22:22:29 2001 +Received: from lancelot.cit.nepean.uws.edu.au (lancelot.cit.uws.edu.au [137.154.148.30]) + by day.uws.edu.au (8.11.1/8.11.1) with ESMTP id f1K6MN404936; + Tue, 20 Feb 2001 17:22:24 +1100 (EST) +Received: from hotmail.com (law2-f35.hotmail.com [216.32.181.35]) + by lancelot.cit.nepean.uws.edu.au (8.10.0.Beta10/8.10.0.Beta10) with ESMTP id f1K6MJb13619; + Tue, 20 Feb 2001 17:22:19 +1100 (EST) +Received: from mail pickup service by hotmail.com with Microsoft SMTPSVC; + Mon, 19 Feb 2001 22:21:44 -0800 +Received: from 203.54.221.89 by lw2fd.hotmail.msn.com with HTTP; Tue, 20 Feb 2001 06:21:44 GMT +X-Originating-IP: [203.54.221.89] +From: "lara devine" +To: amalinow@cit.nepean.uws.edu.au, transmission_@hotmail.com, + lalexand@cit.nepean.uws.edu.au, dconroy@cit.nepean.uws.edu.au, + pumpkin7@bigpond.com, jwalker@cit.nepean.uws.edu.au, + dgoerge@cit.nepean.uws.edu.au, batty_horny@hotmail.com, + ikvesic@start.com.au +Subject: Fwd: Goldfish +Date: Tue, 20 Feb 2001 06:21:44 +Mime-Version: 1.0 +Content-Type: text/plain; format=flowed +Message-ID: +X-OriginalArrivalTime: 20 Feb 2001 06:21:44.0718 (UTC) FILETIME=[658BDAE0:01C09B05] + + + + +>> >Two builders (Chris and James) are seated either side of a table in a +> > >rough +> > >pub when a well-dressed man enters, orders beer and sits on a stool at +> > >the bar. +> > >The two builders start to speculate about the occupation of the suit. +> > > +> > >Chris: - I reckon he's an accountant. +> > > +> > >James: - No way - he's a stockbroker. +> > > +> > >Chris: - He ain't no stockbroker! A stockbroker wouldn't come in here! +> > > +> > >The argument repeats itself for some time until the volume of beer gets +> > >the better of Chris and he makes for the toilet. On entering the toilet +> > >he +> > >sees that the suit is standing at a urinal. Curiosity and the several +> > >beers +> > >get the better of the builder... +> > > +> > >Chris: - 'scuse me.... no offence meant, but me and me mate were +> > wondering +> > > +> > > what you do for a living? +> > > +> > >Suit: - No offence taken! I'm a Logical Scientist by profession! +> > > +> > >Chris: - Oh! What's that then? +> > > +> > >Suit:- I'll try to explain by example... Do you have a goldfish at +>home? +> > > +> > >Chris:- Er...mmm... well yeah, I do as it happens! +> > > +> > >Suit: - Well, it's logical to follow that you keep it in a bowl or in a +> > >pond. Which is it? +> > > +> > >Chris: - It's in a pond! +> > > +> > >Suit: - Well then it's reasonable to suppose that you have a large +> > >garden +> > >then? +> > > +> > >Chris: - As it happens, yes I have got a big garden! +> > > +> > >Suit: - Well then it's logical to assume that in this town that if you +> > >have a large garden that you have a large house? +> > > +> > >Chris: - As it happens I've got a five bedroom house... built it +>myself! +> > > +> > >Suit: - Well given that you've built a five-bedroom house it is logical +> > >to asume that you haven't built it just for yourself and that you are +> > >quite +> > >probably married? +> > > +> > >Chris: - Yes I am married, I live with my wife and three children! +> > > +> > >Suit: - Well then it is logical to assume that you are sexually active +> > >with your wife on a regular basis? +> > > +> > >Chris:- Yep! Four nights a week! +> > > +> > >Suit: - Well then it is logical to suggest that you do not masturbate +> > >very often? +> > > +> > >Chris: - Me? Never. +> > > +> > >Suit: - Well there you are! That's logical science at work! +> > > +> > >Chris:- How's that then? +> > > +> > >Suit: - Well from finding out that you had a goldfish, I've told you +> > >about the size of garden you have, size of house, your family and your +> > >sex +> > >life! +> > > +> > >Chris: - I see! That's pretty impressive... thanks mate! +> > > +> > >Both leave the toilet and Chris returns to his mate. +> > > +> > >James: - I see the suit was in there. Did you ask him what he does? +> > > +> > >Chris: - Yep! He's a logical scientist! +> > > +> > >James: What's a logical Scientist? +> > > +> > >Chris: - I'll try and explain. Do you have a goldfish? +> > > +> > >James: - Nope. +> > > +> > >Chris: - Well then, you're a wanker. +> + +_________________________________________________________________________ +Get Your Private, Free E-mail from MSN Hotmail at http://www.hotmail.com. +` + +/* +1 0 166 1514 +1.1 186 260 259 +1.2 280 374 416 +1.3 437 530 584 +1.4 605 769 1514 +*/ +func TestStreamMimeProcessor(t *testing.T) { + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + cfg := &AppConfig{ + LogFile: "tests/testlog", + AllowedHosts: []string{"grr.la"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "debug": { + "log_reads": true, + }, + }, + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger", + "save_stream": "mimeanalyzer|headersparser|compress|Decompress|debug", + }, + }, + }, + } + d := Daemon{Config: cfg} + + if err := d.Start(); err != nil { + t.Error(err) + } + + go func() { + time.Sleep(time.Second * 15) + // for debugging deadlocks + //pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + //os.Exit(1) + }() + + // change \n to \r\n + mime0 = strings.Replace(mime2, "\n", "\r\n", -1) + // lets have a talk with the server + if err := talkToServer("127.0.0.1:2525", mime0); err != nil { + t.Error(err) + } + + d.Shutdown() + + b, err := ioutil.ReadFile("tests/testlog") + if err != nil { + t.Error("could not read logfile") + return + } + + // lets check for fingerprints + if strings.Index(string(b), "debug stream") < 0 { + t.Error("did not log: Debug stream") + } + + if strings.Index(string(b), "Error") != -1 { + t.Error("There was an error", string(b)) + } + +} + +var email = `From: Al Gore +To: White House Transportation Coordinator +Subject: [Fwd: Map of Argentina with Description] +MIME-Version: 1.0 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=ncr424; d=reliancegeneral.co.in; + h=List-Unsubscribe:MIME-Version:From:To:Reply-To:Date:Subject:Content-Type:Content-Transfer-Encoding:Message-ID; i=prospects@prospects.reliancegeneral.co.in; + bh=F4UQPGEkpmh54C7v3DL8mm2db1QhZU4gRHR1jDqffG8=; + b=MVltcq6/I9b218a370fuNFLNinR9zQcdBSmzttFkZ7TvV2mOsGrzrwORT8PKYq4KNJNOLBahswXf + GwaMjDKT/5TXzegdX/L3f/X4bMAEO1einn+nUkVGLK4zVQus+KGqm4oP7uVXjqp70PWXScyWWkbT + 1PGUwRfPd/HTJG5IUqs= +Content-Type: multipart/mixed; + boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" + +This is a multi-part message in MIME format. +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Fred, + +Fire up Air Force One! We're going South! + +Thanks, +Al +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Return-Path: +Received: from mailhost.whitehouse.gov ([192.168.51.200]) + by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 + for ; + Mon, 13 Aug 1998 l8:14:23 +1000 +Received: from the_big_box.whitehouse.gov ([192.168.51.50]) + by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 + for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 + Date: Mon, 13 Aug 1998 17:42:41 +1000 +Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> +From: Bill Clinton +To: A1 (The Enforcer) Gore +Subject: Map of Argentina with Description +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="DC8------------DC8638F443D87A7F0726DEF7" + +This is a multi-part message in MIME format. +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Hi A1, + +I finally figured out this MIME thing. Pretty cool. I'll send you +some sax music in .au files next week! + +Anyway, the attached image is really too small to get a good look at +Argentina. Try this for a much better map: + +http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm + +Then again, shouldn't the CIA have something like that? + +Bill +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: image/gif; name="map_of_Argentina.gif" +Content-Transfer-Encoding: base64 +Content-Disposition: inline; filename="map_of_Argentina.gif" + +R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w +wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad +GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow +BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX +U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz +7itICBxISKDBgwgTKjyYAAA7 +--DC8------------DC8638F443D87A7F0726DEF7-- + +--D7F------------D7FD5A0B8AB9C65CCDBFA872-- + +` + +func TestStreamChunkSaver(t *testing.T) { + if err := os.Truncate("tests/testlog", 0); err != nil { + t.Error(err) + } + + go func() { + time.Sleep(time.Second * 15) + // for debugging deadlocks + //pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + //os.Exit(1) + }() + + cfg := &AppConfig{ + LogFile: "tests/testlog", + AllowedHosts: []string{"grr.la"}, + BackendConfig: backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "chunksaver": { + "chunk_size": 1024 * 32, + "stream_buffer_size": 1024 * 16, + "storage_engine": "memory", + }, + }, + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger", + "save_stream": "mimeanalyzer|chunksaver", + "save_timeout": "5", + }, + }, + }, + } + + d := Daemon{Config: cfg} + + if err := d.Start(); err != nil { + t.Error(err) + } + + // change \n to \r\n + email = strings.Replace(email, "\n", "\r\n", -1) + // lets have a talk with the server + if err := talkToServer("127.0.0.1:2525", email); err != nil { + t.Error(err) + } + time.Sleep(time.Second * 1) + d.Shutdown() + + b, err := ioutil.ReadFile("tests/testlog") + if err != nil { + t.Error("could not read logfile") + return + } + fmt.Println(string(b)) + + // lets check for fingerprints + if strings.Index(string(b), "Debug stream") < 0 { + // t.Error("did not log: Debug stream") + } + +} diff --git a/backends/backend.go b/backends/backend.go index 48a3051c..acd4cf9b 100644 --- a/backends/backend.go +++ b/backends/backend.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/flashmob/go-guerrilla/log" "github.com/flashmob/go-guerrilla/mail" + "io" "reflect" "strconv" "strings" @@ -15,19 +16,25 @@ import ( var ( Svc *service - // Store the constructor for making an new processor decorator. + // processors store the constructors for for composing a new processor using a decorator pattern. processors map[string]ProcessorConstructor - b Backend + // Streamers store the constructors for composing a new stream-based processor using a decorator pattern. + Streamers map[string]StreamProcessorConstructor ) func init() { Svc = &service{} processors = make(map[string]ProcessorConstructor) + Streamers = make(map[string]StreamProcessorConstructor) } +const DefaultGateway = "default" + type ProcessorConstructor func() Decorator +type StreamProcessorConstructor func() *StreamDecorator + // Backends process received mail. Depending on the implementation, they can store mail in the database, // write to a file, check for spam, re-transmit to another server, etc. // Must return an SMTP message (i.e. "250 OK") and a boolean indicating @@ -37,6 +44,11 @@ type Backend interface { Process(*mail.Envelope) Result // ValidateRcpt validates the last recipient that was pushed to the mail envelope ValidateRcpt(e *mail.Envelope) RcptError + ProcessBackground(e *mail.Envelope) + // ProcessStream is the alternative for Process, a stream is read from io.Reader + ProcessStream(r io.Reader, e *mail.Envelope) (Result, int64, error) + // StreamOn signals if ProcessStream can be used + StreamOn() bool // Initializes the backend, eg. creates folders, sets-up database connections Initialize(BackendConfig) error // Initializes the backend after it was Shutdown() @@ -45,13 +57,10 @@ type Backend interface { Shutdown() error // Start Starts a backend that has been initialized Start() error + // returns the name of the backend + Name() string } -type BackendConfig map[string]interface{} - -// All config structs extend from this -type BaseConfig interface{} - type notifyMsg struct { err error queuedID string @@ -91,19 +100,19 @@ func (r *result) Code() int { return code } -func NewResult(r ...interface{}) Result { - buf := new(result) - for _, item := range r { +func NewResult(param ...interface{}) Result { + r := new(result) + for _, item := range param { switch v := item.(type) { case error: - _, _ = buf.WriteString(v.Error()) + _, _ = r.WriteString(v.Error()) case fmt.Stringer: - _, _ = buf.WriteString(v.String()) + _, _ = r.WriteString(v.String()) case string: - _, _ = buf.WriteString(v) + _, _ = r.WriteString(v) } } - return buf + return r } type processorInitializer interface { @@ -169,7 +178,7 @@ func (s *service) SetMainlog(l log.Logger) { s.mainlog.Store(l) } -// AddInitializer adds a function that implements ProcessorShutdowner to be called when initializing +// AddInitializer adds a function that implements processorInitializer to be called when initializing func (s *service) AddInitializer(i processorInitializer) { s.Lock() defer s.Unlock() @@ -192,10 +201,10 @@ func (s *service) reset() { // Initialize initializes all the processors one-by-one and returns any errors. // Subsequent calls to Initialize will not call the initializer again unless it failed on the previous call // so Initialize may be called again to retry after getting errors -func (s *service) initialize(backend BackendConfig) Errors { +func (s *service) Initialize(backend BackendConfig) Errors { s.Lock() defer s.Unlock() - var errors Errors + var errors Errors = nil failed := make([]processorInitializer, 0) for i := range s.initializers { if err := s.initializers[i].Initialize(backend); err != nil { @@ -236,17 +245,34 @@ func (s *service) AddProcessor(name string, p ProcessorConstructor) { c = func() Decorator { return p() } - // add to our processors list processors[strings.ToLower(name)] = c } +func (s *service) AddStreamProcessor(name string, p StreamProcessorConstructor) { + // wrap in a constructor since we want to defer calling it + var c StreamProcessorConstructor + c = func() *StreamDecorator { + return p() + } + Streamers[strings.ToLower(name)] = c +} + // extractConfig loads the backend config. It has already been unmarshalled -// configData contains data from the main config file's "backend_config" value +// "group" refers +// cfg contains data from the main config file's "backend_config" value // configType is a Processor's specific config value. // The reason why using reflection is because we'll get a nice error message if the field is missing // the alternative solution would be to json.Marshal() and json.Unmarshal() however that will not give us any // error messages -func (s *service) ExtractConfig(configData BackendConfig, configType BaseConfig) (interface{}, error) { +func (s *service) ExtractConfig(section ConfigSection, group string, cfg BackendConfig, configType BaseConfig) (interface{}, error) { + group = strings.ToLower(group) + + var configData ConfigGroup + if v, ok := cfg[section][group]; ok { + configData = v + } else { + return configType, nil + } // Use reflection so that we can provide a nice error message v := reflect.ValueOf(configType).Elem() // so that we can set the values //m := reflect.ValueOf(configType).Elem() diff --git a/backends/config.go b/backends/config.go new file mode 100644 index 00000000..fb42233c --- /dev/null +++ b/backends/config.go @@ -0,0 +1,486 @@ +package backends + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "reflect" + "strings" + "time" +) + +type ConfigGroup map[string]interface{} + +type BackendConfig map[ConfigSection]map[string]ConfigGroup + +const ( + validateRcptTimeout = time.Second * 5 + defaultProcessor = "Debugger" + + // streamBufferSize sets the size of the buffer for the streaming processors, + // can be configured using `stream_buffer_size` + configStreamBufferSize = 4096 + configSaveWorkersCount = 1 + configValidateWorkersCount = 1 + configStreamWorkersCount = 1 + configBackgroundWorkersCount = 1 + configSaveProcessSize = 64 + configValidateProcessSize = 64 + // configTimeoutSave: default timeout for saving email, if 'save_timeout' not present in config + configTimeoutSave = time.Second * 30 + // configTimeoutValidateRcpt default timeout for validating rcpt to, if 'val_rcpt_timeout' not present in config + configTimeoutValidateRcpt = time.Second * 5 + configTimeoutStream = time.Second * 30 + configSaveStreamSize = 64 + configPostProcessSize = 64 +) + +func (c *BackendConfig) SetValue(section ConfigSection, name string, key string, value interface{}) { + if *c == nil { + *c = make(BackendConfig, 0) + } + if (*c)[section] == nil { + (*c)[section] = make(map[string]ConfigGroup) + } + if (*c)[section][name] == nil { + (*c)[section][name] = make(ConfigGroup) + } + (*c)[section][name][key] = value +} + +func (c *BackendConfig) GetValue(section ConfigSection, name string, key string) interface{} { + if (*c)[section] == nil { + return nil + } + if (*c)[section][name] == nil { + return nil + } + if v, ok := (*c)[section][name][key]; ok { + return &v + } + return nil +} + +// toLower normalizes the backendconfig lowercases the config's keys +func (c BackendConfig) toLower() { + for section, v := range c { + for k2, v2 := range v { + if k2_lower := strings.ToLower(k2); k2 != k2_lower { + c[section][k2_lower] = v2 + delete(c[section], k2) // delete the non-lowercased key + } + } + } +} + +func (c BackendConfig) lookupGroup(section ConfigSection, name string) ConfigGroup { + if v, ok := c[section][name]; ok { + return v + } + return nil +} + +// ConfigureDefaults sets default values for the backend config, +// if no backend config was added before starting, then use a default config +// otherwise, see what required values were missed in the config and add any missing with defaults +func (c *BackendConfig) ConfigureDefaults() error { + // set the defaults if no value has been configured + // (always use lowercase) + if c.GetValue(ConfigGateways, "default", "save_workers_size") == nil { + c.SetValue(ConfigGateways, "default", "save_workers_size", 1) + } + if c.GetValue(ConfigGateways, "default", "save_process") == nil { + c.SetValue(ConfigGateways, "default", "save_process", "HeadersParser|Header|Debugger") + } + if c.GetValue(ConfigProcessors, "header", "primary_mail_host") == nil { + h, err := os.Hostname() + if err != nil { + return err + } + c.SetValue(ConfigProcessors, "header", "primary_mail_host", h) + } + if c.GetValue(ConfigProcessors, "debugger", "log_received_mails") == nil { + c.SetValue(ConfigProcessors, "debugger", "log_received_mails", true) + } + return nil +} + +// UnmarshalJSON custom handling of the ConfigSection keys (they're enumerated) +func (c *BackendConfig) UnmarshalJSON(b []byte) error { + temp := make(map[string]map[string]ConfigGroup) + err := json.Unmarshal(b, &temp) + if err != nil { + return err + } + if *c == nil { + *c = make(BackendConfig) + } + for key, val := range temp { + // map the key to a ConfigSection type + var section ConfigSection + if err := json.Unmarshal([]byte("\""+key+"\""), §ion); err != nil { + return err + } + if (*c)[section] == nil { + (*c)[section] = make(map[string]ConfigGroup) + } + (*c)[section] = val + } + return nil + +} + +// MarshalJSON custom handling of ConfigSection keys (since JSON keys need to be strings) +func (c *BackendConfig) MarshalJSON() ([]byte, error) { + temp := make(map[string]map[string]ConfigGroup) + for key, val := range *c { + // convert they key to a string + temp[key.String()] = val + } + return json.Marshal(temp) +} + +type ConfigSection int + +const ( + ConfigProcessors ConfigSection = iota + ConfigStreamProcessors + ConfigGateways +) + +func (o ConfigSection) String() string { + switch o { + case ConfigProcessors: + return "processors" + case ConfigStreamProcessors: + return "stream_processors" + case ConfigGateways: + return "gateways" + } + return "unknown" +} + +func (o *ConfigSection) UnmarshalJSON(b []byte) error { + str := strings.Trim(string(b), `"`) + str = strings.ToLower(str) + switch { + case str == "processors": + *o = ConfigProcessors + case str == "stream_processors": + *o = ConfigStreamProcessors + case str == "gateways": + *o = ConfigGateways + default: + return errors.New("incorrect config section [" + str + "], may be processors, stream_processors or gateways") + } + return nil +} + +func (o *ConfigSection) MarshalJSON() ([]byte, error) { + ret := o.String() + if ret == "unknown" { + return []byte{}, errors.New("unknown config section") + } + return []byte(ret), nil +} + +// All config structs extend from this +type BaseConfig interface{} + +type stackConfigExpression struct { + alias string + name string +} + +func (e stackConfigExpression) String() string { + if e.alias == e.name || e.alias == "" { + return e.name + } + return fmt.Sprintf("%s:%s", e.alias, e.name) +} + +type notFoundError func(s string) error + +type stackConfig struct { + list []stackConfigExpression + notFound notFoundError +} + +type aliasMap map[string]string + +// newAliasMap scans through the configured processors to produce a mapping +// alias -> processor name. This mapping is used to determine what configuration to use +// when making a new processor +func newAliasMap(cfg map[string]ConfigGroup) aliasMap { + am := make(aliasMap, 0) + for k, _ := range cfg { + var alias, name string + // format: : + if i := strings.Index(k, ":"); i > 0 && len(k) > i+2 { + alias = k[0:i] + name = k[i+1:] + } else { + alias = k + name = k + } + am[strings.ToLower(alias)] = strings.ToLower(name) + } + return am +} + +func NewStackConfig(config string, am aliasMap) (ret *stackConfig) { + ret = new(stackConfig) + cfg := strings.ToLower(strings.TrimSpace(config)) + if cfg == "" { + return + } + items := strings.Split(cfg, "|") + ret.list = make([]stackConfigExpression, len(items)) + pos := 0 + for i := range items { + pos = len(items) - 1 - i // reverse order, since decorators are stacked + ret.list[i] = stackConfigExpression{alias: items[pos], name: items[pos]} + if processor, ok := am[items[pos]]; ok { + ret.list[i].name = processor + } + } + return ret +} + +func newStackProcessorConfig(config string, am aliasMap) (ret *stackConfig) { + ret = NewStackConfig(config, am) + ret.notFound = func(s string) error { + return errors.New(fmt.Sprintf("processor [%s] not found", s)) + } + return ret +} + +func newStackStreamProcessorConfig(config string, am aliasMap) (ret *stackConfig) { + ret = NewStackConfig(config, am) + ret.notFound = func(s string) error { + return errors.New(fmt.Sprintf("stream processor [%s] not found", s)) + } + return ret +} + +// Changes returns a list of gateways whose config changed +func (c BackendConfig) Changes(oldConfig BackendConfig) (changed, added, removed map[string]bool) { + // check the processors if changed + changed = make(map[string]bool, 0) + added = make(map[string]bool, 0) + removed = make(map[string]bool, 0) + cp := ConfigProcessors + csp := ConfigStreamProcessors + cg := ConfigGateways + changedProcessors := changedConfigGroups( + oldConfig[cp], c[cp]) + changedStreamProcessors := changedConfigGroups( + oldConfig[csp], c[csp]) + configType := BaseConfig(&GatewayConfig{}) + aliasMapStream := newAliasMap(c[csp]) + aliasMapProcessor := newAliasMap(c[cp]) + // oldList keeps a track of gateways that have been compared for changes. + // We remove the from the list when a gateway was processed + // remaining items are assumed to be removed from the new config + oldList := map[string]bool{} + for g := range oldConfig[cg] { + oldList[g] = true + } + // go through all the gateway configs, + // make a list of all the ones that have processors whose config had changed + for key, _ := range c[cg] { + // check the processors in the SaveProcess and SaveStream settings to see if + // they changed. If changed, then make a record of which gateways use the processors + e, _ := Svc.ExtractConfig(ConfigGateways, key, c, configType) + bcfg := e.(*GatewayConfig) + config := NewStackConfig(bcfg.SaveProcess, aliasMapProcessor) + for _, v := range config.list { + if _, ok := changedProcessors[v.name]; ok { + changed[key] = true + } + } + + config = NewStackConfig(bcfg.SaveStream, aliasMapStream) + for _, v := range config.list { + if _, ok := changedStreamProcessors[v.name]; ok { + changed[key] = true + } + } + if o, ok := oldConfig[cg][key]; ok { + delete(oldList, key) + if !reflect.DeepEqual(c[cg][key], o) { + // whats changed + changed[key] = true + } + } else { + // whats been added + added[key] = true + } + } + // whats been removed + for p := range oldList { + removed[p] = true + } + return +} + +func changedConfigGroups(old map[string]ConfigGroup, new map[string]ConfigGroup) map[string]bool { + diff, added, removed := compareConfigGroup(old, new) + var all []string + all = append(all, diff...) + all = append(all, removed...) + all = append(all, added...) + changed := make(map[string]bool, 0) + for p := range all { + changed[strings.ToLower(all[p])] = true + } + return changed +} + +// compareConfigGroup compares two config groups +// returns a list of keys that changed, been added or removed to new +func compareConfigGroup(old map[string]ConfigGroup, new map[string]ConfigGroup) (diff, added, removed []string) { + diff = make([]string, 0) + added = make([]string, 0) + removed = make([]string, 0) + for p := range new { + if o, ok := old[p]; ok { + delete(old, p) + if !reflect.DeepEqual(new[p], o) { + // whats changed + diff = append(diff, p) + } + } else { + // whats been added + added = append(added, p) + } + } + // whats been removed + for p := range old { + removed = append(removed, p) + } + return +} + +type GatewayConfig struct { + // SaveWorkersCount controls how many concurrent workers to start. Defaults to 1 + SaveWorkersCount int `json:"save_workers_size,omitempty"` + // ValidateWorkersCount controls how many concurrent recipient validation workers to start. Defaults to 1 + ValidateWorkersCount int `json:"validate_workers_size,omitempty"` + // StreamWorkersCount controls how many concurrent stream workers to start. Defaults to 1 + StreamWorkersCount int `json:"stream_workers_size,omitempty"` + // BackgroundWorkersCount controls how many concurrent background stream workers to start. Defaults to 1 + BackgroundWorkersCount int `json:"background_workers_size,omitempty"` + + // SaveProcess controls which processors to chain in a stack for saving email tasks + SaveProcess string `json:"save_process,omitempty"` + // SaveProcessSize limits the amount of messages waiting in the queue to get processed by SaveProcess + SaveProcessSize int `json:"save_process_size,omitempty"` + // ValidateProcess is like ProcessorStack, but for recipient validation tasks + ValidateProcess string `json:"validate_process,omitempty"` + // ValidateProcessSize limits the amount of messages waiting in the queue to get processed by ValidateProcess + ValidateProcessSize int `json:"validate_process_size,omitempty"` + + // TimeoutSave is duration before timeout when saving an email, eg "29s" + TimeoutSave string `json:"save_timeout,omitempty"` + // TimeoutValidateRcpt duration before timeout when validating a recipient, eg "1s" + TimeoutValidateRcpt string `json:"val_rcpt_timeout,omitempty"` + // TimeoutStream duration before timeout when processing a stream eg "1s" + TimeoutStream string `json:"stream_timeout,omitempty"` + + // StreamBufferLen controls the size of the output buffer, in bytes. Default is 4096 + StreamBufferSize int `json:"stream_buffer_size,omitempty"` + // SaveStream is same as a SaveProcess, but uses the StreamProcessor stack instead + SaveStream string `json:"save_stream,omitempty"` + // SaveStreamSize limits the amount of messages waiting in the queue to get processed by SaveStream + SaveStreamSize int `json:"save_stream_size,omitempty"` + + // PostProcessSize controls the length of thq queue for background processing + PostProcessSize int `json:"post_process_size,omitempty"` + // PostProcessProducer specifies which StreamProcessor to use for reading data to the post process + PostProcessProducer string `json:"post_process_producer,omitempty"` + // PostProcessConsumer is same as SaveStream, but controls + PostProcessConsumer string `json:"post_process_consumer,omitempty"` +} + +// saveWorkersCount gets the number of workers to use for saving email by reading the save_workers_size config value +// Returns 1 if no config value was set +func (c *GatewayConfig) saveWorkersCount() int { + if c.SaveWorkersCount <= 0 { + return configSaveWorkersCount + } + return c.SaveWorkersCount +} + +func (c *GatewayConfig) validateWorkersCount() int { + if c.ValidateWorkersCount <= 0 { + return configValidateWorkersCount + } + return c.ValidateWorkersCount +} + +func (c *GatewayConfig) streamWorkersCount() int { + if c.StreamWorkersCount <= 0 { + return configStreamWorkersCount + } + return c.StreamWorkersCount +} + +func (c *GatewayConfig) backgroundWorkersCount() int { + if c.BackgroundWorkersCount <= 0 { + return configBackgroundWorkersCount + } + return c.BackgroundWorkersCount +} +func (c *GatewayConfig) saveProcessSize() int { + if c.SaveProcessSize <= 0 { + return configSaveProcessSize + } + return c.SaveProcessSize +} + +func (c *GatewayConfig) validateProcessSize() int { + if c.ValidateProcessSize <= 0 { + return configValidateProcessSize + } + return c.ValidateProcessSize +} + +func (c *GatewayConfig) saveStreamSize() int { + if c.SaveStreamSize <= 0 { + return configSaveStreamSize + } + return c.SaveStreamSize +} + +func (c *GatewayConfig) postProcessSize() int { + if c.PostProcessSize <= 0 { + return configPostProcessSize + } + return c.PostProcessSize +} + +// saveTimeout returns the maximum amount of seconds to wait before timing out a save processing task +func (gw *BackendGateway) saveTimeout() time.Duration { + if gw.gwConfig.TimeoutSave == "" { + return configTimeoutSave + } + t, err := time.ParseDuration(gw.gwConfig.TimeoutSave) + if err != nil { + return configTimeoutSave + } + return t +} + +// validateRcptTimeout returns the maximum amount of seconds to wait before timing out a recipient validation task +func (gw *BackendGateway) validateRcptTimeout() time.Duration { + if gw.gwConfig.TimeoutValidateRcpt == "" { + return configTimeoutValidateRcpt + } + t, err := time.ParseDuration(gw.gwConfig.TimeoutValidateRcpt) + if err != nil { + return configTimeoutValidateRcpt + } + return t +} diff --git a/backends/decorate_stream.go b/backends/decorate_stream.go new file mode 100644 index 00000000..ef0073b4 --- /dev/null +++ b/backends/decorate_stream.go @@ -0,0 +1,54 @@ +package backends + +import ( + "encoding/json" + "github.com/flashmob/go-guerrilla/mail" +) + +type streamOpenWith func(e *mail.Envelope) error +type streamCloseWith func() error +type streamConfigureWith func(cfg ConfigGroup) error +type streamShutdownWith func() error + +// We define what a decorator to our processor will look like +// StreamProcessor argument is the underlying processor that we're decorating +// the additional ...interface argument is not needed, but can be useful for dependency injection +type StreamDecorator struct { + // Decorate is called first. The StreamProcessor will be the next processor called + // after this one finished. + Decorate func(StreamProcessor, ...interface{}) StreamProcessor + e *mail.Envelope + // Open is called at the start of each email + Open streamOpenWith + // Close is called when the email finished writing + Close streamCloseWith + // Configure is always called after Decorate, only once for the entire lifetime + // it can open database connections, test file permissions, etc + Configure streamConfigureWith + // Shutdown is called to release any resources before StreamDecorator is destroyed + // typically to close any database connections, cleanup any files, etc + Shutdown streamShutdownWith + // GetEmail returns a reader for reading the data of ab email, + // it may return nil if no email is available + GetEmail func(emailID uint64) (SeekPartReader, error) +} + +func (s StreamDecorator) ExtractConfig(cfg ConfigGroup, i interface{}) error { + data, err := json.Marshal(cfg) + if err != nil { + return err + } + err = json.Unmarshal(data, i) + if err != nil { + return err + } + return nil +} + +// DecorateStream will decorate a StreamProcessor with a slice of passed decorators +func DecorateStream(c StreamProcessor, ds []*StreamDecorator) (StreamProcessor, []*StreamDecorator) { + for i := range ds { + c = ds[i].Decorate(c) + } + return c, ds +} diff --git a/backends/gateway.go b/backends/gateway.go index 6b7cb655..2fc7789c 100644 --- a/backends/gateway.go +++ b/backends/gateway.go @@ -3,52 +3,53 @@ package backends import ( "errors" "fmt" + "io" + "reflect" "strconv" "sync" "time" - "runtime/debug" - "strings" - "github.com/flashmob/go-guerrilla/log" "github.com/flashmob/go-guerrilla/mail" "github.com/flashmob/go-guerrilla/response" + "runtime/debug" ) -var ErrProcessorNotFound error - // A backend gateway is a proxy that implements the Backend interface. // It is used to start multiple goroutine workers for saving mail, and then distribute email saving to the workers // via a channel. Shutting down via Shutdown() will stop all workers. // The rest of this program always talks to the backend via this gateway. type BackendGateway struct { + // name is the name of the gateway given in the config + name string // channel for distributing envelopes to workers - conveyor chan *workerMsg + conveyor chan *workerMsg + conveyorValidation chan *workerMsg + conveyorStream chan *workerMsg + conveyorStreamBg chan *workerMsg // waits for backend workers to start/stop wg sync.WaitGroup workStoppers []chan bool processors []Processor - validators []Processor + validators []ValidatingProcessor + streamers []streamer + background []streamer + + producer *StreamDecorator + + decoratorLookup map[ConfigSection]map[string]*StreamDecorator + + workerID int // controls access to state sync.Mutex State backendState config BackendConfig gwConfig *GatewayConfig -} -type GatewayConfig struct { - // WorkersSize controls how many concurrent workers to start. Defaults to 1 - WorkersSize int `json:"save_workers_size,omitempty"` - // SaveProcess controls which processors to chain in a stack for saving email tasks - SaveProcess string `json:"save_process,omitempty"` - // ValidateProcess is like ProcessorStack, but for recipient validation tasks - ValidateProcess string `json:"validate_process,omitempty"` - // TimeoutSave is duration before timeout when saving an email, eg "29s" - TimeoutSave string `json:"gw_save_timeout,omitempty"` - // TimeoutValidateRcpt duration before timeout when validating a recipient, eg "1s" - TimeoutValidateRcpt string `json:"gw_val_rcpt_timeout,omitempty"` + //buffers []byte // stream output buffer + buffers map[int][]byte } // workerMsg is what get placed on the BackendGateway.saveMailChan channel @@ -59,6 +60,66 @@ type workerMsg struct { notifyMe chan *notifyMsg // select the task type task SelectTask + // io.Reader for streamed processor + r io.Reader +} + +type streamer struct { + // StreamProcessor is a chain of StreamProcessor + sp StreamProcessor + // so that we can call Open and Close + d []*StreamDecorator +} + +func (s streamer) Write(p []byte) (n int, err error) { + return s.sp.Write(p) +} + +func (s *streamer) open(e *mail.Envelope) error { + var err Errors + for i := range s.d { + if s.d[i].Open != nil { + if e := s.d[i].Open(e); e != nil { + err = append(err, e) + } + } + } + if len(err) == 0 { + return nil + } + return err +} + +func (s *streamer) close() error { + var err Errors + // close in reverse order + for i := len(s.d) - 1; i >= 0; i-- { + if s.d[i].Close != nil { + if e := s.d[i].Close(); e != nil { + err = append(err, e) + } + } + } + if len(err) == 0 { + return nil + } + return err +} + +func (s *streamer) shutdown() error { + var err Errors + // shutdown in reverse order + for i := len(s.d) - 1; i >= 0; i-- { + if s.d[i].Shutdown != nil { + if e := s.d[i].Shutdown(); e != nil { + err = append(err, e) + } + } + } + if len(err) == 0 { + return nil + } + return err } type backendState int @@ -70,12 +131,6 @@ const ( BackendStateShuttered BackendStateError BackendStateInitialized - - // default timeout for saving email, if 'gw_save_timeout' not present in config - saveTimeout = time.Second * 30 - // default timeout for validating rcpt to, if 'gw_val_rcpt_timeout' not present in config - validateRcptTimeout = time.Second * 5 - defaultProcessor = "Debugger" ) func (s backendState) String() string { @@ -96,18 +151,18 @@ func (s backendState) String() string { // New makes a new default BackendGateway backend, and initializes it using // backendConfig and stores the logger -func New(backendConfig BackendConfig, l log.Logger) (Backend, error) { +func New(name string, backendConfig BackendConfig, l log.Logger) (Backend, error) { Svc.SetMainlog(l) - gateway := &BackendGateway{} + gateway := &BackendGateway{name: name} + backendConfig.toLower() + // keep the a copy of the config + gateway.config = backendConfig err := gateway.Initialize(backendConfig) if err != nil { return nil, fmt.Errorf("error while initializing the backend: %s", err) } - // keep the config known to be good. - gateway.config = backendConfig - b = Backend(gateway) - return b, nil + return gateway, nil } var workerMsgPool = sync.Pool{ @@ -126,6 +181,10 @@ func (w *workerMsg) reset(e *mail.Envelope, task SelectTask) { w.task = task } +func (gw *BackendGateway) Name() string { + return gw.name +} + // Process distributes an envelope to one of the backend workers with a TaskSaveMail task func (gw *BackendGateway) Process(e *mail.Envelope) Result { if gw.State != BackendStateRunning { @@ -133,6 +192,7 @@ func (gw *BackendGateway) Process(e *mail.Envelope) Result { } // borrow a workerMsg from the pool workerMsg := workerMsgPool.Get().(*workerMsg) + defer workerMsgPool.Put(workerMsg) workerMsg.reset(e, TaskSaveMail) // place on the channel so that one of the save mail workers can pick it up gw.conveyor <- workerMsg @@ -167,13 +227,13 @@ func (gw *BackendGateway) Process(e *mail.Envelope) Result { return NewResult(response.Canned.FailBackendTransaction, response.SP, err) case <-time.After(gw.saveTimeout()): - Log().Error("Backend has timed out while saving email") - e.Lock() // lock the envelope - it's still processing here, we don't want the server to recycle it + Log().Fields("queuedId", e.QueuedId).Error("backend has timed out while saving email") + e.Add(1) // lock the envelope - it's still processing here, we don't want the server to recycle it go func() { // keep waiting for the backend to finish processing <-workerMsg.notifyMe - e.Unlock() - workerMsgPool.Put(workerMsg) + Log().Fields("queuedId", e.QueuedId).Error("finished processing mail after timeout") + e.Done() }() return NewResult(response.Canned.FailBackendTimeout) } @@ -191,30 +251,168 @@ func (gw *BackendGateway) ValidateRcpt(e *mail.Envelope) RcptError { } // place on the channel so that one of the save mail workers can pick it up workerMsg := workerMsgPool.Get().(*workerMsg) + defer workerMsgPool.Put(workerMsg) workerMsg.reset(e, TaskValidateRcpt) - gw.conveyor <- workerMsg + gw.conveyorValidation <- workerMsg // wait for the validation to complete // or timeout select { case status := <-workerMsg.notifyMe: - workerMsgPool.Put(workerMsg) if status.err != nil { return status.err } return nil case <-time.After(gw.validateRcptTimeout()): - e.Lock() + Log().Fields("queuedId", e.QueuedId).Error("backend has timed out while validating rcpt") + e.Add(1) // lock the envelope - it's still processing here, we don't want the server to recycle it go func() { <-workerMsg.notifyMe - e.Unlock() - workerMsgPool.Put(workerMsg) - Log().Error("Backend has timed out while validating rcpt") + Log().Fields("queuedId", e.QueuedId).Error("finished validating rcpt after timeout") + e.Done() }() return StorageTimeout } } +func (gw *BackendGateway) StreamOn() bool { + return len(gw.gwConfig.SaveStream) != 0 +} + +// newStreamDecorator creates a new StreamDecorator and calls Configure with its corresponding configuration +// cs - the item of 'list' property, result from newStackStreamProcessorConfig() +// section - which section of the config +func (gw *BackendGateway) newStreamDecorator(cs stackConfigExpression, section ConfigSection) *StreamDecorator { + if makeFunc, ok := Streamers[cs.name]; !ok { + return nil + } else { + d := makeFunc() + config := gw.config.lookupGroup(section, cs.String()) + if config == nil { + config = ConfigGroup{} + } + if d.Configure != nil { + if err := d.Configure(config); err != nil { + return nil + } + } + return d + } +} + +func (gw *BackendGateway) ProcessBackground(e *mail.Envelope) { + + if d := gw.producer; d == nil { + Log().Error("gateway has failed creating a post_process_producer, check config") + return + } else { + r, err := d.GetEmail(e.MessageID) + if err != nil { + Log().Fields("queuedID", e.QueuedId, "messageID", e.MessageID). + Error("gateway background process aborted: email with messageID not found") + return + } + + // borrow a workerMsg from the pool + workerMsg := workerMsgPool.Get().(*workerMsg) + defer workerMsgPool.Put(workerMsg) + + workerMsg.reset(e, TaskSaveMailStream) + workerMsg.r = r + + // place on the channel so that one of the save mail workers can pick it up + // buffered channel will block if full + select { + case gw.conveyorStreamBg <- workerMsg: + break + case <-time.After(gw.saveTimeout()): + Log().Fields("queuedID", e.QueuedId).Error("post-processing timeout - queue full, aborting") + return + } + // process in the background + for { + select { + case status := <-workerMsg.notifyMe: + // email saving transaction completed + var fields []interface{} + var code int + if status.result != nil { + code = status.result.Code() + fields = append(fields, "queuedID", e.QueuedId, "code", code) + + } + if code > 200 && code < 300 { + fields = append(fields, "messageID", e.MessageID) + Log().Fields(fields...).Info("background process done") + return + } + if status.err != nil { + fields = append(fields, "error", status.err) + } + if len(fields) > 0 { + Log().Fields(fields...).Error("post-process completed with an error") + return + } + // both result & error are nil (should not happen) + Log().Fields("queuedID", e.QueuedId).Error("no response from backend - post-process did not return a result or an error") + return + case <-time.After(gw.saveTimeout()): + Log().Fields("queuedID", e.QueuedId).Error("background post-processing timed-out, will keep waiting") + // don't return here, keep waiting for workerMsg.notifyMe + } + } + } +} + +func (gw *BackendGateway) ProcessStream(r io.Reader, e *mail.Envelope) (Result, int64, error) { + res := response.Canned + if gw.State != BackendStateRunning { + return NewResult(res.FailBackendNotRunning, response.SP, gw.State), 0, errors.New(res.FailBackendNotRunning.String()) + } + // borrow a workerMsg from the pool + workerMsg := workerMsgPool.Get().(*workerMsg) + workerMsgPool.Put(workerMsg) + workerMsg.reset(e, TaskSaveMailStream) + workerMsg.r = r + // place on the channel so that one of the save mail workers can pick it up + gw.conveyorStream <- workerMsg + // wait for the save to complete + // or timeout + select { + case status := <-workerMsg.notifyMe: + // email saving transaction completed + if status.result == BackendResultOK && status.queuedID != "" { + return NewResult(res.SuccessMessageQueued, response.SP, status.queuedID), e.Size, status.err + } + // A custom result, there was probably an error, if so, log it + if status.result != nil { + return status.result, e.Size, status.err + } + // if there was no result, but there's an error, then make a new result from the error + if status.err != nil { + if _, err := strconv.Atoi(status.err.Error()[:3]); err != nil { + return NewResult(res.FailBackendTransaction, response.SP, status.err), e.Size, status.err + } + return NewResult(status.err), e.Size, status.err + } + // both result & error are nil (should not happen) + err := errors.New("no response from backend - processor did not return a result or an error") + Log().Error(err) + return NewResult(res.FailBackendTransaction, response.SP, err), e.Size, err + + case <-time.After(gw.saveTimeout()): + Log().Fields("queuedID", e.QueuedId).Error("backend has timed out while saving email stream") + e.Add(1) // lock the envelope - it's still processing here, we don't want the server to recycle it + go func() { + // keep waiting for the backend to finish processing + <-workerMsg.notifyMe + e.Done() + Log().Fields("queuedID", e.QueuedId).Info("backend has finished saving email stream after timeout") + }() + return NewResult(res.FailBackendTimeout), -1, errors.New("gateway timeout") + } +} + // Shutdown shuts down the backend and leaves it in BackendStateShuttered state func (gw *BackendGateway) Shutdown() error { gw.Lock() @@ -224,6 +422,18 @@ func (gw *BackendGateway) Shutdown() error { gw.stopWorkers() // wait for workers to stop gw.wg.Wait() + for stream := range gw.streamers { + err := gw.streamers[stream].shutdown() + if err != nil { + Log().Fields("error", err, "gateway", gw.name).Error("failed shutting down stream") + } + } + for stream := range gw.background { + err := gw.background[stream].shutdown() + if err != nil { + Log().Fields("error", err, "gateway", gw.name).Error("failed shutting down background stream") + } + } // call shutdown on all processor shutdowners if err := Svc.shutdown(); err != nil { return err @@ -243,7 +453,6 @@ func (gw *BackendGateway) Reinitialize() error { err := gw.Initialize(gw.config) if err != nil { - fmt.Println("reinitialize to ", gw.config, err) return fmt.Errorf("error while initializing the backend: %s", err) } @@ -256,19 +465,15 @@ func (gw *BackendGateway) Reinitialize() error { // This function uses the config value save_process or validate_process to figure out which Decorator to use func (gw *BackendGateway) newStack(stackConfig string) (Processor, error) { var decorators []Decorator - cfg := strings.ToLower(strings.TrimSpace(stackConfig)) - if len(cfg) == 0 { - //cfg = strings.ToLower(defaultProcessor) + c := newStackProcessorConfig(stackConfig, newAliasMap(gw.config[ConfigProcessors])) + if len(c.list) == 0 { return NoopProcessor{}, nil } - items := strings.Split(cfg, "|") - for i := range items { - name := items[len(items)-1-i] // reverse order, since decorators are stacked - if makeFunc, ok := processors[name]; ok { + for i := range c.list { + if makeFunc, ok := processors[c.list[i].name]; ok { decorators = append(decorators, makeFunc()) } else { - ErrProcessorNotFound = fmt.Errorf("processor [%s] not found", name) - return nil, ErrProcessorNotFound + return nil, c.notFound(c.list[i].name) } } // build the call-stack of decorators @@ -276,13 +481,44 @@ func (gw *BackendGateway) newStack(stackConfig string) (Processor, error) { return p, nil } +func (gw *BackendGateway) newStreamStack(stackConfig string) (streamer, error) { + var decorators []*StreamDecorator + + noop := streamer{NoopStreamProcessor{}, decorators} + groupName := ConfigStreamProcessors + c := newStackStreamProcessorConfig(stackConfig, newAliasMap(gw.config[groupName])) + if len(c.list) == 0 { + return noop, nil + } + for i := range c.list { + if d := gw.newStreamDecorator(c.list[i], groupName); d != nil { + if gw.decoratorLookup[groupName] == nil { + gw.decoratorLookup[groupName] = make(map[string]*StreamDecorator) + } + gw.decoratorLookup[groupName][c.list[i].String()] = d + decorators = append(decorators, d) + } else { + return streamer{nil, decorators}, c.notFound(c.list[i].name) + } + } + // build the call-stack of decorators + sp, decorators := DecorateStream(&DefaultStreamProcessor{}, decorators) + return streamer{sp, decorators}, nil +} + // loadConfig loads the config for the GatewayConfig func (gw *BackendGateway) loadConfig(cfg BackendConfig) error { configType := BaseConfig(&GatewayConfig{}) // Note: treat config values as immutable // if you need to change a config value, change in the file then // send a SIGHUP - bcfg, err := Svc.ExtractConfig(cfg, configType) + if gw.name == "" { + gw.name = DefaultGateway + } + if _, ok := cfg[ConfigGateways][gw.name]; !ok { + return errors.New("no such gateway configured: " + gw.name) + } + bcfg, err := Svc.ExtractConfig(ConfigGateways, gw.name, cfg, configType) if err != nil { return err } @@ -302,21 +538,21 @@ func (gw *BackendGateway) Initialize(cfg BackendConfig) error { gw.State = BackendStateError return err } - workersSize := gw.workersSize() - if workersSize < 1 { - gw.State = BackendStateError - return errors.New("must have at least 1 worker") - } + gw.buffers = make(map[int][]byte) // individual buffers are made later + gw.decoratorLookup = make(map[ConfigSection]map[string]*StreamDecorator) gw.processors = make([]Processor, 0) - gw.validators = make([]Processor, 0) - for i := 0; i < workersSize; i++ { + gw.validators = make([]ValidatingProcessor, 0) + gw.streamers = make([]streamer, 0) + gw.background = make([]streamer, 0) + for i := 0; i < gw.gwConfig.saveWorkersCount(); i++ { p, err := gw.newStack(gw.gwConfig.SaveProcess) if err != nil { gw.State = BackendStateError return err } gw.processors = append(gw.processors, p) - + } + for i := 0; i < gw.gwConfig.validateWorkersCount(); i++ { v, err := gw.newStack(gw.gwConfig.ValidateProcess) if err != nil { gw.State = BackendStateError @@ -324,14 +560,35 @@ func (gw *BackendGateway) Initialize(cfg BackendConfig) error { } gw.validators = append(gw.validators, v) } - // initialize processors - if err := Svc.initialize(cfg); err != nil { - gw.State = BackendStateError + for i := 0; i < gw.gwConfig.streamWorkersCount(); i++ { + s, err := gw.newStreamStack(gw.gwConfig.SaveStream) + if err != nil { + gw.State = BackendStateError + return err + } + gw.streamers = append(gw.streamers, s) + } + for i := 0; i < gw.gwConfig.backgroundWorkersCount(); i++ { + c, err := gw.newStreamStack(gw.gwConfig.PostProcessConsumer) + if err != nil { + gw.State = BackendStateError + return err + } + gw.background = append(gw.background, c) + } + if err = gw.initProducer(); err != nil { return err } - if gw.conveyor == nil { - gw.conveyor = make(chan *workerMsg, workersSize) + + // Initialize processors & stream processors + if err := Svc.Initialize(cfg); err != nil { + gw.State = BackendStateError + return err } + gw.conveyor = make(chan *workerMsg, gw.gwConfig.saveProcessSize()) + gw.conveyorValidation = make(chan *workerMsg, gw.gwConfig.validateProcessSize()) + gw.conveyorStream = make(chan *workerMsg, gw.gwConfig.saveStreamSize()) + gw.conveyorStreamBg = make(chan *workerMsg, gw.gwConfig.postProcessSize()) // ready to start gw.State = BackendStateInitialized return nil @@ -342,33 +599,12 @@ func (gw *BackendGateway) Start() error { gw.Lock() defer gw.Unlock() if gw.State == BackendStateInitialized || gw.State == BackendStateShuttered { - // we start our workers - workersSize := gw.workersSize() // make our slice of channels for stopping gw.workStoppers = make([]chan bool, 0) - // set the wait group - gw.wg.Add(workersSize) - - for i := 0; i < workersSize; i++ { - stop := make(chan bool) - go func(workerId int, stop chan bool) { - // blocks here until the worker exits - for { - state := gw.workDispatcher( - gw.conveyor, - gw.processors[workerId], - gw.validators[workerId], - workerId+1, - stop) - // keep running after panic - if state != dispatcherStatePanic { - break - } - } - gw.wg.Done() - }(i, stop) - gw.workStoppers = append(gw.workStoppers, stop) - } + gw.startWorkers(gw.conveyor, gw.processors) + gw.startWorkers(gw.conveyorValidation, gw.validators) + gw.startWorkers(gw.conveyorStream, gw.streamers) + gw.startWorkers(gw.conveyorStreamBg, gw.background) gw.State = BackendStateRunning return nil } else { @@ -376,37 +612,38 @@ func (gw *BackendGateway) Start() error { } } -// workersSize gets the number of workers to use for saving email by reading the save_workers_size config value -// Returns 1 if no config value was set -func (gw *BackendGateway) workersSize() int { - if gw.gwConfig.WorkersSize <= 0 { - return 1 - } - return gw.gwConfig.WorkersSize -} - -// saveTimeout returns the maximum amount of seconds to wait before timing out a save processing task -func (gw *BackendGateway) saveTimeout() time.Duration { - if gw.gwConfig.TimeoutSave == "" { - return saveTimeout - } - t, err := time.ParseDuration(gw.gwConfig.TimeoutSave) - if err != nil { - return saveTimeout +func (gw *BackendGateway) startWorkers(conveyor chan *workerMsg, processors interface{}) { + p := reflect.ValueOf(processors) + if reflect.TypeOf(processors).Kind() != reflect.Slice { + panic("processors must be a slice") } - return t -} - -// validateRcptTimeout returns the maximum amount of seconds to wait before timing out a recipient validation task -func (gw *BackendGateway) validateRcptTimeout() time.Duration { - if gw.gwConfig.TimeoutValidateRcpt == "" { - return validateRcptTimeout - } - t, err := time.ParseDuration(gw.gwConfig.TimeoutValidateRcpt) - if err != nil { - return validateRcptTimeout + // set the wait group (when stopping, it will block for all goroutines to exit) + gw.wg.Add(p.Len()) + for i := 0; i < p.Len(); i++ { + // set the buffer + gw.buffers[gw.workerID] = gw.makeBuffer() + // stop is a channel used for stopping the worker + stop := make(chan bool) + // start the worker and keep it running + go func(workerId int, stop chan bool, i int) { + // blocks here until the worker exits + // for-loop used so that if workDispatcher panics, re-enter gw.workDispatcher + for { + state := gw.workDispatcher( + conveyor, + p.Index(i).Interface(), + workerId, + stop) + // keep running after panic + if state != dispatcherStatePanic { + break + } + } + gw.wg.Done() + }(gw.workerID, stop, i) + gw.workStoppers = append(gw.workStoppers, stop) + gw.workerID++ } - return t } type dispatcherState int @@ -421,15 +658,13 @@ const ( func (gw *BackendGateway) workDispatcher( workIn chan *workerMsg, - save Processor, - validate Processor, + processor interface{}, workerId int, stop chan bool) (state dispatcherState) { var msg *workerMsg defer func() { - // panic recovery mechanism: it may panic when processing // since processors may call arbitrary code, some may be 3rd party / unstable // we need to detect the panic, and notify the backend that it failed & unlock the envelope @@ -446,32 +681,97 @@ func (gw *BackendGateway) workDispatcher( }() state = dispatcherStateIdle - Log().Infof("processing worker started (#%d)", workerId) + Log().Fields("id", workerId+1, "gateway", gw.name). + Info("processing worker started") for { select { case <-stop: state = dispatcherStateStopped - Log().Infof("stop signal for worker (#%d)", workerId) + Log().Fields("id", workerId+1, "gateway", gw.name). + Info("stop signal for worker") return case msg = <-workIn: state = dispatcherStateWorking // recovers from panic if in this state - if msg.task == TaskSaveMail { - result, err := save.Process(msg.e, msg.task) + switch v := processor.(type) { + case Processor: + result, err := v.Process(msg.e, msg.task) state = dispatcherStateNotify - msg.notifyMe <- ¬ifyMsg{err: err, result: result, queuedID: msg.e.QueuedId} - } else { - result, err := validate.Process(msg.e, msg.task) + msg.notifyMe <- ¬ifyMsg{err: err, result: result, queuedID: msg.e.QueuedId.String()} + case ValidatingProcessor: + result, err := v.Process(msg.e, msg.task) state = dispatcherStateNotify msg.notifyMe <- ¬ifyMsg{err: err, result: result} + case streamer: + err := v.open(msg.e) + if err == nil { + if msg.e.Size, err = io.CopyBuffer(v, msg.r, gw.buffers[workerId]); err != nil { + Log().Fields("error", err, "workerID", workerId+1).Error("stream writing failed") + } + if err = v.close(); err != nil { + Log().Fields("error", err, "workerID", workerId+1).Error("stream closing failed") + } + } + state = dispatcherStateNotify + var result Result + if err != nil { + result = NewResult(response.Canned.FailBackendTransaction, err) + } else { + result = NewResult(response.Canned.SuccessMessageQueued, response.SP, msg.e.QueuedId) + } + msg.notifyMe <- ¬ifyMsg{err: err, result: result, queuedID: msg.e.QueuedId.String()} + } } state = dispatcherStateIdle } } +func (gw *BackendGateway) makeBuffer() []byte { + if gw.buffers == nil { + gw.buffers = make(map[int][]byte) + } + size := configStreamBufferSize + if gw.gwConfig.StreamBufferSize > 0 { + size = gw.gwConfig.StreamBufferSize + } + return make([]byte, size) +} + // stopWorkers sends a signal to all workers to stop func (gw *BackendGateway) stopWorkers() { for i := range gw.workStoppers { gw.workStoppers[i] <- true } + gw.workerID = 0 +} + +func (gw *BackendGateway) initProducer() error { + notValid := errors.New("gateway has no valid [post_process_producer] configured") + if gw.gwConfig.PostProcessConsumer == "" { + // consumer not configured, so not active + return nil + } + if gw.gwConfig.PostProcessProducer == "" { + return notValid + } + section := ConfigStreamProcessors // which section of the config (stream_processors) + m := newAliasMap(gw.config[section]) + c := newStackStreamProcessorConfig(gw.gwConfig.PostProcessProducer, m) + if len(c.list) == 0 { + return notValid + } + // check it there's already an instance of it + if gw.decoratorLookup[section] != nil { + if v, ok := gw.decoratorLookup[section][c.list[0].String()]; ok { + gw.producer = v + return nil + } + } + if d := gw.newStreamDecorator(c.list[0], section); d != nil { + // use a new instance + gw.producer = d + return nil + } else { + return errors.New("please check gateway config [post_process_producer]") + } } diff --git a/backends/gateway_test.go b/backends/gateway_test.go index c8db6894..f9f4095b 100644 --- a/backends/gateway_test.go +++ b/backends/gateway_test.go @@ -1,9 +1,12 @@ package backends import ( + "bufio" + "bytes" "fmt" "github.com/flashmob/go-guerrilla/log" "github.com/flashmob/go-guerrilla/mail" + "io" "strings" "testing" "time" @@ -19,9 +22,17 @@ func TestStates(t *testing.T) { func TestInitialize(t *testing.T) { c := BackendConfig{ - "save_process": "HeadersParser|Debugger", - "log_received_mails": true, - "save_workers_size": "1", + ConfigProcessors: { + "Debugger": { + "log_received_mails": true, + }, + }, + ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger", + "save_workers_size": "1", + }, + }, } gateway := &BackendGateway{} @@ -38,7 +49,7 @@ func TestInitialize(t *testing.T) { if gateway.conveyor == nil { t.Error("gateway.conveyor should not be nil") - } else if cap(gateway.conveyor) != gateway.workersSize() { + } else if cap(gateway.conveyor) != gateway.gwConfig.saveProcessSize() { t.Error("gateway.conveyor channel buffer cap does not match worker size, cap was", cap(gateway.conveyor)) } @@ -50,9 +61,17 @@ func TestInitialize(t *testing.T) { func TestStartProcessStop(t *testing.T) { c := BackendConfig{ - "save_process": "HeadersParser|Debugger", - "log_received_mails": true, - "save_workers_size": 2, + ConfigProcessors: { + "Debugger": { + "log_received_mails": true, + }, + }, + ConfigGateways: { + "default": { + "save_process": "HeadersParser|Debugger", + "save_workers_size": "2", + }, + }, } gateway := &BackendGateway{} @@ -71,22 +90,31 @@ func TestStartProcessStop(t *testing.T) { t.Fail() } if gateway.State != BackendStateRunning { - t.Error("gateway.State is not in rinning state, got ", gateway.State) + t.Error("gateway.State is not in running state, got ", gateway.State) } // can we place an envelope on the conveyor channel? e := &mail.Envelope{ RemoteIP: "127.0.0.1", - QueuedId: "abc12345", + QueuedId: mail.QueuedID(1, 2), Helo: "helo.example.com", MailFrom: mail.Address{User: "test", Host: "example.com"}, TLS: true, } e.PushRcpt(mail.Address{User: "test", Host: "example.com"}) - e.Data.WriteString("Subject:Test\n\nThis is a test.") + //e.Data.WriteString("Subject:Test\n\nThis is a test.") + in := "Subject: Test\n\nThis is a test.\n.\n" + mdr := mail.NewMimeDotReader(bufio.NewReader(bytes.NewBufferString(in)), 1) + i, err := io.Copy(&e.Data, mdr) + if err != nil && err != io.EOF { + t.Error(err, "cannot copy buffer", i, err) + } + if p := mdr.Parts(); p != nil && len(p) > 0 { + e.Header = p[0].Headers + } notify := make(chan *notifyMsg) - gateway.conveyor <- &workerMsg{e, notify, TaskSaveMail} + gateway.conveyor <- &workerMsg{e, notify, TaskSaveMail, nil} // it should not produce any errors // headers (subject) should be parsed. diff --git a/backends/p_debugger.go b/backends/p_debugger.go index 6c94604a..6ef4e0f2 100644 --- a/backends/p_debugger.go +++ b/backends/p_debugger.go @@ -12,6 +12,8 @@ import ( // Description : Log received emails // ---------------------------------------------------------------------------------- // Config Options: log_received_mails bool - log if true +// : sleep_seconds - how many seconds to pause for, useful to force a +// : timeout. If sleep_seconds is 1 then a panic will be induced // --------------:------------------------------------------------------------------- // Input : e.MailFrom, e.RcptTo, e.Header // ---------------------------------------------------------------------------------- @@ -32,7 +34,8 @@ func Debugger() Decorator { var config *debuggerConfig initFunc := InitializeWith(func(backendConfig BackendConfig) error { configType := BaseConfig(&debuggerConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + bcfg, err := Svc.ExtractConfig( + ConfigProcessors, defaultProcessor, backendConfig, configType) if err != nil { return err } @@ -44,21 +47,19 @@ func Debugger() Decorator { return ProcessWith(func(e *mail.Envelope, task SelectTask) (Result, error) { if task == TaskSaveMail { if config.LogReceivedMails { - Log().Infof("Mail from: %s / to: %v", e.MailFrom.String(), e.RcptTo) - Log().Info("Headers are:", e.Header) + Log().Fields("queuedID", e.QueuedId, "from", e.MailFrom.String(), "to", e.RcptTo).Info("save mail") + Log().Fields("queuedID", e.QueuedId, "headers", e.Header).Info("headers dump") + Log().Fields("queuedID", e.QueuedId, "body", e.Data.String()).Info("body dump") } - if config.SleepSec > 0 { - Log().Infof("sleeping for %d", config.SleepSec) + Log().Fields("queuedID", e.QueuedId, "sleep", config.SleepSec).Info("sleeping") time.Sleep(time.Second * time.Duration(config.SleepSec)) - Log().Infof("woke up") + Log().Fields("queuedID", e.QueuedId).Info("woke up") if config.SleepSec == 1 { panic("panic on purpose") } - } - // continue to the next Processor in the decorator stack return p.Process(e, task) } else { diff --git a/backends/p_guerrilla_db_redis.go b/backends/p_guerrilla_db_redis.go index f2905a68..0bd6eea2 100644 --- a/backends/p_guerrilla_db_redis.go +++ b/backends/p_guerrilla_db_redis.go @@ -54,7 +54,6 @@ type GuerrillaDBAndRedisBackend struct { type stmtCache [GuerrillaDBAndRedisBatchMax]*sql.Stmt type guerrillaDBAndRedisConfig struct { - NumberOfWorkers int `json:"save_workers_size"` Table string `json:"mail_table"` Driver string `json:"sql_driver"` DSN string `json:"sql_dsn"` @@ -69,7 +68,7 @@ type guerrillaDBAndRedisConfig struct { // Now we need to convert each type and copy into the guerrillaDBAndRedisConfig struct func (g *GuerrillaDBAndRedisBackend) loadConfig(backendConfig BackendConfig) (err error) { configType := BaseConfig(&guerrillaDBAndRedisConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + bcfg, err := Svc.ExtractConfig(ConfigProcessors, "guerrillaredisdb", backendConfig, configType) if err != nil { return err } @@ -78,10 +77,6 @@ func (g *GuerrillaDBAndRedisBackend) loadConfig(backendConfig BackendConfig) (er return nil } -func (g *GuerrillaDBAndRedisBackend) getNumberOfWorkers() int { - return g.config.NumberOfWorkers -} - type redisClient struct { isConnected bool conn RedisConn @@ -191,24 +186,23 @@ func (g *GuerrillaDBAndRedisBackend) doQuery(c int, db *sql.DB, insertStmt *sql. var execErr error defer func() { if r := recover(); r != nil { - //logln(1, fmt.Sprintf("Recovered in %v", r)) - Log().Error("Recovered form panic:", r, string(debug.Stack())) sum := 0 for _, v := range *vals { if str, ok := v.(string); ok { sum = sum + len(str) } } - Log().Errorf("panic while inserting query [%s] size:%d, err %v", r, sum, execErr) + Log().Fields("panic", fmt.Sprintf("%v", r), + "size", sum, + "error", execErr, + "stack", string(debug.Stack())). + Error("panic while inserting query") panic("query failed") } }() // prepare the query used to insert when rows reaches batchMax insertStmt = g.prepareInsertQuery(c, db) _, execErr = insertStmt.Exec(*vals...) - //if rand.Intn(2) == 1 { - // return errors.New("uggabooka") - //} if execErr != nil { Log().WithError(execErr).Error("There was a problem the insert") } @@ -253,7 +247,7 @@ func (g *GuerrillaDBAndRedisBackend) insertQueryBatcher( // retry the sql query attempts := 3 for i := 0; i < attempts; i++ { - Log().Infof("retrying query query rows[%c] ", c) + Log().Fields("rows", c).Info("retrying query query rows ") time.Sleep(time.Second) err = g.doQuery(c, db, insertStmt, &vals) if err == nil { @@ -279,7 +273,7 @@ func (g *GuerrillaDBAndRedisBackend) insertQueryBatcher( select { // it may panic when reading on a closed feeder channel. feederOK detects if it was closed case <-stop: - Log().Infof("MySQL query batcher stopped (#%d)", batcherId) + Log().Fields("batcherID", batcherId).Info("MySQL query batcher stopped") // Insert any remaining rows inserter(count) feederOk = false @@ -289,7 +283,12 @@ func (g *GuerrillaDBAndRedisBackend) insertQueryBatcher( vals = append(vals, row...) count++ - Log().Debug("new feeder row:", row, " cols:", len(row), " count:", count, " worker", batcherId) + Log().Fields( + "row", row, + "cols", len(row), + "count", count, + "worker", batcherId, + ).Debug("new feeder row") if count >= GuerrillaDBAndRedisBatchMax { inserter(GuerrillaDBAndRedisBatchMax) } @@ -365,7 +364,8 @@ func GuerrillaDbRedis() Decorator { Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error { configType := BaseConfig(&guerrillaDBAndRedisConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + bcfg, err := Svc.ExtractConfig( + ConfigProcessors, "guerrillaredisdb", backendConfig, configType) if err != nil { return err } @@ -382,7 +382,7 @@ func GuerrillaDbRedis() Decorator { // we loop so that if insertQueryBatcher panics, it can recover and go in again for { if feederOK := g.insertQueryBatcher(feeder, db, qbID, stop); !feederOK { - Log().Debugf("insertQueryBatcher exited (#%d)", qbID) + Log().Fields("qbID", qbID).Debug("insertQueryBatcher exited") return } Log().Debug("resuming insertQueryBatcher") @@ -394,16 +394,19 @@ func GuerrillaDbRedis() Decorator { })) Svc.AddShutdowner(ShutdownWith(func() error { - if err := db.Close(); err != nil { - Log().WithError(err).Error("close mysql failed") - } else { - Log().Infof("closed mysql") + if db != nil { + if err := db.Close(); err != nil { + Log().WithError(err).Error("close sql database") + } else { + Log().Info("closed sql database") + } } + if redisClient.conn != nil { if err := redisClient.conn.Close(); err != nil { Log().WithError(err).Error("close redis failed") } else { - Log().Infof("closed redis") + Log().Info("closed redis") } } // send a close signal to all query batchers to exit. @@ -434,20 +437,12 @@ func GuerrillaDbRedis() Decorator { e.MailFrom.String(), e.Subject, ts) - e.QueuedId = hash - + e.QueuedId.FromHex(hash) // Add extra headers - protocol := "SMTP" - if e.ESMTP { - protocol = "E" + protocol - } - if e.TLS { - protocol = protocol + "S" - } var addHead string addHead += "Delivered-To: " + to + "\r\n" addHead += "Received: from " + e.RemoteIP + " ([" + e.RemoteIP + "])\r\n" - addHead += " by " + e.RcptTo[0].Host + " with " + protocol + " id " + hash + "@" + e.RcptTo[0].Host + ";\r\n" + addHead += " by " + e.RcptTo[0].Host + " with " + e.Protocol().String() + " id " + hash + "@" + e.RcptTo[0].Host + ";\r\n" addHead += " " + time.Now().Format(time.RFC1123Z) + "\r\n" // data will be compressed when printed, with addHead added to beginning diff --git a/backends/p_header.go b/backends/p_header.go index 1bab2984..e2aa48c7 100644 --- a/backends/p_header.go +++ b/backends/p_header.go @@ -6,7 +6,7 @@ import ( "time" ) -type HeaderConfig struct { +type headerConfig struct { PrimaryHost string `json:"primary_mail_host"` } @@ -15,7 +15,7 @@ type HeaderConfig struct { // ---------------------------------------------------------------------------------- // Description : Adds delivery information headers to e.DeliveryHeader // ---------------------------------------------------------------------------------- -// Config Options: none +// Config Options: primary_mail_host - string of the primary mail hostname // --------------:------------------------------------------------------------------- // Input : e.Helo // : e.RemoteAddress @@ -34,15 +34,15 @@ func init() { // Sets e.DeliveryHeader part of the envelope with the generated header func Header() Decorator { - var config *HeaderConfig + var config *headerConfig Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error { - configType := BaseConfig(&HeaderConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + configType := BaseConfig(&headerConfig{}) + bcfg, err := Svc.ExtractConfig(ConfigProcessors, "header", backendConfig, configType) if err != nil { return err } - config = bcfg.(*HeaderConfig) + config = bcfg.(*headerConfig) return nil })) @@ -54,18 +54,11 @@ func Header() Decorator { if len(e.Hashes) > 0 { hash = e.Hashes[0] } - protocol := "SMTP" - if e.ESMTP { - protocol = "E" + protocol - } - if e.TLS { - protocol = protocol + "S" - } var addHead string addHead += "Delivered-To: " + to + "\n" addHead += "Received: from " + e.RemoteIP + " ([" + e.RemoteIP + "])\n" if len(e.RcptTo) > 0 { - addHead += " by " + e.RcptTo[0].Host + " with " + protocol + " id " + hash + "@" + e.RcptTo[0].Host + ";\n" + addHead += " by " + e.RcptTo[0].Host + " with " + e.Protocol().String() + " id " + hash + "@" + e.RcptTo[0].Host + ";\n" } addHead += " " + time.Now().Format(time.RFC1123Z) + "\n" // save the result diff --git a/backends/p_redis.go b/backends/p_redis.go index e63587e7..73450d61 100644 --- a/backends/p_redis.go +++ b/backends/p_redis.go @@ -61,7 +61,7 @@ func Redis() Decorator { // read the config into RedisProcessorConfig Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error { configType := BaseConfig(&RedisProcessorConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + bcfg, err := Svc.ExtractConfig(ConfigProcessors, "redis", backendConfig, configType) if err != nil { return err } @@ -88,7 +88,7 @@ func Redis() Decorator { if task == TaskSaveMail { hash := "" if len(e.Hashes) > 0 { - e.QueuedId = e.Hashes[0] + e.QueuedId.FromHex(e.Hashes[0]) hash = e.Hashes[0] var stringer fmt.Stringer // a compressor was set diff --git a/backends/p_redis_test.go b/backends/p_redis_test.go index eae4dd34..086f73d4 100644 --- a/backends/p_redis_test.go +++ b/backends/p_redis_test.go @@ -11,14 +11,22 @@ import ( func TestRedisGeneric(t *testing.T) { - e := mail.NewEnvelope("127.0.0.1", 1) + e := mail.NewEnvelope("127.0.0.1", 1, 10) e.RcptTo = append(e.RcptTo, mail.Address{User: "test", Host: "grr.la"}) l, _ := log.GetLogger("./test_redis.log", "debug") - g, err := New(BackendConfig{ - "save_process": "Hasher|Redis", - "redis_interface": "127.0.0.1:6379", - "redis_expire_seconds": 7200, + g, err := New("default", BackendConfig{ + ConfigProcessors: { + "redis": { + "redis_interface": "127.0.0.1:6379", + "redis_expire_seconds": 7200, + }, + }, + ConfigGateways: { + "default": { + "save_process": "Hasher|Redis", + }, + }, }, l) if err != nil { t.Error(err) diff --git a/backends/p_sql.go b/backends/p_sql.go index 8925b186..9e7e8fee 100644 --- a/backends/p_sql.go +++ b/backends/p_sql.go @@ -142,14 +142,19 @@ func (s *SQLProcessor) prepareInsertQuery(rows int, db *sql.DB) *sql.Stmt { func (s *SQLProcessor) doQuery(c int, db *sql.DB, insertStmt *sql.Stmt, vals *[]interface{}) (execErr error) { defer func() { if r := recover(); r != nil { - Log().Error("Recovered form panic:", r, string(debug.Stack())) sum := 0 for _, v := range *vals { if str, ok := v.(string); ok { sum = sum + len(str) } } - Log().Errorf("panic while inserting query [%s] size:%d, err %v", r, sum, execErr) + Log().Fields( + "panic", fmt.Sprintf("%v", r), + "size", sum, + "error", execErr, + "stack", string(debug.Stack()), + ). + Error("panic while inserting query") panic("query failed") } }() @@ -194,7 +199,7 @@ func SQL() Decorator { // open the database connection (it will also check if we can select the table) Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error { configType := BaseConfig(&SQLProcessorConfig{}) - bcfg, err := Svc.ExtractConfig(backendConfig, configType) + bcfg, err := Svc.ExtractConfig(ConfigProcessors, "sql", backendConfig, configType) if err != nil { return err } @@ -224,9 +229,8 @@ func SQL() Decorator { hash := "" if len(e.Hashes) > 0 { hash = e.Hashes[0] - e.QueuedId = e.Hashes[0] + e.QueuedId.FromHex(e.Hashes[0]) } - var co *DataCompressor // a compressor was set by the Compress processor if c, ok := e.Values["zlib-compressor"]; ok { diff --git a/backends/p_sql_test.go b/backends/p_sql_test.go index 076297a4..d795fa91 100644 --- a/backends/p_sql_test.go +++ b/backends/p_sql_test.go @@ -31,14 +31,15 @@ func TestSQL(t *testing.T) { t.Fatal("get logger:", err) } - cfg := BackendConfig{ - "save_process": "sql", - "mail_table": *mailTableFlag, - "primary_mail_host": "example.com", - "sql_driver": *sqlDriverFlag, - "sql_dsn": *sqlDSNFlag, - } - backend, err := New(cfg, logger) + cfg := BackendConfig{} + + cfg.SetValue(ConfigGateways, DefaultGateway, "save_process", "sql") + cfg.SetValue(ConfigProcessors, "sql", "mail_table", *mailTableFlag) + cfg.SetValue(ConfigProcessors, "sql", "primary_mail_host", "example.com") + cfg.SetValue(ConfigProcessors, "sql", "sql_driver", *sqlDriverFlag) + cfg.SetValue(ConfigProcessors, "sql", "sql_dsn", *sqlDSNFlag) + + backend, err := New(DefaultGateway, cfg, logger) if err != nil { t.Fatal("new backend:", err) } diff --git a/backends/processor.go b/backends/processor.go index e5a29a7e..651afad9 100644 --- a/backends/processor.go +++ b/backends/processor.go @@ -2,6 +2,7 @@ package backends import ( "github.com/flashmob/go-guerrilla/mail" + "io" ) type SelectTask int @@ -9,6 +10,7 @@ type SelectTask int const ( TaskSaveMail SelectTask = iota TaskValidateRcpt + TaskSaveMailStream ) func (o SelectTask) String() string { @@ -17,6 +19,8 @@ func (o SelectTask) String() string { return "save mail" case TaskValidateRcpt: return "validate recipient" + case TaskSaveMailStream: + return "save mail stream" } return "[unnamed task]" } @@ -49,3 +53,27 @@ func (w DefaultProcessor) Process(e *mail.Envelope, task SelectTask) (Result, er // if no processors specified, skip operation type NoopProcessor struct{ DefaultProcessor } + +type StreamProcessor interface { + io.Writer +} + +type StreamProcessWith func(p []byte) (n int, err error) + +func (f StreamProcessWith) Write(p []byte) (n int, err error) { + // delegate to the anonymous function + return f(p) +} + +type DefaultStreamProcessor struct{} + +func (w DefaultStreamProcessor) Write(p []byte) (n int, err error) { + return len(p), nil +} + +// NoopStreamProcessor does nothing, it's a placeholder when no stream processors have been configured +type NoopStreamProcessor struct{ DefaultStreamProcessor } + +type ValidatingProcessor interface { + Processor +} diff --git a/backends/reader.go b/backends/reader.go new file mode 100644 index 00000000..ffcee130 --- /dev/null +++ b/backends/reader.go @@ -0,0 +1,8 @@ +package backends + +import "io" + +type SeekPartReader interface { + io.Reader + SeekPart(part int) error +} diff --git a/backends/s_buffer.go b/backends/s_buffer.go new file mode 100644 index 00000000..3fe738c0 --- /dev/null +++ b/backends/s_buffer.go @@ -0,0 +1,47 @@ +package backends + +import ( + "bytes" + "github.com/flashmob/go-guerrilla/mail" + "io" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: buffer +// ---------------------------------------------------------------------------------- +// Description : Buffers the message data to envelope.Data +// ---------------------------------------------------------------------------------- +// Config Options: +// --------------:------------------------------------------------------------------- +// Input : +// ---------------------------------------------------------------------------------- +// Output : envelope.Data +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["buffer"] = func() *StreamDecorator { + return StreamProcess() + } +} + +// Buffers to envelope.Data so that processors can be called on it at the end +func StreamProcess() *StreamDecorator { + sd := &StreamDecorator{} + sd.Decorate = + + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var envelope *mail.Envelope + sd.Open = func(e *mail.Envelope) error { + envelope = e + return nil + } + + return StreamProcessWith(func(p []byte) (int, error) { + tr := io.TeeReader(bytes.NewReader(p), sp) + n, err := envelope.Data.ReadFrom(tr) + return int(n), err + }) + } + + return sd +} diff --git a/backends/s_compress.go b/backends/s_compress.go new file mode 100644 index 00000000..5a218165 --- /dev/null +++ b/backends/s_compress.go @@ -0,0 +1,48 @@ +package backends + +import ( + "compress/zlib" + "io" + + "github.com/flashmob/go-guerrilla/mail" +) + +func init() { + Streamers["compress"] = func() *StreamDecorator { + return StreamCompress() + } +} + +type streamCompressConfig struct { + CompressLevel int `json:"compress_level,omitempty"` +} + +func StreamCompress() *StreamDecorator { + sd := &StreamDecorator{} + var config streamCompressConfig + sd.Configure = func(cfg ConfigGroup) error { + if _, ok := cfg["compress_level"]; !ok { + cfg["compress_level"] = zlib.BestSpeed + } + return sd.ExtractConfig(cfg, &config) + } + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var zw io.WriteCloser + sd.Open = func(e *mail.Envelope) error { + var err error + zw, err = zlib.NewWriterLevel(sp, config.CompressLevel) + return err + } + + sd.Close = func() error { + return zw.Close() + } + + return StreamProcessWith(func(p []byte) (int, error) { + return zw.Write(p) + }) + + } + return sd +} diff --git a/backends/s_debug.go b/backends/s_debug.go new file mode 100644 index 00000000..18d79268 --- /dev/null +++ b/backends/s_debug.go @@ -0,0 +1,64 @@ +package backends + +import ( + "github.com/flashmob/go-guerrilla/mail" + "time" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: debugger +// ---------------------------------------------------------------------------------- +// Description : Log received emails +// ---------------------------------------------------------------------------------- +// Config Options: log_reads bool - log if true +// : sleep_seconds - how many seconds to pause for, useful to force a +// : timeout. If sleep_seconds is 1 then a panic will be induced +// --------------:------------------------------------------------------------------- +// Input : email envelope +// ---------------------------------------------------------------------------------- +// Output : none (only output to the log if enabled) +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["debug"] = func() *StreamDecorator { + return StreamDebug() + } +} + +type streamDebuggerConfig struct { + LogReads bool `json:"log_reads"` + SleepSec int `json:"sleep_seconds,omitempty"` +} + +func StreamDebug() *StreamDecorator { + sd := &StreamDecorator{} + var config streamDebuggerConfig + var envelope *mail.Envelope + sd.Configure = func(cfg ConfigGroup) error { + return sd.ExtractConfig(cfg, &config) + } + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + sd.Open = func(e *mail.Envelope) error { + envelope = e + return nil + } + return StreamProcessWith(func(p []byte) (int, error) { + + if config.LogReads { + Log().Fields("queuedID", envelope.QueuedId, "payload", string(p)).Info("debug stream") + } + if config.SleepSec > 0 { + Log().Fields("queuedID", envelope.QueuedId, "sleep", config.SleepSec).Info("sleeping") + time.Sleep(time.Second * time.Duration(config.SleepSec)) + Log().Fields("queuedID", envelope.QueuedId).Info("woke up") + + if config.SleepSec == 1 { + panic("panic on purpose") + } + } + return sp.Write(p) + }) + } + return sd +} diff --git a/backends/s_decompress.go b/backends/s_decompress.go new file mode 100644 index 00000000..91775af5 --- /dev/null +++ b/backends/s_decompress.go @@ -0,0 +1,98 @@ +package backends + +import ( + "bytes" + "compress/zlib" + "github.com/flashmob/go-guerrilla/mail" + "io" + "sync" +) + +func init() { + Streamers["decompress"] = func() *StreamDecorator { + return StreamDecompress() + } +} + +// StreamDecompress is a PoC demonstrating how we can connect an io.Reader to our Writer +// We use an io.Pipe to connect the two, writing to one end of the pipe, while +// consuming the output on the other end of the pipe. + +func StreamDecompress() *StreamDecorator { + sd := &StreamDecorator{} + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var ( + zr io.ReadCloser + pr *io.PipeReader + pw *io.PipeWriter + ) + var wg sync.WaitGroup + // consumer runs as a gorouitne. + // It connects the zlib reader with the read-end of the pipe + // then copies the output down to the next stream processor + // consumer will exit of the pipe gets closed or on error + consumer := func() { + defer wg.Done() + var err error + for { + if zr == nil { + zr, err = zlib.NewReader(pr) + if err != nil { + _ = pr.CloseWithError(err) + return + } + } + + _, err := io.Copy(sp, zr) + if err != nil { + _ = pr.CloseWithError(err) + return + } + } + } + + // start our consumer goroutine + sd.Open = func(e *mail.Envelope) error { + pr, pw = io.Pipe() + wg.Add(1) + go consumer() + return nil + } + + // close both ends of the pipes when finished + sd.Close = func() error { + // stop the consumer + errR := pr.Close() + errW := pw.Close() + if zr != nil { + if err := zr.Close(); err != nil { + return err + } + } + if errR != nil { + return errR + } + if errW != nil { + return errW + } + // wait for the consumer to stop + wg.Wait() + pr = nil + pw = nil + zr = nil + return nil + } + + return StreamProcessWith(func(p []byte) (n int, err error) { + // take the output and copy on the pipe, for the consumer to pick up + N, err := io.Copy(pw, bytes.NewReader(p)) + if N > 0 { + n = int(N) + } + return + }) + + } + return sd +} diff --git a/backends/s_header.go b/backends/s_header.go new file mode 100644 index 00000000..e292a58b --- /dev/null +++ b/backends/s_header.go @@ -0,0 +1,90 @@ +package backends + +import ( + "github.com/flashmob/go-guerrilla/mail" + "io" + "strings" + "time" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: header +// ---------------------------------------------------------------------------------- +// Description : Adds delivery information headers to e.DeliveryHeader +// ---------------------------------------------------------------------------------- +// Config Options: primary_mail_host - string of the primary mail hostname +// --------------:------------------------------------------------------------------- +// Input : e.Helo +// : e.RemoteAddress +// : e.RcptTo +// : e.Hashes +// ---------------------------------------------------------------------------------- +// Output : Sets e.DeliveryHeader with additional delivery info +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["header"] = func() *StreamDecorator { + return StreamHeader() + } +} + +type streamHeader struct { + addHead []byte + w io.Writer + i int +} + +func newStreamHeader(w io.Writer) *streamHeader { + sc := new(streamHeader) + sc.w = w + return sc +} + +func (sh *streamHeader) addHeader(e *mail.Envelope, config *headerConfig) { + to := strings.TrimSpace(e.RcptTo[0].User) + "@" + config.PrimaryHost + hash := "unknown" + if len(e.Hashes) > 0 { + hash = e.Hashes[0] + } + var addHead string + addHead += "Delivered-To: " + to + "\n" + addHead += "Received: from " + e.Helo + " (" + e.Helo + " [" + e.RemoteIP + "])\n" + if len(e.RcptTo) > 0 { + addHead += " by " + e.RcptTo[0].Host + " with SMTP id " + hash + "@" + e.RcptTo[0].Host + ";\n" + } + addHead += " " + time.Now().Format(time.RFC1123Z) + "\n" + sh.addHead = []byte(addHead) +} + +func StreamHeader() *StreamDecorator { + hc := headerConfig{} + sd := &StreamDecorator{} + sd.Configure = func(cfg ConfigGroup) error { + return sd.ExtractConfig(cfg, &hc) + } + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var sh *streamHeader + sd.Open = func(e *mail.Envelope) error { + sh = newStreamHeader(sp) + sh.addHeader(e, &hc) + return nil + } + return StreamProcessWith(func(p []byte) (int, error) { + if sh.i < len(sh.addHead) { + for { + if N, err := sh.w.Write(sh.addHead[sh.i:]); err != nil { + return N, err + } else { + sh.i += N + if sh.i >= len(sh.addHead) { + break + } + } + } + } + return sp.Write(p) + }) + } + return sd +} diff --git a/backends/s_headers_parser.go b/backends/s_headers_parser.go new file mode 100644 index 00000000..e7807df9 --- /dev/null +++ b/backends/s_headers_parser.go @@ -0,0 +1,77 @@ +package backends + +import ( + "github.com/flashmob/go-guerrilla/mail" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: HeadersParser +// ---------------------------------------------------------------------------------- +// Description : Populates the envelope.Header value. +// : It also decodes the subject to UTF-8 +//----------------------------------------------------------------------------------- +// Requires : "mimeanalyzer" stream processor to be enabled before it +// ---------------------------------------------------------------------------------- +// Config Options: None +// --------------:------------------------------------------------------------------- +// Input : e.MimeParts generated by the mimeanalyzer processor +// ---------------------------------------------------------------------------------- +// Output : populates e.Header and e.Subject values of the envelope. +// : Any encoded data in the subject is decoded to UTF-8 +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["headersparser"] = func() *StreamDecorator { + return StreamHeadersParser() + } +} + +const stateHeaderScanning = 0 +const stateHeaderNotScanning = 1 + +func StreamHeadersParser() *StreamDecorator { + + sd := &StreamDecorator{} + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var ( + state byte + envelope *mail.Envelope + ) + + sd.Open = func(e *mail.Envelope) error { + state = stateHeaderScanning + envelope = e + return nil + } + + sd.Close = func() error { + return nil + } + + return StreamProcessWith(func(p []byte) (int, error) { + switch state { + case stateHeaderScanning: + if envelope.MimeParts != nil { + // copy the the headers of the first mime-part to envelope.Header + // then call envelope.ParseHeaders() + if len(*envelope.MimeParts) > 0 { + headers := (*envelope.MimeParts)[0].Headers + if headers != nil && len(headers) > 0 { + state = stateHeaderNotScanning + envelope.Header = headers + _ = envelope.ParseHeaders() + } + } + } + return sp.Write(p) + } + // state is stateHeaderNotScanning + // just forward everything to the underlying writer + return sp.Write(p) + + }) + } + + return sd +} diff --git a/backends/s_mimeanalyzer.go b/backends/s_mimeanalyzer.go new file mode 100644 index 00000000..31372139 --- /dev/null +++ b/backends/s_mimeanalyzer.go @@ -0,0 +1,94 @@ +package backends + +import ( + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/mimeparse" +) + +// ---------------------------------------------------------------------------------- +// Name : Mime Analyzer +// ---------------------------------------------------------------------------------- +// Description : Analyse the MIME structure of a stream. +// : Headers of each part are unfolded and saved in a *mime.Parts struct. +// : No decoding or any other processing. +// ---------------------------------------------------------------------------------- +// Config Options: +// --------------:------------------------------------------------------------------- +// Input : +// ---------------------------------------------------------------------------------- +// Output : MimeParts (of type *mime.Parts) stored in the envelope.MimeParts field +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["mimeanalyzer"] = func() *StreamDecorator { + return StreamMimeAnalyzer() + } +} + +func StreamMimeAnalyzer() *StreamDecorator { + + sd := &StreamDecorator{} + var ( + envelope *mail.Envelope + mimeErr error + parser *mimeparse.Parser + ) + sd.Configure = func(cfg ConfigGroup) error { + parser = mimeparse.NewMimeParser() + return nil + } + sd.Shutdown = func() error { + // assumed that parser has been closed, but we can call close again just to make sure + _ = parser.Close() + parser = nil + return nil + } + + sd.Decorate = + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + + sd.Open = func(e *mail.Envelope) error { + parser.Open() + envelope = e + mimeErr = nil + envelope.MimeError = nil + return nil + } + + sd.Close = func() error { + closeErr := parser.Close() + if mimeErr == nil { + mimeErr = closeErr + } + + envelope.MimeError = mimeErr + + if mimeErr != nil { + Log().WithError(closeErr).Warn("mime parse error in mimeanalyzer on close") + envelope.MimeError = nil + + if err, ok := mimeErr.(*mimeparse.Error); ok && err.ParseError() { + // dont propagate parse errors && NotMime error + return nil + } + } + return mimeErr + } + + return StreamProcessWith(func(p []byte) (int, error) { + if envelope.MimeParts == nil { + envelope.MimeParts = &parser.Parts + } + if mimeErr == nil { + mimeErr = parser.Parse(p) + if mimeErr != nil { + Log().WithError(mimeErr).Warn("mime parse error in mimeanalyzer") + } + } + return sp.Write(p) + }) + } + + return sd + +} diff --git a/backends/s_transformer.go b/backends/s_transformer.go new file mode 100644 index 00000000..f9e12b94 --- /dev/null +++ b/backends/s_transformer.go @@ -0,0 +1,337 @@ +package backends + +import ( + "bytes" + "io" + "regexp" + "sync" + + "github.com/flashmob/go-guerrilla/chunk/transfer" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/mimeparse" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: transformer +// ---------------------------------------------------------------------------------- +// Description : Transforms from base64 / q-printable to 8bit and converts charset to utf-8 +// ---------------------------------------------------------------------------------- +// Config Options: +// --------------:------------------------------------------------------------------- +// Input : envelope.MimeParts +// ---------------------------------------------------------------------------------- +// Output : 8bit mime message, with charsets decoded to UTF-8 +// : Note that this processor changes the body counts. Therefore, it makes +// : a new instance of envelope.MimeParts which is then populated +// : by parsing the new re-written message +// ---------------------------------------------------------------------------------- + +func init() { + Streamers["transformer"] = func() *StreamDecorator { + return Transformer() + } +} + +// Transform stream processor: convert an email to UTF-8 +type Transform struct { + sp io.Writer + isBody bool // the next bytes to be sent are body? + buf bytes.Buffer + current *mimeparse.Part + decoder io.Reader + pr *io.PipeReader + pw *io.PipeWriter + partsCachedOriginal *mimeparse.Parts + envelope *mail.Envelope + + // we re-parse the output since the counts have changed + // parser implements the io.Writer interface, here output will be sent to it and then forwarded to the next processor + parser *mimeparse.Parser +} + +// swap caches the original parts from envelope.MimeParts +// and point them to our parts +func (t *Transform) swap() *mimeparse.Parts { + parts := t.envelope.MimeParts + if parts != nil { + t.partsCachedOriginal = parts + parts = &t.parser.Parts + return parts + } + return nil +} + +// unswap points the parts from MimeParts back to the original ones +func (t *Transform) unswap() { + if t.envelope.MimeParts != nil { + t.envelope.MimeParts = t.partsCachedOriginal + } +} + +// regexpCharset captures the charset value +var regexpCharset = regexp.MustCompile("(?i)charset=\"?(.+)\"?") // (?i) is a flag for case-insensitive + +func (t *Transform) ReWrite(b []byte, last bool) (count int, err error) { + defer func() { + count = len(b) + }() + if !t.isBody { + // Header re-write, how it works + // we place the partial header's bytes on a buffer from which we can read one line at a time + // then we match and replace the lines we want, output replaced live. + // The following re-writes are mde: + // - base64 => 8bit + // - supported non-utf8 charset => utf8 + if i, err := io.Copy(&t.buf, bytes.NewReader(b)); err != nil { + return int(i), err + } + var charsetProcessed bool + charsetFrom := "" + for { + line, rErr := t.buf.ReadBytes('\n') + if rErr == nil { + if !charsetProcessed { + // is charsetFrom supported? + exists := t.current.Headers.Get("content-type") + if exists != "" { + charsetProcessed = true + charsetFrom = t.current.ContentType.Charset() + if !mail.SupportsCharset(charsetFrom) { + charsetFrom = "" + } + } + } + + if bytes.Contains(line, []byte("Content-Transfer-Encoding: base64")) { + line = bytes.Replace(line, []byte("base64"), []byte("8bit"), 1) + } else if bytes.Contains(line, []byte("charset")) { + if match := regexpCharset.FindSubmatch(line); match != nil && len(match) > 0 { + // test if the encoding is supported + if charsetFrom != "" { + // it's supported, we can change it to utf8 + line = regexpCharset.ReplaceAll(line, []byte("charset=utf8")) + } + } + } + _, err = io.Copy(t.parser, bytes.NewReader(line)) + if err != nil { + return + } + if line[0] == '\n' { + // end of header + break + } + } else { + return + } + } + } else { + + if ct := t.current.ContentType.Supertype(); ct == "multipart" || ct == "message" { + _, err = io.Copy(t.parser, bytes.NewReader(b)) + return + } + + // Body Decode, how it works: + // First, the decoder is setup, depending on the source encoding type. + // Next, since the decoder is an io.Reader, we need to use a pipe to connect it. + // Subsequent calls write to the pipe in a goroutine and the parent-thread copies the result to the output stream + // The routine stops feeding the decoder data before EndingPosBody, and not decoding anything after, but still + // outputting the un-decoded remainder. + // The decoder is destroyed at the end of the body (when last == true) + + t.pr, t.pw = io.Pipe() + if t.decoder == nil { + t.buf.Reset() + // the decoder will be reading from an underlying pipe + charsetFrom := t.current.ContentType.Charset() + if charsetFrom == "" { + charsetFrom = mail.MostCommonCharset + } + + if mail.SupportsCharset(charsetFrom) { + t.decoder, err = transfer.NewBodyDecoder(t.pr, transfer.ParseEncoding(t.current.TransferEncoding), charsetFrom) + if err != nil { + return + } + t.current.Charset = "utf8" + t.current.TransferEncoding = "8bit" + } + } + + wg := sync.WaitGroup{} + wg.Add(1) + + // out is the slice that will be decoded + var out []byte + // remainder will not be decoded. Typically, this contains the boundary maker, and we want to preserve it + var remainder []byte + if t.current.EndingPosBody > 0 { + size := t.current.EndingPosBody - t.current.StartingPosBody - 1 // -1 since we do not want \n + out = b[:size] + remainder = b[size:] + } else { + // use the entire slice + out = b + } + go func() { + // stream our slice to the pipe + defer wg.Done() + _, pRrr := io.Copy(t.pw, bytes.NewReader(out)) + if pRrr != nil { + _ = t.pw.CloseWithError(err) + return + } + _ = t.pw.Close() + }() + // do the decoding + var i int64 + i, err = io.Copy(t.parser, t.decoder) + // wait for the pipe to finish + wg.Wait() + _ = t.pr.Close() + + if last { + t.decoder = nil + } + count += int(i) + if err != nil { + return + } + // flush any remainder + if len(remainder) > 0 { + i, err = io.Copy(t.parser, bytes.NewReader(remainder)) + count += int(i) + if err != nil { + return + } + } + } + return count, err +} + +func (t *Transform) Reset() { + t.decoder = nil +} + +func Transformer() *StreamDecorator { + + var ( + msgPos uint + progress int + ) + reWriter := Transform{} + + sd := &StreamDecorator{} + sd.Decorate = + + func(sp StreamProcessor, a ...interface{}) StreamProcessor { + var ( + envelope *mail.Envelope + // total is the total number of bytes written + total int64 + // pos tracks the current position of the output slice + pos int + // written is the number of bytes written out in this call + written int + ) + + if reWriter.sp == nil { + reWriter.sp = sp + } + + sd.Open = func(e *mail.Envelope) error { + envelope = e + if reWriter.parser == nil { + reWriter.parser = mimeparse.NewMimeParserWriter(sp) + reWriter.parser.Open() + } + reWriter.envelope = envelope + return nil + } + + sd.Close = func() error { + total = 0 + return reWriter.parser.Close() + } + + end := func(part *mimeparse.Part, offset uint, p []byte, start uint) (int, error) { + var err error + var count int + + count, err = reWriter.ReWrite(p[pos:start-offset], true) + + written += count + if err != nil { + return count, err + } + reWriter.current = part + pos += count + return count, nil + } + + return StreamProcessWith(func(p []byte) (count int, err error) { + pos = 0 + written = 0 + parts := envelope.MimeParts + if parts != nil && len(*parts) > 0 { + + // we are going to change envelope.MimeParts to our own copy with our own counts + envelope.MimeParts = reWriter.swap() + defer func() { + reWriter.unswap() + total += int64(written) + }() + + offset := msgPos + reWriter.current = (*parts)[0] + for i := progress; i < len(*parts); i++ { + part := (*parts)[i] + // break chunk on new part + if part.StartingPos > 0 && part.StartingPos >= msgPos { + count, err = end(part, offset, p, part.StartingPos) + if err != nil { + break + } + msgPos = part.StartingPos + reWriter.isBody = false + + } + // break chunk on header (found the body) + if part.StartingPosBody > 0 && part.StartingPosBody >= msgPos { + count, err = end(part, offset, p, part.StartingPosBody) + if err != nil { + break + } + reWriter.isBody = true + msgPos += uint(count) + + } + + // if on the latest (last) part, and yet there is still data to be written out + if len(*parts)-1 == i && len(p)-1 > pos { + count, err = reWriter.ReWrite(p[pos:], false) + + written += count + if err != nil { + break + } + pos += count + msgPos += uint(count) + } + // if there's no more data + if pos >= len(p) { + break + } + } + if len(*parts) > 2 { + progress = len(*parts) - 2 // skip to 2nd last part, assume previous parts are already processed + } + } + // note that in this case, ReWrite method will output the stream to further processors down the line + // here we just return back with the result + return written, err + }) + } + return sd +} diff --git a/chunk/buffer.go b/chunk/buffer.go new file mode 100644 index 00000000..ee7de6fb --- /dev/null +++ b/chunk/buffer.go @@ -0,0 +1,172 @@ +package chunk + +import ( + "crypto/md5" + "errors" + "hash" + "strings" + + "github.com/flashmob/go-guerrilla/mail/mimeparse" +) + +type flushEvent func() error + +type chunkingBuffer struct { + buf []byte + flushTrigger flushEvent +} + +// Flush signals that it's time to write the buffer out to storage +func (c *chunkingBuffer) Flush() error { + if len(c.buf) == 0 { + return nil + } + if c.flushTrigger != nil { + if err := c.flushTrigger(); err != nil { + return err + } + } + c.Reset() + return nil +} + +// Reset sets the length back to 0, making it re-usable +func (c *chunkingBuffer) Reset() { + c.buf = c.buf[:0] // set the length back to 0 +} + +// Write takes a p slice of bytes and writes it to the buffer. +// It will never grow the buffer, flushing it as soon as it's full. +func (c *chunkingBuffer) Write(p []byte) (i int, err error) { + remaining := len(p) // number of bytes remaining to write + bufCap := cap(c.buf) + for { + free := bufCap - len(c.buf) + if free > remaining { + // enough of room in the buffer + c.buf = append(c.buf, p[i:i+remaining]...) + i += remaining + return + } else { + // fill the buffer to the 'brim' with a slice from p + c.buf = append(c.buf, p[i:i+free]...) + remaining -= free + i += free + err = c.Flush() + if err != nil { + return i, err + } + if remaining == 0 { + return + } + } + } +} + +// CapTo caps the internal buffer to specified number of bytes, sets the length back to 0 +func (c *chunkingBuffer) CapTo(n int) { + if cap(c.buf) == n { + return + } + c.buf = make([]byte, 0, n) +} + +// ChunkingBufferMime decorates chunkingBuffer, defining what to do when a flush event is triggered +// in other words, +type ChunkingBufferMime struct { + chunkingBuffer + current *mimeparse.Part + Info PartsInfo + md5 hash.Hash + database Storage +} + +func NewChunkedBytesBufferMime() *ChunkingBufferMime { + b := new(ChunkingBufferMime) + b.chunkingBuffer.flushTrigger = func() error { + return b.onFlush() + } + b.md5 = md5.New() + b.buf = make([]byte, 0, chunkMaxBytes) + return b +} + +func (b *ChunkingBufferMime) SetDatabase(database Storage) { + b.database = database +} + +// onFlush is called whenever the flush event fires. +// - It saves the chunk to disk and adds the chunk's hash to the list. +// - It builds the b.Info.Parts structure +func (b *ChunkingBufferMime) onFlush() error { + b.md5.Write(b.buf) + var chash HashKey + copy(chash[:], b.md5.Sum([]byte{})) + if b.current == nil { + return errors.New("b.current part is nil") + } + if size := len(b.Info.Parts); size > 0 && b.Info.Parts[size-1].PartId == b.current.Node { + // existing part, just append the hash + lastPart := &b.Info.Parts[size-1] + lastPart.ChunkHash = append(lastPart.ChunkHash, chash) + b.fillInfo(lastPart, size-1) + lastPart.Size += uint(len(b.buf)) + } else { + // add it as a new part + part := ChunkedPart{ + PartId: b.current.Node, + ChunkHash: []HashKey{chash}, + ContentBoundary: b.Info.boundary(b.current.ContentBoundary), + Size: uint(len(b.buf)), + } + b.fillInfo(&part, 0) + b.Info.Parts = append(b.Info.Parts, part) + b.Info.Count++ + } + if err := b.database.AddChunk(b.buf, chash[:]); err != nil { + return err + } + return nil +} + +func (b *ChunkingBufferMime) fillInfo(cp *ChunkedPart, index int) { + if cp.ContentType == "" && b.current.ContentType != nil { + cp.ContentType = b.current.ContentType.String() + } + if cp.Charset == "" && b.current.Charset != "" { + cp.Charset = b.current.Charset + } + if cp.TransferEncoding == "" && b.current.TransferEncoding != "" { + cp.TransferEncoding = b.current.TransferEncoding + } + if cp.ContentDisposition == "" && b.current.ContentDisposition != "" { + cp.ContentDisposition = b.current.ContentDisposition + if strings.Contains(cp.ContentDisposition, "attach") { + b.Info.HasAttach = true + } + } + if cp.ContentType != "" { + if b.Info.TextPart == -1 && strings.Contains(cp.ContentType, "text/plain") { + b.Info.TextPart = index + } else if b.Info.HTMLPart == -1 && strings.Contains(cp.ContentType, "text/html") { + b.Info.HTMLPart = index + } + } +} + +// Reset decorates the Reset method of the chunkingBuffer +func (b *ChunkingBufferMime) Reset() { + b.md5.Reset() + b.chunkingBuffer.Reset() +} + +// CurrentPart sets the current mime part that's being buffered +func (b *ChunkingBufferMime) CurrentPart(cp *mimeparse.Part) { + if b.current == nil { + b.Info = *NewPartsInfo() + b.Info.Parts = make([]ChunkedPart, 0, 3) + b.Info.TextPart = -1 + b.Info.HTMLPart = -1 + } + b.current = cp +} diff --git a/chunk/chunk.go b/chunk/chunk.go new file mode 100644 index 00000000..a79387a2 --- /dev/null +++ b/chunk/chunk.go @@ -0,0 +1,149 @@ +package chunk + +import ( + "bytes" + "compress/zlib" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "sync" +) + +const chunkMaxBytes = 1024 * 16 +const hashByteSize = 16 + +type HashKey [hashByteSize]byte + +// Pack takes a slice and copies each byte to HashKey internal representation +func (h *HashKey) Pack(b []byte) { + if len(b) < hashByteSize { + return + } + copy(h[:], b[0:hashByteSize]) +} + +// String implements the Stringer interface from fmt.Stringer +func (h HashKey) String() string { + return base64.RawStdEncoding.EncodeToString(h[0:hashByteSize]) +} + +// Hex returns the hash, encoded in hexadecimal +func (h HashKey) Hex() string { + return fmt.Sprintf("%x", h[:]) +} + +// UnmarshalJSON implements the Unmarshaler interface from encoding/json +func (h *HashKey) UnmarshalJSON(b []byte) error { + dbuf := make([]byte, base64.RawStdEncoding.DecodedLen(len(b[1:len(b)-1]))) + _, err := base64.RawStdEncoding.Decode(dbuf, b[1:len(b)-1]) + if err != nil { + return err + } + h.Pack(dbuf) + return nil +} + +// MarshalJSON implements the Marshaler interface from encoding/json +// The value is marshaled as a raw base64 to save some bytes +// eg. instead of typically using hex, de17038001170380011703ff01170380 would be represented as 3hcDgAEXA4ABFwP/ARcDgA +func (h *HashKey) MarshalJSON() ([]byte, error) { + return []byte(`"` + h.String() + `"`), nil +} + +// PartsInfo describes the mime-parts contained in the email +type PartsInfo struct { + Count uint32 `json:"c"` // number of parts + TextPart int `json:"tp"` // index of the main text part to display + HTMLPart int `json:"hp"` // index of the main html part to display (if any) + HasAttach bool `json:"a"` // is there an attachment? + Parts []ChunkedPart `json:"p"` // info describing a mime-part + CBoundaries []string `json:"cbl"` // content boundaries list + Err error `json:"e"` // any error encountered (mimeparse.MimeError) +} + +var bp sync.Pool // bytes.buffer pool + +// ChunkedPart contains header information about a mime-part, including keys pointing to where the data is stored at +type ChunkedPart struct { + PartId string `json:"i"` + Size uint `json:"s"` + ChunkHash []HashKey `json:"h"` // sequence of hashes the data is stored at + ContentType string `json:"t"` + Charset string `json:"c"` + TransferEncoding string `json:"e"` + ContentDisposition string `json:"d"` + ContentBoundary int `json:"cb"` // index to the CBoundaries list in PartsInfo +} + +func NewPartsInfo() *PartsInfo { + pi := new(PartsInfo) + bp = sync.Pool{ + // if not available, then create a new one + New: func() interface{} { + var b bytes.Buffer + return &b + }, + } + return pi +} + +// boundary takes a string and returns the index of the string in the info.CBoundaries slice +func (info *PartsInfo) boundary(cb string) int { + for i := range info.CBoundaries { + if info.CBoundaries[i] == cb { + return i + } + } + info.CBoundaries = append(info.CBoundaries, cb) + return len(info.CBoundaries) - 1 +} + +// UnmarshalJSON unmarshals the JSON and decompresses using zlib +func (info *PartsInfo) UnmarshalJSONZlib(b []byte) error { + + r, err := zlib.NewReader(bytes.NewReader(b[1 : len(b)-1])) + if err != nil { + return err + } + all, err := ioutil.ReadAll(r) + if err != nil { + return err + } + err = json.Unmarshal(all, info) + if err != nil { + return err + } + return nil +} + +// MarshalJSONZlib marshals and compresses the bytes using zlib +func (info *PartsInfo) MarshalJSONZlib() ([]byte, error) { + if len(info.Parts) == 0 { + return []byte{}, errors.New("message contained no parts, was mime analyzer") + } + buf, err := json.Marshal(info) + if err != nil { + return buf, err + } + // borrow a buffer form the pool + compressed := bp.Get().(*bytes.Buffer) + // put back in the pool + defer func() { + compressed.Reset() + bp.Put(compressed) + }() + + zlibw, err := zlib.NewWriterLevel(compressed, 9) + if err != nil { + return buf, err + } + if _, err := zlibw.Write(buf); err != nil { + return buf, err + } + if err := zlibw.Close(); err != nil { + return buf, err + } + return []byte(`"` + compressed.String() + `"`), nil +} diff --git a/chunk/chunk_test.go b/chunk/chunk_test.go new file mode 100644 index 00000000..e9a340f6 --- /dev/null +++ b/chunk/chunk_test.go @@ -0,0 +1,706 @@ +package chunk + +import ( + "bytes" + "fmt" + "github.com/flashmob/go-guerrilla/mail/smtp" + "io" + "os" + "strings" + "testing" + + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/chunk/transfer" + "github.com/flashmob/go-guerrilla/mail" +) + +func TestChunkedBytesBuffer(t *testing.T) { + var in string + + var buf chunkingBuffer + buf.CapTo(64) + + // the data to write is over-aligned + in = `123456789012345678901234567890123456789012345678901234567890abcde12345678901234567890123456789012345678901234567890123456789abcdef` // len == 130 + i, _ := buf.Write([]byte(in[:])) + if i != len(in) { + t.Error("did not write", len(in), "bytes") + } + + // the data to write is aligned + var buf2 chunkingBuffer + buf2.CapTo(64) + in = `123456789012345678901234567890123456789012345678901234567890abcde12345678901234567890123456789012345678901234567890123456789abcd` // len == 128 + i, _ = buf2.Write([]byte(in[:])) + if i != len(in) { + t.Error("did not write", len(in), "bytes") + } + + // the data to write is under-aligned + var buf3 chunkingBuffer + buf3.CapTo(64) + in = `123456789012345678901234567890123456789012345678901234567890abcde12345678901234567890123456789012345678901234567890123456789ab` // len == 126 + i, _ = buf3.Write([]byte(in[:])) + if i != len(in) { + t.Error("did not write", len(in), "bytes") + } + + // the data to write is smaller than the buffer + var buf4 chunkingBuffer + buf4.CapTo(64) + in = `1234567890` // len == 10 + i, _ = buf4.Write([]byte(in[:])) + if i != len(in) { + t.Error("did not write", len(in), "bytes") + } + + // what if the buffer already contains stuff before Write is called + // and the buffer len is smaller than the len of the slice of bytes we pass it? + var buf5 chunkingBuffer + buf5.CapTo(5) + buf5.buf = append(buf5.buf, []byte{'a', 'b', 'c'}...) + in = `1234567890` // len == 10 + i, _ = buf5.Write([]byte(in[:])) + if i != len(in) { + t.Error("did not write", len(in), "bytes") + } +} + +var n1 = `From: Al Gore +To: White House Transportation Coordinator +Subject: [Fwd: Map of Argentina with Description] +MIME-Version: 1.0 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=ncr424; d=reliancegeneral.co.in; + h=List-Unsubscribe:MIME-Version:From:To:Reply-To:Date:Subject:Content-Type:Content-Transfer-Encoding:Message-ID; i=prospects@prospects.reliancegeneral.co.in; + bh=F4UQPGEkpmh54C7v3DL8mm2db1QhZU4gRHR1jDqffG8=; + b=MVltcq6/I9b218a370fuNFLNinR9zQcdBSmzttFkZ7TvV2mOsGrzrwORT8PKYq4KNJNOLBahswXf + GwaMjDKT/5TXzegdX/L3f/X4bMAEO1einn+nUkVGLK4zVQus+KGqm4oP7uVXjqp70PWXScyWWkbT + 1PGUwRfPd/HTJG5IUqs= +Content-Type: multipart/mixed; + boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" + +This is a multi-part message in MIME format. +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: text/plain; charset=utf8 +Content-Transfer-Encoding: 7bit + +Fred, + +Fire up Air Force One! We're going South! + +Thanks, +Al +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Return-Path: +Received: from mailhost.whitehouse.gov ([192.168.51.200]) + by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 + for ; + Mon, 13 Aug 1998 l8:14:23 +1000 +Received: from the_big_box.whitehouse.gov ([192.168.51.50]) + by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 + for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 + Date: Mon, 13 Aug 1998 17:42:41 +1000 +Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> +From: Bill Clinton +To: A1 (The Enforcer) Gore +Subject: Map of Argentina with Description +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="DC8------------DC8638F443D87A7F0726DEF7" + +This is a multi-part message in MIME format. +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: text/plain; charset=utf8 +Content-Transfer-Encoding: 7bit + +Hi A1, + +I finally figured out this MIME thing. Pretty cool. I'll send you +some sax music in .au files next week! + +Anyway, the attached image is really too small to get a good look at +Argentina. Try this for a much better map: + +http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm + +Then again, shouldn't the CIA have something like that? + +Bill +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: image/png; name="three.png" +Content-Transfer-Encoding: 8bit + +` + +var email = `From: Al Gore +To: White House Transportation Coordinator +Subject: [Fwd: Map of Argentina with Description] +MIME-Version: 1.0 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=ncr424; d=reliancegeneral.co.in; + h=List-Unsubscribe:MIME-Version:From:To:Reply-To:Date:Subject:Content-Type:Content-Transfer-Encoding:Message-ID; i=prospects@prospects.reliancegeneral.co.in; + bh=F4UQPGEkpmh54C7v3DL8mm2db1QhZU4gRHR1jDqffG8=; + b=MVltcq6/I9b218a370fuNFLNinR9zQcdBSmzttFkZ7TvV2mOsGrzrwORT8PKYq4KNJNOLBahswXf + GwaMjDKT/5TXzegdX/L3f/X4bMAEO1einn+nUkVGLK4zVQus+KGqm4oP7uVXjqp70PWXScyWWkbT + 1PGUwRfPd/HTJG5IUqs= +Content-Type: multipart/mixed; + boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" + +This is a multi-part message in MIME format. +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Fred, + +Fire up Air Force One! We're going South! + +Thanks, +Al +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Return-Path: +Received: from mailhost.whitehouse.gov ([192.168.51.200]) + by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 + for ; + Mon, 13 Aug 1998 l8:14:23 +1000 +Received: from the_big_box.whitehouse.gov ([192.168.51.50]) + by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 + for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 + Date: Mon, 13 Aug 1998 17:42:41 +1000 +Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> +From: Bill Clinton +To: A1 (The Enforcer) Gore +Subject: Map of Argentina with Description +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="DC8------------DC8638F443D87A7F0726DEF7" + +This is a multi-part message in MIME format. +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Hi A1, + +I finally figured out this MIME thing. Pretty cool. I'll send you +some sax music in .au files next week! + +Anyway, the attached image is really too small to get a good look at +Argentina. Try this for a much better map: + +http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm + +Then again, shouldn't the CIA have something like that? + +Bill +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: image/gif; name="three.gif" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; filename="three.gif" + +R0lGODlhEAAeAPAAAP///wAAACH5BAEAAAAALAAAAAAQAB4AAAIfhI+py+0Po5y0onCD3lzbD15K +R4ZmiAHk6p3uC8dWAQA7 +--DC8------------DC8638F443D87A7F0726DEF7-- + +--D7F------------D7FD5A0B8AB9C65CCDBFA872-- + +` + +var email2 = `Delivered-To: b@sharklasers.com +Received: from aaa.cn (aaa.cn [220.178.145.250]) + by 163.com with SMTP id 41f596e02e4da6a74d878a630a7f175e@163.com; + Tue, 17 Sep 2019 01:16:43 +0000 +From: "=?utf-8?Q?=E6=B1=9F=E5=8D=97=E6=A2=A6=E6=96=AD=E6=A8=AA=E6=B1=9F=E6=B8=9A?=" +To: +Subject: =?utf-8?Q?=E5=BA=94=E5=8A=9B=E6=AF=94b?= +Date: Tue, 17 Sep 2019 09:16:29 +0800 +Message-ID: +MIME-Version: 1.0 +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: 8bit +X-Priority: 3 (Normal) +X-MSMail-Priority: Normal +X-Mailer: Microsoft Outlook IMO, Build 9.0.2416 (9.0.2911.0) +Importance: Normal +X-MimeOLE: Produced By Microsoft MimeOLE V6.00.2900.2180 + +

Date:2019-09-17
Account:b@163.com +

+ +

+

丅<交通指挥灯>載<硝化作用>а<柿子>ρ<竞争性领域>ρ<毛虫>签<非持久性病毒>到<街灯>領<麦蜘蛛>１<求索>６<企业标准>╇<气候条件>，<农村劳动力转移>登<穴深>杁<预备队>2<农机具>Ч<营养袋育苗>棋<生物肥料>牌<光照强度>2<改良系谱法>４<加大投入>9<茶叶产业办公室>８<省局>8<辐照>４<百亩>.<上行列车>ｃ<竹节>δ<山嘴>м<股所>，<科学谋划>吋<领海>款<异常型种子>１<董家村> <平面球形图>０<先来> <藜叶斑病>０<竞职>宋<二棱大麦>２<茎枯病> <宁洱县>０<共枕>鎵<钙镁磷肥>專<校正>員<砍青>４<独具特色>8<装草机>６<斑螟>1<补偿机制>３<创意策划>8<投稿家>３<茶点>2<量天尺枯萎腐烂病>０<河尾>翎<台湾稻螟>。<春城晚报>

+

Your words appear idle to me; give them proof, and I will listen.

` + +var email3 = `Delivered-To: nevaeh@sharklasers.com +Received: from bb_dyn_pb-146-88-38-36.violin.co.th (bb_dyn_pb-146-88-38-36.violin.co.th [146.88.38.36]) + by sharklasers.com with SMTP id d0e961595a207a79ab84603750372de8@sharklasers.com; + Tue, 17 Sep 2019 01:13:00 +0000 +Received: from mx03.listsystemsf.net [100.20.38.85] by mxs.perenter.com with SMTP; Tue, 17 Sep 2019 04:57:59 +0500 +Received: from mts.locks.grgtween.net ([Tue, 17 Sep 2019 04:52:27 +0500]) + by webmail.halftomorrow.com with LOCAL; Tue, 17 Sep 2019 04:52:27 +0500 +Received: from mail.naihautsui.co.kr ([61.220.30.1]) by mtu67.syds.piswix.net with ASMTP; Tue, 17 Sep 2019 04:47:25 +0500 +Received: from unknown (HELO mx03.listsystemsf.net) (Tue, 17 Sep 2019 04:41:45 +0500) + by smtp-server1.cfdenselr.com with LOCAL; Tue, 17 Sep 2019 04:41:45 +0500 +Message-ID: <78431AF2.E9B20F56@violin.co.th> +Date: Tue, 17 Sep 2019 04:14:56 +0500 +Reply-To: "Nevaeh" +From: "Nevaeh" +User-Agent: Mozilla 4.73 [de]C-CCK-MCD DT (Win98; U) +X-Accept-Language: en-us +MIME-Version: 1.0 +To: "Nevaeh" +Subject: czy m�glbys spotkac sie ze mna w weekend? +Content-Type: text/html; + charset="iso-8859-1"" +Content-Transfer-Encoding: base64 + +PCFkb2N0eXBlIGh0bWw+DQo8aHRtbD4NCjxoZWFkPg0KPG1ldGEgY2hhcnNldD0idXRmLTgiPg0K +PC9oZWFkPg0KPGJvZHk+DQo8dGFibGUgd2lkdGg9IjYwMCIgYm9yZGVyPSIwIiBhbGlnbj0iY2Vu +dGVyIiBzdHlsZT0iZm9udC1mYW1pbHk6IEFyaWFsOyBmb250LXNpemU6IDE4cHgiPg0KPHRib2R5 +Pg0KPHRyPg0KPHRoIGhlaWdodD0iNjAiIHNjb3BlPSJjb2wiPk5hamdvcmV0c3plIGR6aWV3Y3p5 +bnkgaSBzYW1vdG5lIGtvYmlldHksIGt083JlIGNoY2Egc2Vrc3UuPG9sPjwvb2w+PC90aD4NCjwv +dHI+DQo8dGQgaGVpZ2h0PSIyMjMiIGFsaWduPSJjZW50ZXIiPjxwPk5hIG5hc3plaiBzdHJvbmll +IGdyb21hZHpvbmUgc2EgbWlsaW9ueSBwcm9maWxpIGtvYmlldC4gV3N6eXNjeSBjaGNhIHRlcmF6 +IHBpZXByenljLjwvcD4NCjxoZWFkZXI+PC9oZWFkZXI+DQo8cD5OYSBwcnp5a2xhZCBzYSBXIFRX +T0lNIE1JRVNDSUUuIENoY2VzeiBpbm55Y2g/IFpuYWpkeiBuYSBuYXN6ZWogc3Ryb25pZSE8L3A+ +DQo8dGFibGUgY2xhc3M9Im1jbkJ1dHRvbkNvbnRlbnRDb250YWluZXIiIHN0eWxlPSJib3JkZXIt +Y29sbGFwc2U6IHNlcGFyYXRlICEgaW1wb3J0YW50O2JvcmRlci1yYWRpdXM6IDNweDtiYWNrZ3Jv +dW5kLWNvbG9yOiAjRTc0MTQxOyIgYm9yZGVyPSIwIiBjZWxsc3BhY2luZz0iMCIgY2VsbHBhZGRp +bmc9IjAiPg0KIDx0Ym9keT4NCiA8dHI+DQogPHRkIGNsYXNzPSJtY25CdXR0b25Db250ZW50IiBz +dHlsZT0iZm9udC1mYW1pbHk6IEFyaWFsOyBmb250LXNpemU6IDIycHg7IHBhZGRpbmc6IDE1cHgg +MjVweDsiIHZhbGlnbj0ibWlkZGxlIiBhbGlnbj0iY2VudGVyIj4NCiA8YSBjbGFzcz0ibWNuQnV0 +dG9uICIgaHJlZj0iaHR0cDovL2JldGhhbnkuc3UiIHRhcmdldD0iX2JsYW5rIiBzdHlsZT0iZm9u +dC13ZWlnaHQ6IG5vcm1hbDtsZXR0ZXItc3BhY2luZzogbm9ybWFsO2xpbmUtaGVpZ2h0OiAxMDAl +O3RleHQtYWxpZ246IGNlbnRlcjt0ZXh0LWRlY29yYXRpb246IG5vbmU7Y29sb3I6ICNGRkZGRkY7 +Ij5odHRwOi8vYmV0aGFueS5zdTwvYT4NCiA8L3RkPg0KIDwvdHI+DQogPC90Ym9keT4NCiA8L3Rh +YmxlPjx0YWJsZSB3aWR0aD0iMjglIiBib3JkZXI9IjAiPjx0Ym9keT48dHI+PHRkPjwvdGQ+PHRk +PjwvdGQ+PHRkPjwvdGQ+PC90cj48L3Rib2R5PjwvdGFibGU+DQo8dGFibGUgc3R5bGU9Im1pbi13 +aWR0aDoxMDAlOyIgY2xhc3M9Im1jblRleHRDb250ZW50Q29udGFpbmVyIiBhbGlnbj0ibGVmdCIg +Ym9yZGVyPSIwIiBjZWxscGFkZGluZz0iMCIgY2VsbHNwYWNpbmc9IjAiIHdpZHRoPSIxMDAlIj4N +Cjx0Ym9keT4NCiA8dHI+DQo8dGQgYWxpZ249ImNlbnRlciIgdmFsaWduPSJ0b3AiIGNsYXNzPSJt +Y25UZXh0Q29udGVudCIgc3R5bGU9InBhZGRpbmc6IDlweCAxOHB4O2NvbG9yOiAjNkI2QjZCO2Zv +bnQtZmFtaWx5OiBWZXJkYW5hLEdlbmV2YSxzYW5zLXNlcmlmO2ZvbnQtc2l6ZTogMTFweDsiPg0K +VXp5aiB0ZWdvIGxpbmt1LCBqZXNsaSBwcnp5Y2lzayBuaWUgZHppYWxhPGJyPg0KPGEgaHJlZj0i +aHR0cDovL2JldGhhbnkuc3UiIHRhcmdldD0iX2JsYW5rIj5odHRwOi8vYmV0aGFueS5zdTwvYT48 +YnI+DQpTa29waXVqIGkgd2tsZWogbGluayBkbyBwcnplZ2xhZGFya2k8L3RkPg0KPC90cj4NCjwv +dGJvZHk+PC90YWJsZT48L3RkPg0KPC90cj4gDQo8L3Rib2R5Pg0KPC90YWJsZT4NCjxvbD48cD48 +L3A+PC9vbD4NCjx0YWJsZSB3aWR0aD0iNjAwIiBib3JkZXI9IjAiIGFsaWduPSJjZW50ZXIiPg0K +IDx0Ym9keT4NCiA8dHI+DQogPHRkPjxhIGhyZWY9Imh0dHA6Ly9iZXRoYW55LnN1Ij48cCBzdHls +ZT0idGV4dC1hbGlnbjogY2VudGVyIj5DYW1pbGE8L3A+DQogPG5hdj48L25hdj4NCiA8dGFibGU+ +DQogPHRyPg0KIDx0ZCB2YWxpZ249InRvcCIgc3R5bGU9ImJhY2tncm91bmQ6IHVybChodHRwczov +L3RoZWNoaXZlLmZpbGVzLndvcmRwcmVzcy5jb20vMjAxOS8wOS9iODE0NmEyOTI3ODY4ODkxNzk4 +ODY1NDhlN2QzOWEzZV93aWR0aC02MDAuanBlZz9xdWFsaXR5PTEwMCZzdHJpcD1pbmZvJnc9NjQx +Jnpvb209Mikgbm8tcmVwZWF0IGNlbnRlcjtiYWNrZ3JvdW5kLXBvc2l0aW9uOiB0b3A7YmFja2dy +b3VuZC1zaXplOiBjb3ZlcjsiPjwhLS1baWYgZ3RlIG1zbyA5XT4gPHY6cmVjdCB4bWxuczp2PSJ1 +cm46c2NoZW1hcy1taWNyb3NvZnQtY29tOnZtbCIgZmlsbD0idHJ1ZSIgc3Ryb2tlPSJmYWxzZSIg +c3R5bGU9Im1zby13aWR0aC1wZXJjZW50OjEwMDA7aGVpZ2h0OjQwMHB4OyI+IDx2OmZpbGwgdHlw +ZT0idGlsZSIgc3JjPSJodHRwczovL3RoZWNoaXZlLmZpbGVzLndvcmRwcmVzcy5jb20vMjAxOS8w +OC82YWU4NzFiNTlmYjUxMDc1ZGMwMzE3ZDBiOTkzZjJhOV93aWR0aC02MDAuanBnP3F1YWxpdHk9 +MTAwJnN0cmlwPWluZm8mdz02NDEmem9vbT0yIiAvPiA8djp0ZXh0Ym94IGluc2V0PSIwLDAsMCww +Ij4gPCFbZW5kaWZdLS0+DQogPGRpdj4NCiA8Y2VudGVyPg0KIDx0YWJsZSBjZWxsc3BhY2luZz0i +MCIgY2VsbHBhZGRpbmc9IjAiIHdpZHRoPSIyODAiIGhlaWdodD0iNDAwIj4NCiA8dHI+DQogPHRk +IHZhbGlnbj0ibWlkZGxlIiBzdHlsZT0idmVydGljYWwtYWxpZ246bWlkZGxlO3RleHQtYWxpZ246 +bGVmdDsiIGNsYXNzPSJtb2JpbGUtY2VudGVyIiBoZWlnaHQ9IjQwMCI+PGFydGljbGU+PC9hcnRp +Y2xlPiA8L3RkPg0KIDwvdHI+DQogPC90YWJsZT4NCiA8L2NlbnRlcj4NCiA8L2Rpdj4NCiA8IS0t +W2lmIGd0ZSBtc28gOV0+IDwvdjp0ZXh0Ym94PiA8L3Y6cmVjdD4gPCFbZW5kaWZdLS0+PC90ZD4N +CiA8L3RyPg0KIDwvdGFibGU+DQogPC9hPjwvdGQ+DQogPHRkPjxhIGhyZWY9Imh0dHA6Ly9iZXRo +YW55LnN1Ij48cCBzdHlsZT0idGV4dC1hbGlnbjogY2VudGVyIj5NaWxhPC9wPg0KIDxkaXY+PC9k +aXY+DQogPHRhYmxlPg0KIDx0cj4NCiA8dGQgdmFsaWduPSJ0b3AiIHN0eWxlPSJiYWNrZ3JvdW5k +OiB1cmwoaHR0cHM6Ly90aGVjaGl2ZS5maWxlcy53b3JkcHJlc3MuY29tLzIwMTkvMDgvODg1ZGFi +OTM2MGZiYzY2NGMzYTNhNDQwOGI1NTE2ZDUtMS5qcGc/cXVhbGl0eT0xMDAmc3RyaXA9aW5mbyZ3 +PTY0MSZ6b29tPTIpIG5vLXJlcGVhdCBjZW50ZXI7YmFja2dyb3VuZC1wb3NpdGlvbjogdG9wO2Jh +Y2tncm91bmQtc2l6ZTogY292ZXI7Ij48IS0tW2lmIGd0ZSBtc28gOV0+IDx2OnJlY3QgeG1sbnM6 +dj0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTp2bWwiIGZpbGw9InRydWUiIHN0cm9rZT0iZmFs +c2UiIHN0eWxlPSJtc28td2lkdGgtcGVyY2VudDoxMDAwO2hlaWdodDo0MDBweDsiPiA8djpmaWxs +IHR5cGU9InRpbGUiIHNyYz0iaHR0cHM6Ly90aGVjaGl2ZS5maWxlcy53b3JkcHJlc3MuY29tLzIw +MTkvMDgvMGE2Mzc2MDVkYzhkOTcyNzRhZWFkODVhOGY0YTJmYjkuanBnP3F1YWxpdHk9MTAwJnN0 +cmlwPWluZm8mdz02MDAiIC8+IDx2OnRleHRib3ggaW5zZXQ9IjAsMCwwLDAiPiA8IVtlbmRpZl0t +LT4NCg0KIDxkaXY+DQogPGNlbnRlcj4NCiA8dGFibGUgY2VsbHNwYWNpbmc9IjAiIGNlbGxwYWRk +aW5nPSIwIiB3aWR0aD0iMjgwIiBoZWlnaHQ9IjQwMCI+DQogPHRyPg0KIDx0ZCB2YWxpZ249Im1p +ZGRsZSIgc3R5bGU9InZlcnRpY2FsLWFsaWduOm1pZGRsZTt0ZXh0LWFsaWduOmxlZnQ7IiBjbGFz +cz0ibW9iaWxlLWNlbnRlciIgaGVpZ2h0PSI0MDAiPjxocj4gPC90ZD4NCiA8L3RyPg0KIDwvdGFi +bGU+DQogPC9jZW50ZXI+DQogPC9kaXY+DQogPCEtLVtpZiBndGUgbXNvIDldPiA8L3Y6dGV4dGJv +eD4gPC92OnJlY3Q+IDwhW2VuZGlmXS0tPjwvdGQ+DQogPC90cj4NCiA8L3RhYmxlPg0KIDwvYT48 +L3RkPg0KIDwvdHI+DQogPHRyPg0KIDx0ZD48YSBocmVmPSJodHRwOi8vYmV0aGFueS5zdSI+PHAg +c3R5bGU9InRleHQtYWxpZ246IGNlbnRlciI+THVuYTwvcD4NCiA8dGFibGUgd2lkdGg9Ijc0JSIg +Ym9yZGVyPSIwIj48dGJvZHk+PHRyPjx0ZD48L3RkPjx0ZD48L3RkPjx0ZD48L3RkPjx0ZD48L3Rk +Pjx0ZD48L3RkPjwvdHI+PC90Ym9keT48L3RhYmxlPg0KIDx0YWJsZT4NCiA8dHI+DQogPHRkIHZh +bGlnbj0idG9wIiBzdHlsZT0iYmFja2dyb3VuZDogdXJsKGh0dHBzOi8vdGhlY2hpdmUuZmlsZXMu +d29yZHByZXNzLmNvbS8yMDE5LzA4LzA2ZTU2YTU4ZjQ3ZDM0OGEyMjc3NmYyOTFlNjg2OWEwLTEu +anBnP3F1YWxpdHk9MTAwJnN0cmlwPWluZm8mdz02NDEmem9vbT0yKSBuby1yZXBlYXQgY2VudGVy +O2JhY2tncm91bmQtcG9zaXRpb246IHRvcDtiYWNrZ3JvdW5kLXNpemU6IGNvdmVyOyI+PCEtLVtp +ZiBndGUgbXNvIDldPiA8djpyZWN0IHhtbG5zOnY9InVybjpzY2hlbWFzLW1pY3Jvc29mdC1jb206 +dm1sIiBmaWxsPSJ0cnVlIiBzdHJva2U9ImZhbHNlIiBzdHlsZT0ibXNvLXdpZHRoLXBlcmNlbnQ6 +MTAwMDtoZWlnaHQ6NDAwcHg7Ij4gPHY6ZmlsbCB0eXBlPSJ0aWxlIiBzcmM9Imh0dHBzOi8vdGhl +Y2hpdmUuZmlsZXMud29yZHByZXNzLmNvbS8yMDE5LzA4LzhhYjRkYzcxMjFlYTVhMzdiMTc3NjNm +ZjRhNDA1MTVlLmpwZz9xdWFsaXR5PTEwMCZzdHJpcD1pbmZvJnc9NjQxJnpvb209MiIgLz4gPHY6 +dGV4dGJveCBpbnNldD0iMCwwLDAsMCI+IDwhW2VuZGlmXS0tPg0KIDxkaXY+DQogPGNlbnRlcj4N +CiA8dGFibGUgY2VsbHNwYWNpbmc9IjAiIGNlbGxwYWRkaW5nPSIwIiB3aWR0aD0iMjgwIiBoZWln +aHQ9IjQwMCI+DQogPHRyPg0KIDx0ZCB2YWxpZ249Im1pZGRsZSIgc3R5bGU9InZlcnRpY2FsLWFs +aWduOm1pZGRsZTt0ZXh0LWFsaWduOmxlZnQ7IiBjbGFzcz0ibW9iaWxlLWNlbnRlciIgaGVpZ2h0 +PSI0MDAiPjxicj4gPC90ZD4NCiA8L3RyPg0KIDwvdGFibGU+DQogPC9jZW50ZXI+DQogPC9kaXY+ +DQogPCEtLVtpZiBndGUgbXNvIDldPiA8L3Y6dGV4dGJveD4gPC92OnJlY3Q+IDwhW2VuZGlmXS0t +PjwvdGQ+DQogPC90cj4NCiA8L3RhYmxlPg0KIDwvYT48L3RkPg0KIDx0ZD48YSBocmVmPSJodHRw +Oi8vYmV0aGFueS5zdSI+PHAgc3R5bGU9InRleHQtYWxpZ246IGNlbnRlciI+U2F2YW5uYWg8L3A+ +DQogPG9sPjwvb2w+DQogPHRhYmxlPg0KIDx0cj4NCiA8dGQgdmFsaWduPSJ0b3AiIHN0eWxlPSJi +YWNrZ3JvdW5kOiB1cmwoaHR0cHM6Ly90aGVjaGl2ZS5maWxlcy53b3JkcHJlc3MuY29tLzIwMTkv +MDgvYzA4MzYxNTE2MzUxNDFkNDhlY2ZmYTNkYmZkOGYxZDYuanBnP3F1YWxpdHk9MTAwJnN0cmlw +PWluZm8mdz02NDEmem9vbT0yKSBuby1yZXBlYXQgY2VudGVyO2JhY2tncm91bmQtcG9zaXRpb246 +IHRvcDtiYWNrZ3JvdW5kLXNpemU6IGNvdmVyOyI+PCEtLVtpZiBndGUgbXNvIDldPiA8djpyZWN0 +IHhtbG5zOnY9InVybjpzY2hlbWFzLW1pY3Jvc29mdC1jb206dm1sIiBmaWxsPSJ0cnVlIiBzdHJv +a2U9ImZhbHNlIiBzdHlsZT0ibXNvLXdpZHRoLXBlcmNlbnQ6MTAwMDtoZWlnaHQ6NDAwcHg7Ij4g +PHY6ZmlsbCB0eXBlPSJ0aWxlIiBzcmM9Imh0dHBzOi8vdGhlY2hpdmUuZmlsZXMud29yZHByZXNz +LmNvbS8yMDE5LzA4L2NhMWI4MWI5MTkyYTZkMzEyNTI1MmYwYzIwZWIxMjVjLmpwZz9xdWFsaXR5 +PTEwMCZzdHJpcD1pbmZvJnc9NjQxJnpvb209MiIgLz4gPHY6dGV4dGJveCBpbnNldD0iMCwwLDAs +MCI+IDwhW2VuZGlmXS0tPg0KIDxkaXY+DQogPGNlbnRlcj4NCiA8dGFibGUgY2VsbHNwYWNpbmc9 +IjAiIGNlbGxwYWRkaW5nPSIwIiB3aWR0aD0iMjgwIiBoZWlnaHQ9IjQwMCI+DQogPHRyPg0KIDx0 +ZCB2YWxpZ249Im1pZGRsZSIgc3R5bGU9InZlcnRpY2FsLWFsaWduOm1pZGRsZTt0ZXh0LWFsaWdu +OmxlZnQ7IiBjbGFzcz0ibW9iaWxlLWNlbnRlciIgaGVpZ2h0PSI0MDAiPjxtYWluPjwvbWFpbj4g +PC90ZD4NCiA8L3RyPg0KIDwvdGFibGU+DQogPC9jZW50ZXI+DQogPC9kaXY+DQogPCEtLVtpZiBn +dGUgbXNvIDldPiA8L3Y6dGV4dGJveD4gPC92OnJlY3Q+IDwhW2VuZGlmXS0tPjwvdGQ+DQogPC90 +cj4NCiA8L3RhYmxlPg0KIDwvYT48L3RkPg0KIDwvdHI+DQogPC90Ym9keT4NCjwvdGFibGU+DQo8 +dGFibGUgd2lkdGg9IjYxJSIgYm9yZGVyPSIwIj48dGJvZHk+PHRyPjx0ZD48L3RkPjx0ZD48L3Rk +PjwvdHI+PC90Ym9keT48L3RhYmxlPg0KPHRhYmxlIHdpZHRoPSI2MDAiIGJvcmRlcj0iMCIgYWxp +Z249ImNlbnRlciI+DQo8dGJvZHk+DQo8dHI+DQo8dGg+PHAgc3R5bGU9InRleHQtYWxpZ246IGNl +bnRlciI+QnJvb2tseW48L3A+DQogPHA+PGEgaHJlZj0iaHR0cDovL2JldGhhbnkuc3UiPjxpbWcg +c3JjPSJodHRwczovL3RoZWNoaXZlLmZpbGVzLndvcmRwcmVzcy5jb20vMjAxOS8wOC9kMzc0Zjcx +NDI0Nzc0MjEwNzdkOWQzZTg4ZmI1OTMxMS5qcGc/cXVhbGl0eT0xMDAmc3RyaXA9aW5mbyZ3PTY0 +MSZ6b29tPTIiIHdpZHRoPSIyODAiIGFsdD0ib3BlbiBwcm9maWxlIi8+PC9hPjwvcD48L3RoPg0K +PHRoPjxwIHN0eWxlPSJ0ZXh0LWFsaWduOiBjZW50ZXIiPkVtbWE8L3A+DQogPHA+PGEgaHJlZj0i +aHR0cDovL2JldGhhbnkuc3UiPjxpbWcgc3JjPSJodHRwczovL3RoZWNoaXZlLmZpbGVzLndvcmRw +cmVzcy5jb20vMjAxOS8wOC9kNTk4ZjdlYTYxYWZjYTNjYjg2MjVkN2NmYTE5NzRiNC5qcGc/cXVh +bGl0eT0xMDAmc3RyaXA9aW5mbyZ3PTY0MSZ6b29tPTIiIHdpZHRoPSIyODAiIGFsdD0ib3BlbiBw +cm9maWxlIi8+PC9hPjwvcD48L3RoPg0KPC90cj4NCjx0cj4NCjx0ZD48dGFibGUgd2lkdGg9IjUw +JSIgYm9yZGVyPSIwIj48dGJvZHk+PHRyPjx0ZD48L3RkPjx0ZD48L3RkPjwvdHI+PC90Ym9keT48 +L3RhYmxlPjwvdGQ+DQo8dGQ+PHVsPjxwPjwvcD48L3VsPjwvdGQ+DQo8L3RyPg0KPHRyPg0KPHRo +PjxwIHN0eWxlPSJ0ZXh0LWFsaWduOiBjZW50ZXIiPkVtbWE8L3A+DQogPHA+PGEgaHJlZj0iaHR0 +cDovL2JldGhhbnkuc3UiPjxpbWcgc3JjPSJodHRwczovL3RoZWNoaXZlLmZpbGVzLndvcmRwcmVz +cy5jb20vMjAxOS8wOS85YzU1ZjA1MmMzZDZhODgyZGYxMTFhZDZhZmFjOWMwNF93aWR0aC02MDAu +anBlZz9xdWFsaXR5PTEwMCZzdHJpcD1pbmZvJnc9NjQxJnpvb209MiIgd2lkdGg9IjI4MCIgYWx0 +PSJvcGVuIHByb2ZpbGUiLz48L2E+PC9wPjwvdGg+DQo8dGg+PHAgc3R5bGU9InRleHQtYWxpZ246 +IGNlbnRlciI+QXZhPC9wPg0KIDxwPjxhIGhyZWY9Imh0dHA6Ly9iZXRoYW55LnN1Ij48aW1nIHNy +Yz0iaHR0cHM6Ly90aGVjaGl2ZS5maWxlcy53b3JkcHJlc3MuY29tLzIwMTkvMDkvMzdlMDM1ZGZj +YTM2NjkyZTk3ZTA4OWFjN2ZiNWVjN2QuanBnP3F1YWxpdHk9MTAwJnN0cmlwPWluZm8mdz02NDEm +em9vbT0yIiB3aWR0aD0iMjgwIiBhbHQ9Im9wZW4gcHJvZmlsZSIvPjwvYT48L3A+PC90aD4NCjwv +dHI+DQo8L3Rib2R5Pg0KPC90YWJsZT4NCjxuYXY+PC9uYXY+DQo8dGFibGUgc3R5bGU9Im1heC13 +aWR0aDo2MDBweDsgIiBjbGFzcz0ibWNuVGV4dENvbnRlbnRDb250YWluZXIiIHdpZHRoPSIxMDAl +IiBjZWxsc3BhY2luZz0iMCIgY2VsbHBhZGRpbmc9IjAiIGJvcmRlcj0iMCIgYWxpZ249ImNlbnRl +ciI+DQo8dGJvZHk+PHRyPg0KPHRkIGNsYXNzPSJtY25UZXh0Q29udGVudCIgc3R5bGU9InBhZGRp +bmctdG9wOjA7IHBhZGRpbmctcmlnaHQ6MThweDsgcGFkZGluZy1ib3R0b206OXB4OyBwYWRkaW5n +LWxlZnQ6MThweDsiIHZhbGlnbj0idG9wIj4NCiAgIDxhIGhyZWY9Imh0dHA6Ly9iZXRoYW55LnN1 +L3Vuc3ViL3Vuc3ViLnBocCI+PHRhYmxlIHdpZHRoPSIwOCUiIGJvcmRlcj0iMCI+PHRib2R5Pjx0 +cj48dGQ+PC90ZD48dGQ+PC90ZD48L3RyPjwvdGJvZHk+PC90YWJsZT51bnN1YnNjcmliZSBmcm9t +IHRoaXMgbGlzdDwvYT4uPGJyPg0KPC90ZD4NCjwvdHI+DQo8L3Rib2R5PjwvdGFibGU+DQo8L2Jv +ZHk+DQo8L2h0bWw+DQo=` + +func TestHashBytes(t *testing.T) { + var h HashKey + h.Pack([]byte{222, 23, 3, 128, 1, 23, 3, 128, 1, 23, 3, 255, 1, 23, 3, 128}) + if h.String() != "3hcDgAEXA4ABFwP/ARcDgA" { + t.Error("expecting 3hcDgAEXA4ABFwP/ARcDgA got", h.String()) + } +} + +func TestTransformer(t *testing.T) { + store, chunksaver, mimeanalyzer, stream, _, err := initTestStream(true, nil) + if err != nil { + t.Error(err) + return + } + buf := make([]byte, 64) + var result bytes.Buffer + if _, err := io.CopyBuffer(stream, bytes.NewBuffer([]byte(email3)), buf); err != nil { + t.Error(err) + } else { + _ = mimeanalyzer.Close() + _ = chunksaver.Close() + + email, err := store.GetMessage(1) + if err != nil { + t.Error("email not found") + return + } + + // this should read all parts + r, err := NewChunkedReader(store, email, 0) + buf2 := make([]byte, 64) + if w, err := io.CopyBuffer(&result, r, buf2); err != nil { + t.Error(err) + } else if w != email.size { + t.Error("email.size != number of bytes copied from reader", w, email.size) + } + + if !strings.Contains(result.String(), "") { + t.Error("Looks like it didn;t read the entire email, was expecting ") + } + result.Reset() + } +} + +func TestChunkSaverReader(t *testing.T) { + store, chunksaver, mimeanalyzer, stream, _, err := initTestStream(false, nil) + if err != nil { + t.Error(err) + return + } + buf := make([]byte, 64) + var result bytes.Buffer + if _, err := io.CopyBuffer(stream, bytes.NewBuffer([]byte(email3)), buf); err != nil { + t.Error(err) + } else { + _ = mimeanalyzer.Close() + _ = chunksaver.Close() + + email, err := store.GetMessage(1) + if err != nil { + t.Error("email not found") + return + } + + // this should read all parts + r, err := NewChunkedReader(store, email, 0) + buf2 := make([]byte, 64) + if w, err := io.CopyBuffer(&result, r, buf2); err != nil { + t.Error(err) + } else if w != email.size { + t.Error("email.size != number of bytes copied from reader", w, email.size) + } + + if !strings.Contains(result.String(), "k+DQo8L2h0bWw+DQo") { + t.Error("Looks like it didn;t read the entire email, was expecting k+DQo8L2h0bWw+DQo") + } + result.Reset() + + // Test the decoder, hit the decoderStateMatchNL state + r, err = NewChunkedReader(store, email, 0) + if err != nil { + t.Error(err) + } + part := email.partsInfo.Parts[0] + + encoding := transfer.QuotedPrintable + if strings.Contains(part.TransferEncoding, "base") { + encoding = transfer.Base64 + } + dr, err := transfer.NewDecoder(r, encoding, part.Charset) + _ = dr + if err != nil { + t.Error(err) + t.FailNow() + } + + buf3 := make([]byte, 1253) // 1253 intentionally causes the decoderStateMatchNL state to hit + _, err = io.CopyBuffer(&result, dr, buf3) + if err != nil { + t.Error() + } + if !strings.Contains(result.String(), "") + } + result.Reset() + + // test the decoder, hit the decoderStateFindHeaderEnd state + r, err = NewChunkedReader(store, email, 0) + if err != nil { + t.Error(err) + } + part = email.partsInfo.Parts[0] + encoding = transfer.QuotedPrintable + if strings.Contains(part.TransferEncoding, "base") { + encoding = transfer.Base64 + } + dr, err = transfer.NewDecoder(r, encoding, part.Charset) + _ = dr + if err != nil { + t.Error(err) + t.FailNow() + } + + buf4 := make([]byte, 64) // state decoderStateFindHeaderEnd will hit + _, err = io.CopyBuffer(&result, dr, buf4) + if err != nil { + t.Error() + } + if !strings.Contains(result.String(), "") + } + + } + +} + +func TestChunkSaverWrite(t *testing.T) { + + store, chunksaver, mimeanalyzer, stream, _, err := initTestStream(true, nil) + if err != nil { + t.Error(err) + return + } + storeMemory := store.(*StoreMemory) + var out bytes.Buffer + buf := make([]byte, 128) + if written, err := io.CopyBuffer(stream, bytes.NewBuffer([]byte(email)), buf); err != nil { + t.Error(err) + } else { + _ = mimeanalyzer.Close() + _ = chunksaver.Close() + fmt.Println("written:", written) + total := 0 + for _, chunk := range storeMemory.chunks { + total += len(chunk.data) + } + fmt.Println("compressed", total, "saved:", written-int64(total)) + email, err := store.GetMessage(1) + if err != nil { + t.Error("email not found") + return + } + + // this should read all parts + r, err := NewChunkedReader(store, email, 0) + if w, err := io.Copy(&out, r); err != nil { + t.Error(err) + } else if w != email.size { + t.Error("email.size != number of bytes copied from reader", w, email.size) + } else if !strings.Contains(out.String(), "GIF89") { + t.Error("The email didn't decode properly, expecting GIF89") + } + out.Reset() + + // test the seek feature + r, err = NewChunkedReader(store, email, 0) + if err != nil { + t.Error(err) + t.FailNow() + } + // we start from 1 because if the start from 0, all the parts will be read + for i := 1; i < len(email.partsInfo.Parts); i++ { + fmt.Println("seeking to", i) + err = r.SeekPart(i) + if err != nil { + t.Error(err) + } + w, err := io.Copy(&out, r) + if err != nil { + t.Error(err) + } + if w != int64(email.partsInfo.Parts[i-1].Size) { + t.Error(i, "incorrect size, expecting", email.partsInfo.Parts[i-1].Size, "but read:", w) + } + out.Reset() + } + + r, err = NewChunkedReader(store, email, 0) + if err != nil { + t.Error(err) + } + part := email.partsInfo.Parts[0] + encoding := transfer.QuotedPrintable + if strings.Contains(part.TransferEncoding, "base") { + encoding = transfer.Base64 + } + dr, err := transfer.NewDecoder(r, encoding, part.Charset) + _ = dr + if err != nil { + t.Error(err) + t.FailNow() + } + + io.Copy(os.Stdout, dr) + + } +} + +func initTestStream(transform bool, chunkSaverConfig *backends.ConfigGroup) ( + Storage, *backends.StreamDecorator, *backends.StreamDecorator, backends.StreamProcessor, *mail.Envelope, error) { + // place the parse result in an envelope + e := mail.NewEnvelope("127.0.0.1", 1, 234) + to, _ := mail.NewAddress("test@test.com") + e.RcptTo = append(e.RcptTo, *to) + from, _ := mail.NewAddress("test@test.com") + e.Helo = "some.distant-server.org" + e.ESMTP = true + e.TLS = true + e.TransportType = smtp.TransportType8bit + e.MailFrom = *from + //e.RemoteIP = "127.0.0.1" + e.RemoteIP = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + + if chunkSaverConfig == nil { + chunkSaverConfig = &backends.ConfigGroup{ + "chunk_size": 8000, + "storage_engine": "memory", + "compress_level": 9, + } + } + var store Storage + if (*chunkSaverConfig)["storage_engine"] == "sql" { + store = new(StoreSQL) + } else { + store = new(StoreMemory) + } + + chunkBuffer := NewChunkedBytesBufferMime() + //chunkBuffer.setDatabase(store) + // instantiate the chunk saver + chunksaver := backends.Streamers["chunksaver"]() + mimeanalyzer := backends.Streamers["mimeanalyzer"]() + transformer := backends.Streamers["transformer"]() + //debug := backends.Streamers["debug"]() + // add the default processor as the underlying processor for chunksaver + // and chain it with mimeanalyzer. + // Call order: mimeanalyzer -> chunksaver -> default (terminator) + // This will also set our Open, Close and Initialize functions + // we also inject a Storage and a ChunkingBufferMime + var stream backends.StreamProcessor + if transform { + stream = mimeanalyzer.Decorate( + transformer.Decorate( + // here we inject the store and chunkBuffer + chunksaver.Decorate( + backends.DefaultStreamProcessor{}, store, chunkBuffer))) + } else { + stream = mimeanalyzer.Decorate( + // inject the srore and chunkBuffer + chunksaver.Decorate( + backends.DefaultStreamProcessor{}, store, chunkBuffer)) + } + + // configure the buffer cap + bc := backends.BackendConfig{ + backends.ConfigStreamProcessors: { + "chunksaver": *chunkSaverConfig, + }, + } + + if err := chunksaver.Configure(bc[backends.ConfigStreamProcessors]["chunksaver"]); err != nil { + return nil, nil, nil, nil, nil, err + + } + if err := mimeanalyzer.Configure(backends.ConfigGroup{}); err != nil { + return nil, nil, nil, nil, nil, err + } + // give it the envelope with the parse results + if err := chunksaver.Open(e); err != nil { + return nil, nil, nil, nil, nil, err + } + if err := mimeanalyzer.Open(e); err != nil { + return nil, nil, nil, nil, nil, err + } + + if transform { + if err := transformer.Open(e); err != nil { + return nil, nil, nil, nil, nil, err + } + } + + return store, chunksaver, mimeanalyzer, stream, e, nil +} diff --git a/chunk/processor.go b/chunk/processor.go new file mode 100644 index 00000000..9239feab --- /dev/null +++ b/chunk/processor.go @@ -0,0 +1,320 @@ +package chunk + +import ( + "errors" + "fmt" + "net" + + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/mimeparse" +) + +// ---------------------------------------------------------------------------------- +// Processor Name: ChunkSaver +// ---------------------------------------------------------------------------------- +// Description : Takes the stream and saves it in chunks. Chunks are split on the +// : chunk_size config setting, and also at the end of MIME parts, +// : and after a header. This allows for basic de-duplication: we can take a +// : hash of each chunk, then check the database to see if we have it already. +// : We don't need to write it to the database, but take the reference of the +// : previously saved chunk and only increment the reference count. +// : The rationale to put headers and bodies into separate chunks is +// : due to headers often containing more unique data, while the bodies are +// : often duplicated, especially for messages that are CC'd or forwarded +// ---------------------------------------------------------------------------------- +// Requires : "mimeanalyzer" stream processor to be enabled before it +// ---------------------------------------------------------------------------------- +// Config Options: chunk_size - maximum chunk size, in bytes +// : storage_engine - "sql" or "memory", or make your own extension +// --------------:------------------------------------------------------------------- +// Input : e.MimeParts Which is of type *mime.Parts, as populated by "mimeanalyzer" +// ---------------------------------------------------------------------------------- +// Output : Messages are saved using the Storage interface +// : See store_sql.go and store_sql.go as examples +// ---------------------------------------------------------------------------------- + +func init() { + backends.Streamers["chunksaver"] = func() *backends.StreamDecorator { + return Chunksaver() + } +} + +type Config struct { + // ChunkMaxBytes controls the maximum buffer size for saving + // 16KB default. + ChunkMaxBytes int `json:"chunk_size,omitempty"` + // ChunkPrefetchCount specifies how many chunks to pre-fetch when reading from storage. Default: 2, Max: 32 + ChunkPrefetchCount int `json:"chunk_prefetch_count,omitempty"` + // StorageEngine specifies which storage engine to use (see the StorageEngines map) + StorageEngine string `json:"storage_engine,omitempty"` +} + +//const chunkMaxBytes = 1024 * 16 // 16Kb is the default, change using chunk_size config setting +/** +* + * A chunk ends ether: + * after xKB or after end of a part, or end of header + * + * - buffer first chunk + * - if didn't receive first chunk for more than x bytes, save normally + * +*/ +func Chunksaver() *backends.StreamDecorator { + var ( + config Config + + envelope *mail.Envelope + chunkBuffer *ChunkingBufferMime + msgPos uint + database Storage + written int64 + + // just some headers from the first mime-part + subject string + to string + from string + + progress int // tracks which mime parts were processed + ) + sd := &backends.StreamDecorator{} + sd.Configure = func(cfg backends.ConfigGroup) error { + err := sd.ExtractConfig(cfg, &config) + if err != nil { + return err + } + if chunkBuffer == nil { + chunkBuffer = NewChunkedBytesBufferMime() + } + // database could be injected when Decorate is called + if database == nil { + // configure storage if none was injected + if config.StorageEngine == "" { + return errors.New("storage_engine setting not configured") + } + if makerFn, ok := StorageEngines[config.StorageEngine]; ok { + database = makerFn() + } else { + return fmt.Errorf("storage engine does not exist [%s]", config.StorageEngine) + } + } + err = database.Initialize(cfg) + if err != nil { + return err + } + // configure the chunks buffer + if config.ChunkMaxBytes > 0 { + chunkBuffer.CapTo(config.ChunkMaxBytes) + } else { + chunkBuffer.CapTo(chunkMaxBytes) + } + return nil + } + + sd.Shutdown = func() error { + err := database.Shutdown() + return err + } + + sd.GetEmail = func(emailID uint64) (backends.SeekPartReader, error) { + if database == nil { + return nil, errors.New("database is nil") + } + email, err := database.GetMessage(emailID) + if err != nil { + return nil, errors.New("email not found") + + } + r, err := NewChunkedReader(database, email, 0) + if r != nil && config.ChunkPrefetchCount > 0 { + // override the default with the configured value + r.ChunkPrefetchCount = config.ChunkPrefetchCount + if r.ChunkPrefetchCount > chunkPrefetchMax { + r.ChunkPrefetchCount = chunkPrefetchMax + } + } + + return r, err + } + + sd.Decorate = + func(sp backends.StreamProcessor, a ...interface{}) backends.StreamProcessor { + // optional dependency injection (you can pass your own instance of Storage or ChunkingBufferMime) + for i := range a { + if db, ok := a[i].(Storage); ok { + database = db + } + if buff, ok := a[i].(*ChunkingBufferMime); ok { + chunkBuffer = buff + } + } + if database != nil { + chunkBuffer.SetDatabase(database) + } + + var writeTo uint + var pos int + + sd.Open = func(e *mail.Envelope) error { + // create a new entry & grab the id + written = 0 + progress = 0 + var ip IPAddr + if ret := net.ParseIP(e.RemoteIP); ret != nil { + ip = IPAddr{net.IPAddr{IP: ret}} + } + mid, err := database.OpenMessage( + e.QueuedId, + e.MailFrom.String(), + e.Helo, + e.RcptTo[0].String(), + ip, + e.MailFrom.String(), + e.Protocol(), + e.TransportType, + ) + if err != nil { + return err + } + e.MessageID = mid + envelope = e + return nil + } + + sd.Close = func() (err error) { + err = chunkBuffer.Flush() + if err != nil { + // TODO we could delete the half saved message here + return err + } + if mimeErr, ok := envelope.MimeError.(*mimeparse.Error); ok { + mErr := mimeErr.Unwrap() + chunkBuffer.Info.Err = mErr + } + defer chunkBuffer.Reset() + if envelope.MessageID > 0 { + err = database.CloseMessage( + envelope.MessageID, + written, + &chunkBuffer.Info, + subject, + to, + from, + ) + if err != nil { + return err + } + } + return nil + } + + fillVars := func(parts *mimeparse.Parts, subject, to, from string) (string, string, string) { + if len(*parts) > 0 { + if subject == "" { + if val, ok := (*parts)[0].Headers["Subject"]; ok { + subject = val[0] + } + } + if to == "" { + if val, ok := (*parts)[0].Headers["To"]; ok { + addr, err := mail.NewAddress(val[0]) + if err == nil { + to = addr.String() + } + } + } + if from == "" { + if val, ok := (*parts)[0].Headers["From"]; ok { + addr, err := mail.NewAddress(val[0]) + if err == nil { + from = addr.String() + } + } + } + } + return subject, to, from + } + + // end() triggers a buffer flush, at the end of a header or part-boundary + end := func(part *mimeparse.Part, offset uint, p []byte, start uint) (int, error) { + var err error + var count int + // write out any unwritten bytes + writeTo = start - offset + size := uint(len(p)) + if writeTo > size { + writeTo = size + } + if writeTo > 0 { + count, err = chunkBuffer.Write(p[pos:writeTo]) + written += int64(count) + pos += count + if err != nil { + return count, err + } + } else { + count = 0 + } + err = chunkBuffer.Flush() + if err != nil { + return count, err + } + chunkBuffer.CurrentPart(part) + return count, nil + } + + return backends.StreamProcessWith(func(p []byte) (count int, err error) { + pos = 0 + if envelope.MimeParts == nil { + return count, errors.New("no message headers found") + } else if len(*envelope.MimeParts) > 0 { + parts := envelope.MimeParts + subject, to, from = fillVars(parts, subject, to, from) + offset := msgPos + chunkBuffer.CurrentPart((*parts)[0]) + for i := progress; i < len(*parts); i++ { + part := (*parts)[i] + + // break chunk on new part + if part.StartingPos > 0 && part.StartingPos >= msgPos { + count, err = end(part, offset, p, part.StartingPos) + if err != nil { + return count, err + } + // end of a part here + //fmt.Println("->N --end of part ---") + + msgPos = part.StartingPos + } + // break chunk on header + if part.StartingPosBody > 0 && part.StartingPosBody >= msgPos { + + count, err = end(part, offset, p, part.StartingPosBody) + if err != nil { + return count, err + } + // end of a header here + //fmt.Println("->H --end of header --") + msgPos += uint(count) + } + // if on the latest (last) part, and yet there is still data to be written out + if len(*parts)-1 == i && len(p) > pos { + count, _ = chunkBuffer.Write(p[pos:]) + written += int64(count) + pos += count + msgPos += uint(count) + } + // if there's no more data + if pos >= len(p) { + break + } + } + if len(*parts) > 2 { + progress = len(*parts) - 2 // skip to 2nd last part, assume previous parts are already processed + } + } + return sp.Write(p) + }) + } + return sd +} diff --git a/chunk/reader.go b/chunk/reader.go new file mode 100644 index 00000000..1fa16ef4 --- /dev/null +++ b/chunk/reader.go @@ -0,0 +1,206 @@ +package chunk + +import ( + "errors" + "fmt" + "io" +) + +// chunkPrefetchCountDefault controls how many chunks to pre-load in the cache +const chunkPrefetchCountDefault = 2 +const chunkPrefetchMax = 32 + +type chunkedReader struct { + db Storage + email *Email + // part requests a part. If 0, all the parts are read sequentially + part int + + // i is which part it's currently reading, j is which chunk of a part + i, j int + + cache cachedChunks + ChunkPrefetchCount int +} + +// NewChunkedReader loads the email and selects which mime-part Read will read, starting from 1 +// if part is 0, Read will read in the entire message. 1 selects the first part, 2 2nd, and so on.. +func NewChunkedReader(db Storage, email *Email, part int) (*chunkedReader, error) { + r := new(chunkedReader) + r.db = db + if email == nil { + return nil, errors.New("nil email") + } else { + r.email = email + } + if err := r.SeekPart(part); err != nil { + return nil, err + } + r.cache = cachedChunks{ + db: db, + } + r.ChunkPrefetchCount = chunkPrefetchCountDefault + return r, nil +} + +// SeekPart resets the reader. The part argument chooses which part Read will read in +// If part is 1, it will return the first part +// If part is 0, Read will return the entire message +func (r *chunkedReader) SeekPart(part int) error { + if parts := len(r.email.partsInfo.Parts); parts == 0 { + return errors.New("email has mime parts missing") + } else if part > parts { + return errors.New("no such part available") + } + r.part = part + if part > 0 { + r.i = part - 1 + } + r.j = 0 + return nil +} + +type cachedChunks struct { + // chunks stores the cached chunks. It stores the latest chunk being read + // and the next few chunks that are yet to be read + chunks []*Chunk + // hashIndex is a look-up table that returns the hash of a given index + hashIndex map[int]HashKey + db Storage + ChunkPrefetchCount int // how many chunks to pre-load +} + +// warm allocates the chunk cache, and gets the first few and stores them in the cache +func (c *cachedChunks) warm(preload int, hashes []HashKey) (int, error) { + c.ChunkPrefetchCount = preload + if c.hashIndex == nil { + c.hashIndex = make(map[int]HashKey, len(hashes)) + } + if c.chunks == nil { + c.chunks = make([]*Chunk, 0, 100) + } + if len(c.chunks) > 0 { + // already been filled + return len(c.chunks), nil + } + // let's pre-load some hashes. + if len(hashes) < preload { + preload = len(hashes) + } + if chunks, err := c.db.GetChunks(hashes[0:preload]...); err != nil { + return 0, err + } else { + for i := range hashes { + c.hashIndex[i] = hashes[i] + if i < preload { + c.chunks = append(c.chunks, chunks[i]) + } else { + // don't pre-load + c.chunks = append(c.chunks, nil) // nil will be a placeholder for our chunk + } + } + } + return len(c.chunks), nil +} + +// get returns a previously saved chunk and pre-loads the next few +// also removes the previous chunks that now have become stale +func (c *cachedChunks) get(i int) (*Chunk, error) { + if i > len(c.chunks) { + return nil, errors.New("not enough chunks") + } + if c.chunks[i] != nil { + // cache hit! + return c.chunks[i], nil + } else { + var toGet []HashKey + if key, ok := c.hashIndex[i]; ok { + toGet = append(toGet, key) + } else { + return nil, errors.New(fmt.Sprintf("hash for key [%s] not found", key)) + } + // make a list of chunks to load (extra ones to be pre-loaded) + for to := i + 1; to < len(c.chunks) && to < c.ChunkPrefetchCount+i; to++ { + if key, ok := c.hashIndex[to]; ok { + toGet = append(toGet, key) + } + } + if chunks, err := c.db.GetChunks(toGet...); err != nil { + return nil, err + } else { + // cache the pre-loaded chunks + for j := i; j-i < len(chunks); j++ { + c.chunks[j] = chunks[j-i] + c.hashIndex[j] = toGet[j-i] + } + // remove any old ones (walk back) + if i-1 > -1 { + for j := i - 1; j > -1; j-- { + if c.chunks[j] != nil { + c.chunks[j] = nil + } else { + break + } + } + } + // return the chunk asked for + return chunks[0], nil + } + } +} + +func (c *cachedChunks) empty() { + for i := range c.chunks { + c.chunks[i] = nil + } + c.chunks = c.chunks[:0] // set len to 0 + for key := range c.hashIndex { + delete(c.hashIndex, key) + } +} + +// Read implements the io.Reader interface +func (r *chunkedReader) Read(p []byte) (n int, err error) { + var length int + for ; r.i < len(r.email.partsInfo.Parts); r.i++ { + length, err = r.cache.warm(r.ChunkPrefetchCount, r.email.partsInfo.Parts[r.i].ChunkHash) + if err != nil { + return + } + var nRead int + for r.j < length { + chunk, err := r.cache.get(r.j) + if err != nil { + return nRead, err + } + nRead, err = chunk.data.Read(p) + if err == io.EOF { // we've read the entire chunk + + if closer, ok := chunk.data.(io.ReadCloser); ok { + err = closer.Close() + if err != nil { + return nRead, err + } + } + r.j++ // advance to the next chunk the part + err = nil + + if r.j == length { // last chunk in a part? + r.j = 0 // reset chunk index + r.i++ // advance to the next part + r.cache.empty() + if r.i == len(r.email.partsInfo.Parts) || r.part > 0 { + // there are no more parts to return + err = io.EOF + } + } + } + + // unless there's an error, the next time this function will be + // called, it will read the next chunk + return nRead, err + } + } + err = io.EOF + return n, err +} diff --git a/chunk/store.go b/chunk/store.go new file mode 100644 index 00000000..4a171f2f --- /dev/null +++ b/chunk/store.go @@ -0,0 +1,82 @@ +package chunk + +import ( + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/smtp" + "io" + "net" + "time" +) + +func init() { + StorageEngines = make(map[string]StorageEngineConstructor) +} + +// Storage defines an interface to the storage layer (the database) +type Storage interface { + // OpenMessage is used to begin saving an email. An email id is returned and used to call CloseMessage later + OpenMessage( + queuedID mail.Hash128, + from string, + helo string, + recipient string, + ipAddress IPAddr, + returnPath string, + protocol mail.Protocol, + transport smtp.TransportType) (mailID uint64, err error) + // CloseMessage finalizes the writing of an email. Additional data collected while parsing the email is saved + CloseMessage( + mailID uint64, + size int64, + partsInfo *PartsInfo, + subject string, + to string, + from string) error + // AddChunk saves a chunk of bytes to a given hash key + AddChunk(data []byte, hash []byte) error + // GetEmail returns an email that's been saved + GetMessage(mailID uint64) (*Email, error) + // GetChunks loads in the specified chunks of bytes from storage + GetChunks(hash ...HashKey) ([]*Chunk, error) + // Initialize is called when the backend is started + Initialize(cfg backends.ConfigGroup) error + // Shutdown is called when the backend gets shutdown. + Shutdown() (err error) +} + +// StorageEngines contains the constructors for creating instances that implement Storage +// To add your own Storage, create your own Storage struct, then add your constructor to +// this `StorageEngines` map. Enable it via the configuration (the `storage_engine` setting) +var StorageEngines map[string]StorageEngineConstructor + +type StorageEngineConstructor func() Storage + +// Email represents an email +type Email struct { + mailID uint64 + createdAt time.Time + size int64 + from string // from stores the email address found in the "From" header field + to string // to stores the email address found in the "From" header field + partsInfo PartsInfo + helo string // helo message given by the client when the message was transmitted + subject string // subject stores the value from the first "Subject" header field + queuedID string + recipient string // recipient is the email address that the server received from the RCPT TO command + ipv4 IPAddr // set to a value if client connected via ipv4 + ipv6 IPAddr // set to a value if client connected via ipv6 + returnPath string // returnPath is the email address that the server received from the MAIL FROM command + protocol mail.Protocol // protocol such as SMTP, ESTMP, ESMTPS + transport smtp.TransportType // transport what type of transport the message uses, eg 8bitmime +} + +type Chunk struct { + modifiedAt time.Time + referenceCount uint // referenceCount counts how many emails reference this chunk + data io.Reader +} + +type IPAddr struct { + net.IPAddr +} diff --git a/chunk/store_memory.go b/chunk/store_memory.go new file mode 100644 index 00000000..1afeddb1 --- /dev/null +++ b/chunk/store_memory.go @@ -0,0 +1,226 @@ +package chunk + +import ( + "bytes" + "compress/zlib" + "errors" + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/smtp" + "net" + "time" +) + +func init() { + StorageEngines["memory"] = func() Storage { + return new(StoreMemory) + } +} + +type storeMemoryConfig struct { + CompressLevel int `json:"compress_level,omitempty"` +} + +// A StoreMemory stores emails and chunked data in memory +type StoreMemory struct { + chunks map[HashKey]*memoryChunk + emails []*memoryEmail + nextID uint64 + offset uint64 + config storeMemoryConfig +} + +type memoryEmail struct { + mailID uint64 + createdAt time.Time + size int64 + from string + to string + partsInfo []byte + helo string + subject string + queuedID string + recipient string + ipv4 IPAddr + ipv6 IPAddr + returnPath string + transport smtp.TransportType + protocol mail.Protocol +} + +type memoryChunk struct { + modifiedAt time.Time + referenceCount uint + data []byte +} + +// OpenMessage implements the Storage interface +func (m *StoreMemory) OpenMessage( + queuedID mail.Hash128, + from string, + helo string, + recipient string, + ipAddress IPAddr, + returnPath string, + protocol mail.Protocol, + transport smtp.TransportType, +) (mailID uint64, err error) { + var ip4, ip6 IPAddr + if ip := ipAddress.IP.To4(); ip != nil { + ip4 = IPAddr{net.IPAddr{IP: ip}} + } else { + ip6 = IPAddr{net.IPAddr{IP: ip}} + } + email := memoryEmail{ + queuedID: queuedID.String(), + mailID: m.nextID, + createdAt: time.Now(), + from: from, + helo: helo, + recipient: recipient, + ipv4: ip4, + ipv6: ip6, + returnPath: returnPath, + transport: transport, + protocol: protocol, + } + m.emails = append(m.emails, &email) + m.nextID++ + return email.mailID, nil +} + +// CloseMessage implements the Storage interface +func (m *StoreMemory) CloseMessage( + mailID uint64, + size int64, + partsInfo *PartsInfo, + subject string, + to string, + from string) error { + if email := m.emails[mailID-m.offset]; email == nil { + return errors.New("email not found") + } else { + email.size = size + if info, err := partsInfo.MarshalJSONZlib(); err != nil { + return err + } else { + email.partsInfo = info + } + email.subject = subject + email.to = to + email.from = from + email.size = size + } + return nil +} + +// AddChunk implements the Storage interface +func (m *StoreMemory) AddChunk(data []byte, hash []byte) error { + var key HashKey + if len(hash) != hashByteSize { + return errors.New("invalid hash") + } + key.Pack(hash) + var compressed bytes.Buffer + zlibw, err := zlib.NewWriterLevel(&compressed, m.config.CompressLevel) + if err != nil { + return err + } + if chunk, ok := m.chunks[key]; ok { + // only update the counters and update time + chunk.referenceCount++ + chunk.modifiedAt = time.Now() + } else { + if _, err := zlibw.Write(data); err != nil { + return err + } + if err := zlibw.Close(); err != nil { + return err + } + // add a new chunk + newChunk := memoryChunk{ + modifiedAt: time.Now(), + referenceCount: 1, + data: compressed.Bytes(), + } + m.chunks[key] = &newChunk + } + return nil +} + +// Initialize implements the Storage interface +func (m *StoreMemory) Initialize(cfg backends.ConfigGroup) error { + + sd := backends.StreamDecorator{} + err := sd.ExtractConfig(cfg, &m.config) + if err != nil { + return err + } + m.offset = 1 + m.nextID = m.offset + m.emails = make([]*memoryEmail, 0, 100) + m.chunks = make(map[HashKey]*memoryChunk, 1000) + if m.config.CompressLevel > 9 || m.config.CompressLevel < 0 { + m.config.CompressLevel = zlib.BestCompression + } + return nil +} + +// Shutdown implements the Storage interface +func (m *StoreMemory) Shutdown() (err error) { + m.emails = nil + m.chunks = nil + return nil +} + +// GetEmail implements the Storage interface +func (m *StoreMemory) GetMessage(mailID uint64) (*Email, error) { + if count := len(m.emails); count == 0 { + return nil, errors.New("storage is empty") + } else if overflow := uint64(count) - m.offset; overflow > mailID-m.offset { + return nil, errors.New("mail not found") + } + email := m.emails[mailID-m.offset] + pi := NewPartsInfo() + if err := pi.UnmarshalJSONZlib(email.partsInfo); err != nil { + return nil, err + } + return &Email{ + mailID: email.mailID, + createdAt: email.createdAt, + size: email.size, + from: email.from, + to: email.to, + partsInfo: *pi, + helo: email.helo, + subject: email.subject, + queuedID: email.queuedID, + recipient: email.recipient, + ipv4: email.ipv4, + ipv6: email.ipv6, + returnPath: email.returnPath, + transport: email.transport, + protocol: email.protocol, + }, nil +} + +// GetChunk implements the Storage interface +func (m *StoreMemory) GetChunks(hash ...HashKey) ([]*Chunk, error) { + result := make([]*Chunk, 0, len(hash)) + var key HashKey + for i := range hash { + key = hash[i] + if c, ok := m.chunks[key]; ok { + zwr, err := zlib.NewReader(bytes.NewReader(c.data)) + if err != nil { + return nil, err + } + result = append(result, &Chunk{ + modifiedAt: c.modifiedAt, + referenceCount: c.referenceCount, + data: zwr, + }) + } + } + return result, nil +} diff --git a/chunk/store_sql.go b/chunk/store_sql.go new file mode 100644 index 00000000..5fd908b8 --- /dev/null +++ b/chunk/store_sql.go @@ -0,0 +1,607 @@ +package chunk + +import ( + "bytes" + "compress/zlib" + "database/sql" + "database/sql/driver" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/smtp" + "github.com/go-sql-driver/mysql" + "net" + "strings" + "sync" + "time" +) + +/* + +SQL schema + +``` + +create schema gmail collate utf8mb4_unicode_ci; + +CREATE TABLE `in_emails` ( + `mail_id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL, + `size` int unsigned NOT NULL, + `from` varbinary(255) NOT NULL, + `to` varbinary(255) NOT NULL, + `parts_info` text COLLATE utf8mb4_unicode_ci, + `helo` varchar(255) COLLATE latin1_swedish_ci NOT NULL, + `subject` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `queued_id` binary(16) NOT NULL, + `recipient` varbinary(255) NOT NULL, + `ipv4_addr` int unsigned DEFAULT NULL, + `ipv6_addr` varbinary(16) DEFAULT NULL, + `return_path` varbinary(255) NOT NULL, + `protocol` set('SMTP','SMTPS','ESMTP','ESMTPS','LMTP','LMTPS') COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'SMTP', + `transport` set('7bit','8bit','unknown','invalid') COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'unknown', + PRIMARY KEY (`mail_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE `in_emails_chunks` ( + `modified_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `reference_count` int unsigned DEFAULT '1', + `data` mediumblob NOT NULL, + `hash` varbinary(16) NOT NULL, + UNIQUE KEY `in_emails_chunks_hash_uindex` (`hash`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; + + +``` + +ipv6_addr is big endian + +TODO compression, configurable SQL strings, logger + +*/ +func init() { + StorageEngines["sql"] = func() Storage { + return new(StoreSQL) + } +} + +type sqlConfig struct { + + // EmailTable is the name of the main database table for the headers + EmailTable string `json:"email_table,omitempty"` + // EmailChunkTable stores the data of the emails in de-duplicated chunks + EmailChunkTable string `json:"email_table_chunks,omitempty"` + + // Connection settings + // Driver to use, eg "mysql" + Driver string `json:"sql_driver,omitempty"` + // DSN (required) is the connection string, eg. + // "user:passt@tcp(127.0.0.1:3306)/db_name?readTimeout=10s&writeTimeout=10s&charset=utf8mb4&collation=utf8mb4_unicode_ci" + DSN string `json:"sql_dsn,omitempty"` + // MaxConnLifetime (optional) is a duration, eg. "30s" + MaxConnLifetime string `json:"sql_max_conn_lifetime,omitempty"` + // MaxOpenConns (optional) specifies the number of maximum open connections + MaxOpenConns int `json:"sql_max_open_conns,omitempty"` + // MaxIdleConns + MaxIdleConns int `json:"sql_max_idle_conns,omitempty"` + + // CompressLevel controls the gzip compression level of email chunks. + // 0 = no compression, 1 == best speed, 9 == best compression, -1 == default, -2 == huffman only + CompressLevel int `json:"compress_level,omitempty"` +} + +// StoreSQL implements the Storage interface +type StoreSQL struct { + config sqlConfig + db *sql.DB + + sqlSelectChunk []*sql.Stmt + sqlInsertEmail *sql.Stmt + sqlInsertChunk *sql.Stmt + sqlFinalizeEmail *sql.Stmt + sqlChunkReferenceIncr *sql.Stmt + sqlChunkReferenceDecr *sql.Stmt + sqlSelectMail *sql.Stmt + + bufferPool sync.Pool +} + +func (s *StoreSQL) StartWorker() (stop chan bool) { + + timeo := time.Second * 1 + stop = make(chan bool) + go func() { + select { + + case <-stop: + return + + case <-time.After(timeo): + t1 := int64(time.Now().UnixNano()) + // do stuff here + + if (time.Now().UnixNano())-t1 > int64(time.Second*3) { + + } + + } + }() + return stop + +} + +func (s *StoreSQL) connect() (*sql.DB, error) { + var err error + if s.db, err = sql.Open(s.config.Driver, s.config.DSN); err != nil { + backends.Log().Error("cannot open database: ", err) + return nil, err + } + if s.config.MaxOpenConns != 0 { + s.db.SetMaxOpenConns(s.config.MaxOpenConns) + } + if s.config.MaxIdleConns != 0 { + s.db.SetMaxIdleConns(s.config.MaxIdleConns) + } + if s.config.MaxConnLifetime != "" { + t, err := time.ParseDuration(s.config.MaxConnLifetime) + if err != nil { + return nil, err + } + s.db.SetConnMaxLifetime(t) + } + // do we have permission to access the table? + _, err = s.db.Query("SELECT mail_id FROM " + s.config.EmailTable + " LIMIT 1") + if err != nil { + return nil, err + } + return s.db, err +} + +func (s *StoreSQL) prepareSql() error { + + // begin inserting an email (before saving chunks) + if stmt, err := s.db.Prepare(`INSERT INTO ` + + s.config.EmailTable + + ` (queued_id, created_at, ` + "`from`" + `, helo, recipient, ipv4_addr, ipv6_addr, return_path, transport, protocol) + VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`); err != nil { + return err + } else { + s.sqlInsertEmail = stmt + } + + // insert a chunk of email's data + if stmt, err := s.db.Prepare(`INSERT INTO ` + + s.config.EmailChunkTable + + ` (data, hash) + VALUES(?, ?)`); err != nil { + return err + } else { + s.sqlInsertChunk = stmt + } + + // finalize the email (the connection closed) + if stmt, err := s.db.Prepare(` + UPDATE ` + s.config.EmailTable + ` + SET size=?, parts_info=?, subject=?, ` + "`to`" + `=?, ` + "`from`" + `=? + WHERE mail_id = ? `); err != nil { + return err + } else { + s.sqlFinalizeEmail = stmt + } + + // Check the existence of a chunk (the reference_count col is incremented if it exists) + // This means we can avoid re-inserting an existing chunk, only update its reference_count + // check the "affected rows" count after executing query + if stmt, err := s.db.Prepare(` + UPDATE ` + s.config.EmailChunkTable + ` + SET reference_count=reference_count+1 + WHERE hash = ? `); err != nil { + return err + } else { + s.sqlChunkReferenceIncr = stmt + } + + // If the reference_count is 0 then it means the chunk has been deleted + // Chunks are soft-deleted for now, hard-deleted by another sweeper query as they become stale. + if stmt, err := s.db.Prepare(` + UPDATE ` + s.config.EmailChunkTable + ` + SET reference_count=reference_count-1 + WHERE hash = ? AND reference_count > 0`); err != nil { + return err + } else { + s.sqlChunkReferenceDecr = stmt + } + + // fetch an email + if stmt, err := s.db.Prepare(` + SELECT * + from ` + s.config.EmailTable + ` + where mail_id=?`); err != nil { + return err + } else { + s.sqlSelectMail = stmt + } + + // fetch a chunk, used in GetChunks + // prepare a query for all possible combinations is prepared + + for i := 0; i < chunkPrefetchMax; i++ { + if stmt, err := s.db.Prepare( + s.getChunksSQL(i + 1), + ); err != nil { + return err + } else { + s.sqlSelectChunk[i] = stmt + } + } + + // TODO sweep old chunks + + // TODO sweep incomplete emails + + return nil +} + +const mysqlYYYY_m_d_s_H_i_s = "2006-01-02 15:04:05" + +// OpenMessage implements the Storage interface +func (s *StoreSQL) OpenMessage( + queuedID mail.Hash128, + from string, + helo string, + recipient string, + ipAddress IPAddr, + returnPath string, + protocol mail.Protocol, + transport smtp.TransportType, +) (mailID uint64, err error) { + + // if it's ipv4 then we want ipv6 to be 0, and vice-versa + var ip4 uint32 + ip6 := make([]byte, 16) + if ip := ipAddress.IP.To4(); ip != nil { + ip4 = binary.BigEndian.Uint32(ip) + } else { + copy(ip6, ipAddress.IP) + } + r, err := s.sqlInsertEmail.Exec( + queuedID.Bytes(), + time.Now().Format(mysqlYYYY_m_d_s_H_i_s), + from, + helo, + recipient, + ip4, + ip6, + returnPath, + transport.String(), + protocol.String()) + if err != nil { + return 0, err + } + id, err := r.LastInsertId() + if err != nil { + return 0, err + } + return uint64(id), err +} + +// AddChunk implements the Storage interface +func (s *StoreSQL) AddChunk(data []byte, hash []byte) error { + // attempt to increment the reference_count (it means the chunk is already in there) + r, err := s.sqlChunkReferenceIncr.Exec(hash) + if err != nil { + return err + } + affected, err := r.RowsAffected() + if err != nil { + return err + } + if affected == 0 { + // chunk isn't in there, let's insert it + + compressed := s.bufferPool.Get().(*bytes.Buffer) + defer func() { + compressed.Reset() + s.bufferPool.Put(compressed) + }() + zlibw, err := zlib.NewWriterLevel(compressed, s.config.CompressLevel) + if err != nil { + return err + } + if _, err := zlibw.Write(data); err != nil { + return err + } + if err := zlibw.Close(); err != nil { + return err + } + _, err = s.sqlInsertChunk.Exec(compressed.Bytes(), hash) + if err != nil { + return err + } + } + return nil +} + +// CloseMessage implements the Storage interface +func (s *StoreSQL) CloseMessage( + mailID uint64, + size int64, + partsInfo *PartsInfo, + subject string, + to string, from string) error { + partsInfoJson, err := json.Marshal(partsInfo) + if err != nil { + return err + } + _, err = s.sqlFinalizeEmail.Exec(size, partsInfoJson, subject, to, from, mailID) + if err != nil { + return err + } + return nil +} + +// Initialize loads the specific database config, connects to the db, prepares statements +func (s *StoreSQL) Initialize(cfg backends.ConfigGroup) error { + sd := backends.StreamDecorator{} + err := sd.ExtractConfig(cfg, &s.config) + if err != nil { + return err + } + if s.config.EmailTable == "" { + s.config.EmailTable = "in_emails" + } + if s.config.EmailChunkTable == "" { + s.config.EmailChunkTable = "in_emails_chunks" + } + if s.config.Driver == "" { + s.config.Driver = "mysql" + } + + s.bufferPool = sync.Pool{ + // if not available, then create a new one + New: func() interface{} { + var b bytes.Buffer + return &b + }, + } + + // because it uses an IN(?) query, so we need a different query for each possible ? combination (max chunkPrefetchMax) + s.sqlSelectChunk = make([]*sql.Stmt, chunkPrefetchMax) + + s.db, err = s.connect() + if err != nil { + return err + } + err = s.prepareSql() + if err != nil { + return err + } + return nil +} + +// Shutdown implements the Storage interface +func (s *StoreSQL) Shutdown() (err error) { + defer func() { + closeErr := s.db.Close() + if closeErr != err { + backends.Log().WithError(err).Error("failed to close sql database") + err = closeErr + } + }() + toClose := []*sql.Stmt{ + s.sqlInsertEmail, + s.sqlFinalizeEmail, + s.sqlInsertChunk, + s.sqlChunkReferenceIncr, + s.sqlChunkReferenceDecr, + s.sqlSelectMail, + } + toClose = append(toClose, s.sqlSelectChunk...) + + for i := range toClose { + if err = toClose[i].Close(); err != nil { + backends.Log().WithError(err).Error("failed to close sql statement") + } + } + return err +} + +// GetEmail implements the Storage interface +func (s *StoreSQL) GetMessage(mailID uint64) (*Email, error) { + + email := &Email{} + var createdAt mysql.NullTime + var transport transportType + var protocol protocol + err := s.sqlSelectMail.QueryRow(mailID).Scan( + &email.mailID, + &createdAt, + &email.size, + &email.from, + &email.to, + &email.partsInfo, + &email.helo, + &email.subject, + &email.queuedID, + &email.recipient, + &email.ipv4, + &email.ipv6, + &email.returnPath, + &protocol, + &transport, + ) + email.createdAt = createdAt.Time + email.protocol = protocol.Protocol + email.transport = transport.TransportType + if err != nil { + return email, err + } + return email, nil +} + +// Value implements the driver.Valuer interface +func (h HashKey) Value() (driver.Value, error) { + return h[:], nil +} + +func (h *HashKey) Scan(value interface{}) error { + b := value.([]uint8) + h.Pack(b) + return nil +} + +type chunkData []uint8 + +func (v chunkData) Value() (driver.Value, error) { + return v[:], nil +} + +func (s *StoreSQL) getChunksSQL(size int) string { + return fmt.Sprintf("SELECT modified_at, reference_count, data, `hash` FROM %s WHERE `hash` in (%s)", + s.config.EmailChunkTable, + "?"+strings.Repeat(",?", size-1), + ) +} + +// GetChunks implements the Storage interface +func (s *StoreSQL) GetChunks(hash ...HashKey) ([]*Chunk, error) { + result := make([]*Chunk, len(hash)) + // we need to wrap these in an interface{} so that they can be passed to db.Query + args := make([]interface{}, len(hash)) + for i := range hash { + args[i] = &hash[i] + } + rows, err := s.sqlSelectChunk[len(args)-1].Query(args...) + defer func() { + if rows != nil { + _ = rows.Close() + } + }() + if err != nil { + return result, err + } + // temp is a lookup table for hash -> chunk + // since rows can come in different order, we need to make sure + // that result is sorted in the order of args + temp := make(map[HashKey]*Chunk, len(hash)) + i := 0 + for rows.Next() { + var createdAt mysql.NullTime + var data chunkData + var h HashKey + c := Chunk{} + if err := rows.Scan( + &createdAt, + &c.referenceCount, + &data, + &h, + ); err != nil { + return result, err + } + c.data, err = zlib.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + c.modifiedAt = createdAt.Time + temp[h] = &c + i++ + } + // re-order the rows according to the order of the args (very important) + for i := range args { + b := args[i].(*HashKey) + if _, ok := temp[*b]; ok { + result[i] = temp[*b] + } + } + if err := rows.Err(); err != nil || i == 0 { + return result, errors.New("data chunks not found") + } + return result, nil +} + +// zap is used in testing, purges everything +func (s *StoreSQL) zap() error { + if r, err := s.db.Exec("DELETE from " + s.config.EmailTable + " "); err != nil { + return err + } else { + affected, _ := r.RowsAffected() + fmt.Println(fmt.Sprintf("deleted %v emails", affected)) + } + + if r, err := s.db.Exec("DELETE from " + s.config.EmailChunkTable + " "); err != nil { + return err + } else { + affected, _ := r.RowsAffected() + fmt.Println(fmt.Sprintf("deleted %v chunks", affected)) + } + + return nil + +} + +// Scan implements database/sql scanner interface, for parsing PartsInfo +func (info *PartsInfo) Scan(value interface{}) error { + if value == nil { + return errors.New("parts_info is null") + } + if data, ok := value.([]byte); !ok { + return errors.New("parts_info is not str") + } else { + if err := json.Unmarshal(data, info); err != nil { + return err + } + } + return nil +} + +// /Scan implements database/sql scanner interface, for parsing net.IPAddr +func (ip *IPAddr) Scan(value interface{}) error { + if value == nil { + return nil + } + if data, ok := value.([]uint8); ok { + if len(data) == 16 { // 128 bits + // ipv6 + ipv6 := make(net.IP, 16) + copy(ipv6, data) + ip.IPAddr.IP = ipv6 + } + } + if data, ok := value.(int64); ok { + // ipv4 + ipv4 := make(net.IP, 4) + binary.BigEndian.PutUint32(ipv4, uint32(data)) + ip.IPAddr.IP = ipv4 + } + + return nil +} + +type transportType struct { + smtp.TransportType +} + +type protocol struct { + mail.Protocol +} + +// Scan implements database/sql scanner interface, for parsing smtp.TransportType +func (t *transportType) Scan(value interface{}) error { + if data, ok := value.([]uint8); ok { + v := smtp.ParseTransportType(string(data)) + t.TransportType = v + } + return nil +} + +// Scan implements database/sql scanner interface, for parsing mail.Protocol +func (p *protocol) Scan(value interface{}) error { + if data, ok := value.([]uint8); ok { + v := mail.ParseProtocolType(string(data)) + p.Protocol = v + } + return nil +} diff --git a/chunk/store_sql_test.go b/chunk/store_sql_test.go new file mode 100644 index 00000000..cefc2e24 --- /dev/null +++ b/chunk/store_sql_test.go @@ -0,0 +1,132 @@ +package chunk + +import ( + "bytes" + "flag" + "fmt" + "github.com/flashmob/go-guerrilla/mail" + "github.com/flashmob/go-guerrilla/mail/smtp" + "io" + "strings" + "testing" + + "github.com/flashmob/go-guerrilla/backends" + "github.com/flashmob/go-guerrilla/chunk/transfer" + _ "github.com/go-sql-driver/mysql" // activate the mysql driver +) + +// This test requires that you pass the -sql-dsn flag, +// eg: go test -run ^TestSQLStore$ -sql-dsn 'user:pass@tcp(127.0.0.1:3306)/dbname?readTimeout=10s&writeTimeout=10s' + +var ( + mailTableFlag = flag.String("mail-table", "in_emails", "Table to use for testing the SQL backend") + chunkTableFlag = flag.String("mail-chunk-table", "in_emails_chunks", "Table to use for testing the chunking SQL backend") + sqlDSNFlag = flag.String("sql-dsn", "", "DSN to use for testing the SQL backend") + sqlDriverFlag = flag.String("sql-driver", "mysql", "Driver to use for testing the SQL backend") +) + +func TestSQLStore(t *testing.T) { + + if *sqlDSNFlag == "" { + t.Skip("requires -sql-dsn to run") + } + + cfg := &backends.ConfigGroup{ + "chunk_size": 150, + "storage_engine": "sql", + "compress_level": 9, + "sql_driver": *sqlDriverFlag, + "sql_dsn": *sqlDSNFlag, + "email_table": *mailTableFlag, + "email_table_chunks": *chunkTableFlag, + } + + store, chunksaver, mimeanalyzer, stream, e, err := initTestStream(false, cfg) + if err != nil { + t.Error(err) + return + } + storeSql := store.(*StoreSQL) + defer func() { + storeSql.zap() // purge everything from db before exiting the test + }() + var out bytes.Buffer + buf := make([]byte, 128) + if written, err := io.CopyBuffer(stream, bytes.NewBuffer([]byte(email)), buf); err != nil { + t.Error(err) + } else { + _ = mimeanalyzer.Close() + _ = chunksaver.Close() + + fmt.Println("written:", written) + + email, err := storeSql.GetMessage(e.MessageID) + + if err != nil { + t.Error("email not found") + return + } + + // check email + if email.transport != smtp.TransportType8bit { + t.Error("email.transport not ", smtp.TransportType8bit.String()) + } + if email.protocol != mail.ProtocolESMTPS { + t.Error("email.protocol not ", mail.ProtocolESMTPS) + } + + // this should read all parts + r, err := NewChunkedReader(storeSql, email, 0) + if w, err := io.Copy(&out, r); err != nil { + t.Error(err) + } else if w != email.size { + t.Error("email.size != number of bytes copied from reader", w, email.size) + } else if !strings.Contains(out.String(), "R0lGODlhEAA") { + t.Error("The email didn't decode properly, expecting R0lGODlhEAA") + } + out.Reset() + //return + // test the seek feature + r, err = NewChunkedReader(storeSql, email, 0) + if err != nil { + t.Error(err) + t.FailNow() + } + // we start from 1 because if the start from 0, all the parts will be read + for i := 1; i < len(email.partsInfo.Parts); i++ { + fmt.Println("seeking to", i) + err = r.SeekPart(i) + if err != nil { + t.Error(err) + } + w, err := io.Copy(&out, r) + if err != nil { + t.Error(err) + } + if w != int64(email.partsInfo.Parts[i-1].Size) { + t.Error(i, "incorrect size, expecting", email.partsInfo.Parts[i-1].Size, "but read:", w) + } + out.Reset() + } + + r, err = NewChunkedReader(storeSql, email, 0) + if err != nil { + t.Error(err) + } + part := email.partsInfo.Parts[0] + encoding := transfer.QuotedPrintable + if strings.Contains(part.TransferEncoding, "base") { + encoding = transfer.Base64 + } + dr, err := transfer.NewDecoder(r, encoding, part.Charset) + _ = dr + if err != nil { + t.Error(err) + t.FailNow() + } + //var decoded bytes.Buffer + //io.Copy(&decoded, dr) + //io.Copy(os.Stdout, dr) + + } +} diff --git a/chunk/transfer/decoder.go b/chunk/transfer/decoder.go new file mode 100644 index 00000000..c9f6ed93 --- /dev/null +++ b/chunk/transfer/decoder.go @@ -0,0 +1,145 @@ +package transfer + +import ( + "bytes" + "encoding/base64" + "github.com/flashmob/go-guerrilla/mail" + "io" + "mime/quotedprintable" + "strings" + + _ "github.com/flashmob/go-guerrilla/mail/encoding" +) + +type Encoding int + +const ( + Base64 Encoding = iota + QuotedPrintable + SevenBit // default, 1-127, 13 & 10 at line endings + EightBit // 998 octets per line, 13 & 10 at line endings + Binary // 8 bit with no line restrictions + +) + +func ParseEncoding(str string) Encoding { + if str == "base64" { + return Base64 + } else if str == "quoted-printable" { + return QuotedPrintable + } else if str == "7bit" { + return SevenBit + } else if str == "8bit" { + return EightBit + } else if str == "binary" { + return Binary + } + return SevenBit +} + +// decoder decodes base64 and q-printable, then converting charset to UTF-8 +type decoder struct { + state int + charset string + transport Encoding + r io.Reader +} + +// NewDecoder reads a MIME-part from an underlying reader r +// then decodes base64 or quoted-printable to 8bit, and converts encoding to UTF-8 +func NewDecoder(r io.Reader, transport Encoding, charset string) (*decoder, error) { + decoder := new(decoder) + decoder.transport = transport + decoder.charset = strings.ToLower(charset) + decoder.r = r + return decoder, nil +} + +func NewBodyDecoder(r io.Reader, transport Encoding, charset string) (*decoder, error) { + d, err := NewDecoder(r, transport, charset) + if err != nil { + return d, err + } + d.state = decoderStateDecodeSetup + return d, nil +} + +const chunkSaverNL = '\n' + +const ( + decoderStateFindHeaderEnd int = iota + decoderStateMatchNL + decoderStateDecodeSetup + decoderStateDecode +) + +func (r *decoder) Read(p []byte) (n int, err error) { + var start, buffered int + if r.state != decoderStateDecode { + // we haven't found the body yet, so read in some input and scan for the end of the header + // i.e. the body starts after "\n\n" is matched + buffered, err = r.r.Read(p) + if buffered == 0 { + return + } + } + buf := p[0:buffered] // we'll use p as a scratch buffer + for { + // The following is a simple state machine to find the body. Once it's found, the machine will go into a + // 'decoderStateDecode' state where the decoding will be performed with each call to Read + switch r.state { + case decoderStateFindHeaderEnd: + // finding the start of the header + if start = bytes.Index(buf, []byte{chunkSaverNL, chunkSaverNL}); start != -1 { + start += 2 // skip the \n\n + r.state = decoderStateDecodeSetup // found the header + continue + } else if buf[len(buf)-1] == chunkSaverNL { + // the last char is a \n so next call to Read will check if it starts with a matching \n + r.state = decoderStateMatchNL + } + case decoderStateMatchNL: + // check the first char if it is a '\n' because last time we matched a '\n' + if buf[0] == '\n' { + // found the header + start = 1 + r.state = decoderStateDecodeSetup + continue + } else { + r.state = decoderStateFindHeaderEnd + continue + } + case decoderStateDecodeSetup: + if start != buffered { + // include any bytes that have already been read + r.r = io.MultiReader(bytes.NewBuffer(buf[start:]), r.r) + } + switch r.transport { + case QuotedPrintable: + r.r = quotedprintable.NewReader(r.r) + case Base64: + r.r = base64.NewDecoder(base64.StdEncoding, r.r) + default: + + } + // conversion to utf-8 + if r.charset != "utf-8" { + r.r, err = mail.Dec.CharsetReader(r.charset, r.r) + if err != nil { + return + } + } + r.state = decoderStateDecode + continue + case decoderStateDecode: + // already found the body, do conversion and decoding + return r.r.Read(p) + } + start = 0 + // haven't found the body yet, continue scanning + buffered, err = r.r.Read(buf) + if buffered == 0 { + return + } + } +} diff --git a/client.go b/client.go index 3ce13b86..4d87d64b 100644 --- a/client.go +++ b/client.go @@ -7,13 +7,12 @@ import ( "errors" "fmt" "net" - "net/textproto" "sync" "time" "github.com/flashmob/go-guerrilla/log" "github.com/flashmob/go-guerrilla/mail" - "github.com/flashmob/go-guerrilla/mail/rfc5321" + "github.com/flashmob/go-guerrilla/mail/smtp" "github.com/flashmob/go-guerrilla/response" ) @@ -48,21 +47,21 @@ type client struct { conn net.Conn bufin *smtpBufferedReader bufout *bufio.Writer - smtpReader *textproto.Reader + smtpReader *mail.MimeDotReader ar *adjustableLimitedReader // guards access to conn connGuard sync.Mutex log log.Logger - parser rfc5321.Parser + parser smtp.Parser } // NewClient allocates a new client. -func NewClient(conn net.Conn, clientID uint64, logger log.Logger, envelope *mail.Pool) *client { +func NewClient(conn net.Conn, clientID uint64, logger log.Logger, envelope *mail.Pool, serverID int) *client { c := &client{ conn: conn, // Envelope will be borrowed from the envelope pool // the envelope could be 'detached' from the client later when processing - Envelope: envelope.Borrow(getRemoteAddr(conn), clientID), + Envelope: envelope.Borrow(getRemoteAddr(conn), clientID, serverID), ConnectedAt: time.Now(), bufin: newSMTPBufferedReader(conn), bufout: bufio.NewWriter(conn), @@ -71,7 +70,7 @@ func NewClient(conn net.Conn, clientID uint64, logger log.Logger, envelope *mail } // used for reading the DATA state - c.smtpReader = textproto.NewReader(c.bufin.Reader) + c.smtpReader = mail.NewMimeDotReader(c.bufin.Reader, 1) return c } @@ -118,6 +117,7 @@ func (c *client) sendResponse(r ...interface{}) { // -End of DATA command // TLS handshake func (c *client) resetTransaction() { + c.smtpReader = mail.NewMimeDotReader(c.bufin.Reader, 1) c.Envelope.ResetTransaction() } @@ -172,7 +172,7 @@ func (c *client) init(conn net.Conn, clientID uint64, ep *mail.Pool) { c.ID = clientID c.errors = 0 // borrow an envelope from the envelope pool - c.Envelope = ep.Borrow(getRemoteAddr(conn), clientID) + c.Envelope = ep.Borrow(getRemoteAddr(conn), clientID, c.ServerID) } // getID returns the client's unique ID @@ -211,7 +211,7 @@ type pathParser func([]byte) error func (c *client) parsePath(in []byte, p pathParser) (mail.Address, error) { address := mail.Address{} var err error - if len(in) > rfc5321.LimitPath { + if len(in) > smtp.LimitPath { return address, errors.New(response.Canned.FailPathTooLong.String()) } if err = p(in); err != nil { @@ -219,9 +219,9 @@ func (c *client) parsePath(in []byte, p pathParser) (mail.Address, error) { } else if c.parser.NullPath { // bounce has empty from address address = mail.Address{} - } else if len(c.parser.LocalPart) > rfc5321.LimitLocalPart { + } else if len(c.parser.LocalPart) > smtp.LimitLocalPart { err = errors.New(response.Canned.FailLocalPartTooLong.String()) - } else if len(c.parser.Domain) > rfc5321.LimitDomain { + } else if len(c.parser.Domain) > smtp.LimitDomain { err = errors.New(response.Canned.FailDomainTooLong.String()) } else { address = mail.Address{ @@ -236,7 +236,3 @@ func (c *client) parsePath(in []byte, p pathParser) (mail.Address, error) { } return address, err } - -func (s *server) rcptTo() (address mail.Address, err error) { - return address, err -} diff --git a/cmd/guerrillad/serve.go b/cmd/guerrillad/serve.go index 0813eeeb..7a4be804 100644 --- a/cmd/guerrillad/serve.go +++ b/cmd/guerrillad/serve.go @@ -47,7 +47,7 @@ func init() { var err error mainlog, err = log.GetLogger(log.OutputStderr.String(), log.InfoLevel.String()) if err != nil && mainlog != nil { - mainlog.WithError(err).Errorf("Failed creating a logger to %s", log.OutputStderr) + mainlog.Fields("error", err, "file", log.OutputStderr).Error("failed creating a logger") } cfgFile := "goguerrilla.conf" // deprecated default name if _, err := os.Stat(cfgFile); err != nil { @@ -83,7 +83,7 @@ func sigHandler() { mainlog.WithError(err).Error("reopening logs failed") } } else if sig == syscall.SIGTERM || sig == syscall.SIGQUIT || sig == syscall.SIGINT || sig == os.Kill { - mainlog.Infof("Shutdown signal caught") + mainlog.Info("shutdown signal caught") go func() { select { // exit if graceful shutdown not finished in 60 sec. @@ -93,10 +93,10 @@ func sigHandler() { } }() d.Shutdown() - mainlog.Infof("Shutdown completed, exiting.") + mainlog.Info("Shutdown completed, exiting") return } else { - mainlog.Infof("Shutdown, unknown signal caught") + mainlog.Info("Shutdown, unknown signal caught") return } } @@ -107,19 +107,20 @@ func serve(cmd *cobra.Command, args []string) { d = guerrilla.Daemon{Logger: mainlog} c, err := readConfig(configPath, pidFile) if err != nil { - mainlog.WithError(err).Fatal("Error while reading config") + mainlog.WithError(err).Fatal("error while reading config") } _ = d.SetConfig(*c) // Check that max clients is not greater than system open file limit. if ok, maxClients, fileLimit := guerrilla.CheckFileLimit(c); !ok { - mainlog.Fatalf("Combined max clients for all servers (%d) is greater than open file limit (%d). "+ - "Please increase your open file limit or decrease max clients.", maxClients, fileLimit) + mainlog.Fields("naxClients", maxClients, "fileLimit", fileLimit). + Fatal("combined max clients for all servers is greater than open file limit, " + + "please increase your open file limit or decrease max clients") } err = d.Start() if err != nil { - mainlog.WithError(err).Error("Error(s) when creating new server(s)") + mainlog.WithError(err).Error("error(s) when creating new server(s)") os.Exit(1) } sigHandler() diff --git a/cmd/guerrillad/serve_test.go b/cmd/guerrillad/serve_test.go index 717a7ab7..c48f45f2 100644 --- a/cmd/guerrillad/serve_test.go +++ b/cmd/guerrillad/serve_test.go @@ -29,7 +29,7 @@ var configJsonA = ` { "log_file" : "../../tests/testlog", "log_level" : "debug", - "pid_file" : "./pidfile.pid", + "pid_file" : "pidfile.pid", "allowed_hosts": [ "guerrillamail.com", "guerrillamailblock.com", @@ -37,11 +37,19 @@ var configJsonA = ` "guerrillamail.net", "guerrillamail.org" ], - "backend_config": { - "save_workers_size" : 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": true - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 1, + "save_process": "HeadersParser|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -90,11 +98,19 @@ var configJsonB = ` "guerrillamail.net", "guerrillamail.org" ], - "backend_config": { - "save_workers_size" : 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": false - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : false + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 1, + "save_process": "HeadersParser|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -128,19 +144,24 @@ var configJsonC = ` "guerrillamail.net", "guerrillamail.org" ], - "backend_config" : - { - "sql_driver": "mysql", - "sql_dsn": "root:ok@tcp(127.0.0.1:3306)/gmail_mail?readTimeout=10s&writeTimeout=10s", - "mail_table":"new_mail", - "redis_interface" : "127.0.0.1:6379", - "redis_expire_seconds" : 7200, - "save_workers_size" : 3, - "primary_mail_host":"sharklasers.com", - "save_workers_size" : 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": true - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + }, + "sql" : { + "sql_dsn": "root:ok@tcp(127.0.0.1:3306)/gmail_mail?readTimeout=10s&writeTimeout=10s", + "mail_table":"new_mail", + "primary_mail_host":"sharklasers.com" + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 3, + "save_process": "HeadersParser|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -189,11 +210,19 @@ var configJsonD = ` "guerrillamail.net", "guerrillamail.org" ], - "backend_config": { - "save_workers_size" : 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": false - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : false + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 1, + "save_process": "HeadersParser|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -242,19 +271,33 @@ var configJsonE = ` "guerrillamail.net", "guerrillamail.org" ], - "backend_config" : - { - "save_process_old": "HeadersParser|Debugger|Hasher|Header|Compressor|Redis|MySql", - "save_process": "GuerrillaRedisDB", - "log_received_mails" : true, - "sql_driver": "mysql", - "sql_dsn": "root:secret@tcp(127.0.0.1:3306)/gmail_mail?readTimeout=10s&writeTimeout=10s", - "mail_table":"new_mail", - "redis_interface" : "127.0.0.1:6379", - "redis_expire_seconds" : 7200, - "save_workers_size" : 3, - "primary_mail_host":"sharklasers.com" - }, + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + }, + "sql" : { + "sql_dsn": "root:secret@tcp(127.0.0.1:3306)/gmail_mail?readTimeout=10s&writeTimeout=10s", + "mail_table":"new_mail", + "sql_driver": "mysql", + "primary_mail_host":"sharklasers.com" + }, + "GuerrillaRedisDB" : { + "sql_dsn": "root:secret@tcp(127.0.0.1:3306)/gmail_mail?readTimeout=10s&writeTimeout=10s", + "mail_table":"new_mail", + "sql_driver": "mysql", + "redis_interface" : "127.0.0.1:6379", + "redis_expire_seconds" : 7200, + "primary_mail_host":"sharklasers.com" + } + }, + "gateways" : { + "default" : { + "save_workers_size" : 3, + "save_process": "GuerrillaRedisDB" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -293,29 +336,39 @@ var configJsonE = ` const testPauseDuration = time.Millisecond * 1010 // reload config -func sigHup() { - if data, err := ioutil.ReadFile("pidfile.pid"); err == nil { - mainlog.Infof("pid read is %s", data) - ecmd := exec.Command("kill", "-HUP", string(data)) - _, err = ecmd.Output() - if err != nil { - mainlog.Infof("could not SIGHUP", err) +func sigHup(pidfile string) { + if data, err := ioutil.ReadFile(pidfile); err == nil { + if pid, err := strconv.Atoi(string(data)); err != nil { + mainlog.WithError(err).Error("could not read pid") + } else { + mainlog.Fields("pid", pid).Info("pid read") + ecmd := exec.Command("kill", "-HUP", string(data)) + _, err = ecmd.Output() + if err != nil { + mainlog.WithError(err).Error("could not SIGHUP") + } } + } else { - mainlog.WithError(err).Info("sighup - Could not read pidfle") + mainlog.Fields("error", err, "file", pidfile).Info("sighup - Could not read pidfle") } } // shutdown after calling serve() -func sigKill() { - if data, err := ioutil.ReadFile("pidfile.pid"); err == nil { - mainlog.Infof("pid read is %s", data) - ecmd := exec.Command("kill", string(data)) - _, err = ecmd.Output() - if err != nil { - mainlog.Infof("could not sigkill", err) +func sigKill(pidfile string) { + if data, err := ioutil.ReadFile(pidfile); err == nil { + if pid, err := strconv.Atoi(string(data)); err != nil { + mainlog.WithError(err).Error("could not read pid") + } else { + mainlog.Fields("pid", pid).Info("pid read") + ecmd := exec.Command("kill", string(data)) + _, err = ecmd.Output() + if err != nil { + mainlog.WithError(err).Info("could not sigkill") + } } + } else { mainlog.WithError(err).Info("sigKill - Could not read pidfle") } @@ -334,6 +387,42 @@ func exponentialBackoff(i int) { time.Sleep(time.Duration(Round(math.Pow(3.0, float64(i))-1.0)*100.0/2.0) * time.Millisecond) } +func matchTestlog(startLine int, args ...interface{}) (bool, error) { + fd, err := os.Open("../../tests/testlog") + if err != nil { + return false, err + } + defer func() { + _ = fd.Close() + }() + for tries := 0; tries < 6; tries++ { + if b, err := ioutil.ReadAll(fd); err != nil { + return false, err + } else { + if test.MatchLog(string(b), startLine, args...) { + return true, nil + } + } + // close and reopen + err = fd.Close() + if err != nil { + return false, err + } + fd = nil + + // sleep + exponentialBackoff(tries) + _ = mainlog.Reopen() + + // re-open + fd, err = os.OpenFile("../../tests/testlog", os.O_RDONLY, 0644) + if err != nil { + return false, err + } + } + return false, nil +} + var grepNotFound error // grepTestlog looks for the `match` string in the testlog @@ -356,7 +445,6 @@ func grepTestlog(match string, lineNumber int) (found int, err error) { var ln int var line string for tries := 0; tries < 6; tries++ { - //fmt.Println("try..", tries) for { ln++ line, err = buff.ReadString('\n') @@ -364,7 +452,6 @@ func grepTestlog(match string, lineNumber int) (found int, err error) { break } if ln > lineNumber { - //fmt.Print(ln, line) if i := strings.Index(line, match); i != -1 { return ln, nil } @@ -496,8 +583,8 @@ func TestCmdConfigChangeEvents(t *testing.T) { } expectedEvents := map[guerrilla.Event]bool{ - guerrilla.EventConfigBackendConfig: false, - guerrilla.EventConfigServerNew: false, + guerrilla.EventConfigBackendConfigChanged: false, // backend_change:backend + guerrilla.EventConfigServerNew: false, // server_change:new_server } mainlog, err = getTestLog() if err != nil { @@ -505,24 +592,37 @@ func TestCmdConfigChangeEvents(t *testing.T) { t.FailNow() } - bcfg := backends.BackendConfig{"log_received_mails": true} - backend, err := backends.New(bcfg, mainlog) - app, err := guerrilla.New(oldconf, backend, mainlog) + oldconf.BackendConfig = backends.BackendConfig{ + backends.ConfigProcessors: {"debugger": {"log_received_mails": true}}, + } + oldconf.BackendConfig.ConfigureDefaults() + + backend, err := backends.New(backends.DefaultGateway, oldconf.BackendConfig, mainlog) + if err != nil { + t.Error("failed to create backend", err) + return + } + app, err := guerrilla.New(oldconf, mainlog, backend) if err != nil { t.Error("Failed to create new app", err) } - toUnsubscribe := map[guerrilla.Event]func(c *guerrilla.AppConfig){} - toUnsubscribeS := map[guerrilla.Event]func(c *guerrilla.ServerConfig){} + toUnsubscribe := map[guerrilla.Event]interface{}{} for event := range expectedEvents { // Put in anon func since range is overwriting event func(e guerrilla.Event) { - if strings.Index(e.String(), "server_change") == 0 { + if strings.Index(e.String(), "backend_change") == 0 { + f := func(c *guerrilla.AppConfig, gateway string) { + expectedEvents[e] = true + } + _ = app.Subscribe(e, f) + toUnsubscribe[e] = f + } else if strings.Index(e.String(), "server_change") == 0 { f := func(c *guerrilla.ServerConfig) { expectedEvents[e] = true } _ = app.Subscribe(e, f) - toUnsubscribeS[e] = f + toUnsubscribe[e] = f } else { f := func(c *guerrilla.AppConfig) { expectedEvents[e] = true @@ -530,7 +630,6 @@ func TestCmdConfigChangeEvents(t *testing.T) { _ = app.Subscribe(e, f) toUnsubscribe[e] = f } - }(event) } @@ -541,10 +640,6 @@ func TestCmdConfigChangeEvents(t *testing.T) { for unevent, unfun := range toUnsubscribe { _ = app.Unsubscribe(unevent, unfun) } - for unevent, unfun := range toUnsubscribeS { - _ = app.Unsubscribe(unevent, unfun) - } - for event, val := range expectedEvents { if val == false { t.Error("Did not fire config change event:", event) @@ -576,13 +671,18 @@ func TestServe(t *testing.T) { cmd := &cobra.Command{} configPath = "configJsonA.json" - go func() { serve(cmd, []string{}) }() - if _, err := grepTestlog("istening on TCP 127.0.0.1:3536", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:3536\"", 0); err != nil { t.Error("server not started") } + + // wait for the pidfle to be written out + if _, err := grepTestlog("pid_file", 0); err != nil { + t.Error("pid_file not written") + } + data, err := ioutil.ReadFile("pidfile.pid") if err != nil { t.Error("error reading pidfile.pid", err) @@ -605,23 +705,22 @@ func TestServe(t *testing.T) { // Would not work on windows as kill is not available. // TODO: Implement an alternative test for windows. if runtime.GOOS != "windows" { - sigHup() + sigHup("pidfile.pid") // did the pidfile change as expected? - if _, err := grepTestlog("Configuration was reloaded", 0); err != nil { - t.Error("server did not catch sighp") + if _, err := grepTestlog("configuration was reloaded", 0); err != nil { + t.Error("server did not catch sighup") } } + if _, err := grepTestlog("gateway with new config started", 0); err != nil { + t.Error("Dummy backend not restarted") + } + // send kill signal and wait for exit d.Shutdown() - // did backend started as expected? - if _, err := grepTestlog("new backend started", 0); err != nil { - t.Error("Dummy backend not restarted") - } - // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } @@ -656,7 +755,7 @@ func TestServerAddEvent(t *testing.T) { }() // allow the server to start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:3536", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:3536\"", 0); err != nil { t.Error("server didn't start") } @@ -676,13 +775,14 @@ func TestServerAddEvent(t *testing.T) { } } // send a sighup signal to the server - sigHup() - if _, err := grepTestlog("[127.0.0.1:2526] Waiting for a new client", 0); err != nil { + sigHup("./pidfile.pid") + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2526\"", 0); err != nil { t.Error("new server didn't start") } if conn, buffin, err := test.Connect(newServer, 20); err != nil { t.Error("Could not connect to new server", newServer.ListenInterface, err) + return } else { if result, err := test.Command(conn, buffin, "HELO example.com"); err == nil { expect := "250 mail.test.com Hello" @@ -696,14 +796,19 @@ func TestServerAddEvent(t *testing.T) { // shutdown the server d.Shutdown() - - // did backend started as expected? - if _, err := grepTestlog("New server added [127.0.0.1:2526]", 0); err != nil { - t.Error("Did not add server [127.0.0.1:2526] after sighup") + // sever added as as expected? + if matched, err := matchTestlog( + 1, "msg", "new server added", + "iface", "127.0.0.1:2526", + "event", "server_change:new_server", + ); !matched { + t.Error("Did not add server [127.0.0.1:2526] after sighup", err) } - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { - t.Error("Server failed to stop") + if matched, err := matchTestlog( + 1, "msg", "backend shutdown completed", + ); !matched { + t.Error("Server failed to stop", err) } } @@ -736,7 +841,7 @@ func TestServerStartEvent(t *testing.T) { go func() { serve(cmd, []string{}) }() - if _, err := grepTestlog("Listening on TCP 127.0.0.1:3536", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:3536\"", 0); err != nil { t.Error("server didn't start") } // now change the config by adding a server @@ -755,10 +860,10 @@ func TestServerStartEvent(t *testing.T) { t.Error(err) } // send a sighup signal to the server - sigHup() + sigHup("pidfile.pid") // see if the new server started? - if _, err := grepTestlog("Listening on TCP 127.0.0.1:2228", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2228\"", 0); err != nil { t.Error("second server didn't start") } @@ -778,7 +883,7 @@ func TestServerStartEvent(t *testing.T) { // shutdown and wait for exit d.Shutdown() - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } @@ -817,7 +922,7 @@ func TestServerStopEvent(t *testing.T) { serve(cmd, []string{}) }() // allow the server to start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:3536", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:3536\"", 0); err != nil { t.Error("server didn't start") } // now change the config by enabling a server @@ -836,9 +941,9 @@ func TestServerStopEvent(t *testing.T) { t.Error(err) } // send a sighup signal to the server - sigHup() + sigHup("pidfile.pid") // detect config change - if _, err := grepTestlog("Listening on TCP 127.0.0.1:2228", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2228\"", 0); err != nil { t.Error("new server didn't start") } @@ -869,9 +974,9 @@ func TestServerStopEvent(t *testing.T) { t.Error(err) } // send a sighup signal to the server - sigHup() + sigHup("pidfile.pid") // detect config change - if _, err := grepTestlog("Server [127.0.0.1:2228] has stopped accepting new clients", 27); err != nil { + if _, err := grepTestlog("msg=\"server has stopped accepting new clients\" iface=\"127.0.0.1:2228\"", 27); err != nil { t.Error("127.0.0.1:2228 did not stop") } @@ -883,7 +988,7 @@ func TestServerStopEvent(t *testing.T) { d.Shutdown() // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } } @@ -950,7 +1055,7 @@ func TestAllowedHostsEvent(t *testing.T) { serve(cmd, []string{}) }() // wait for start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:2552", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2552\"", 0); err != nil { t.Error("server didn't start") } @@ -986,7 +1091,7 @@ func TestAllowedHostsEvent(t *testing.T) { t.Error(err) } // send a sighup signal to the server to reload config - sigHup() + sigHup("pidfile.pid") if _, err := grepTestlog("allowed_hosts config changed", 0); err != nil { t.Error("allowed_hosts config not changed") @@ -1017,7 +1122,7 @@ func TestAllowedHostsEvent(t *testing.T) { d.Shutdown() // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } @@ -1059,7 +1164,7 @@ func TestTLSConfigEvent(t *testing.T) { }() // wait for server to start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:2552", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2552\"", 0); err != nil { t.Error("server didn't start") } @@ -1124,10 +1229,10 @@ func TestTLSConfigEvent(t *testing.T) { t.Error("Did not create cert ", err) } - sigHup() + sigHup("pidfile.pid") // wait for config to reload - if _, err := grepTestlog("Server [127.0.0.1:4655] re-opened", 0); err != nil { + if _, err := matchTestlog(1, "iface", "127.0.0.1:4655", "msg", "server re-opened"); err != nil { t.Error("server didn't catch sighup") } @@ -1148,7 +1253,7 @@ func TestTLSConfigEvent(t *testing.T) { d.Shutdown() // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } @@ -1194,7 +1299,7 @@ func TestBadTLSStart(t *testing.T) { // it should exit by now because the TLS config is incorrect time.Sleep(testPauseDuration) - sigKill() + sigKill("pidfile.pid") serveWG.Wait() return @@ -1247,7 +1352,7 @@ func TestBadTLSReload(t *testing.T) { serve(cmd, []string{}) }() // wait for server to start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:4655", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:4655\"", 0); err != nil { t.Error("server didn't start") } @@ -1283,7 +1388,7 @@ func TestBadTLSReload(t *testing.T) { t.Error(err) } // send a sighup signal to the server to reload config - sigHup() + sigHup("pidfile.pid") // did the config reload reload event fire? There should be config read error if _, err := grepTestlog("could not read config file", 0); err != nil { t.Error("was expecting an error reading config") @@ -1306,7 +1411,7 @@ func TestBadTLSReload(t *testing.T) { d.Shutdown() // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } @@ -1343,7 +1448,7 @@ func TestSetTimeoutEvent(t *testing.T) { serve(cmd, []string{}) }() // wait for start - if _, err := grepTestlog("Listening on TCP 127.0.0.1:4655", 0); err != nil { + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:4655\"", 0); err != nil { t.Error("server didn't start") } @@ -1360,7 +1465,7 @@ func TestSetTimeoutEvent(t *testing.T) { } // send a sighup signal to the server to reload config - sigHup() + sigHup("pidfile.pid") // did config update? if _, err := grepTestlog("a new config has been saved", 0); err != nil { @@ -1392,7 +1497,7 @@ func TestSetTimeoutEvent(t *testing.T) { d.Shutdown() // wait for shutdown - if _, err := grepTestlog("Backend shutdown completed", 0); err != nil { + if _, err := grepTestlog("backend shutdown completed", 0); err != nil { t.Error("server didn't stop") } // so the connection we have opened should timeout by now @@ -1435,7 +1540,8 @@ func TestDebugLevelChange(t *testing.T) { go func() { serve(cmd, []string{}) }() - if _, err := grepTestlog("Listening on TCP 127.0.0.1:2552", 0); err != nil { + + if _, err := grepTestlog("msg=\"listening on TCP\" iface=\"127.0.0.1:2552\"", 0); err != nil { t.Error("server didn't start") } @@ -1462,9 +1568,9 @@ func TestDebugLevelChange(t *testing.T) { t.Error(err) } // send a sighup signal to the server to reload config - sigHup() + sigHup("pidfile.pid") // did the config reload? - if _, err := grepTestlog("Configuration was reloaded", 0); err != nil { + if _, err := grepTestlog("configuration was reloaded", 0); err != nil { t.Error("config did not reload") t.FailNow() } @@ -1484,72 +1590,67 @@ func TestDebugLevelChange(t *testing.T) { d.Shutdown() - // did the log level change to info? - if _, err := grepTestlog("log level changed to [info]", 0); err != nil { - t.Error("log level did not change to [info]") + if ok, err := matchTestlog(1, "msg", "log level changed"); !ok { + t.Error("log level did not change", err) t.FailNow() } } // When reloading with a bad backend config, it should revert to old backend config +// using the API way func TestBadBackendReload(t *testing.T) { + + defer cleanTestArtifacts(t) var err error err = testcert.GenerateCert("mail2.guerrillamail.com", "", 365*24*time.Hour, false, 2048, "P256", "../../tests/") if err != nil { t.Error("failed to generate a test certificate", err) t.FailNow() } - defer cleanTestArtifacts(t) - - mainlog, err = getTestLog() - if err != nil { - t.Error("could not get logger,", err) - t.FailNow() - } - if err = ioutil.WriteFile("configJsonA.json", []byte(configJsonA), 0644); err != nil { t.Error(err) } - cmd := &cobra.Command{} - configPath = "configJsonA.json" - go func() { - serve(cmd, []string{}) - }() - if _, err := grepTestlog("Listening on TCP 127.0.0.1:3536", 0); err != nil { - t.Error("server didn't start") - } - // change the config file to the one with a broken backend - if err = ioutil.WriteFile("configJsonA.json", []byte(configJsonE), 0644); err != nil { - t.Error(err) + d := guerrilla.Daemon{} + _, err = d.LoadConfig("configJsonA.json") + if err != nil { + t.Error("ReadConfig error", err) + return } + err = d.Start() + if err != nil { + t.Error("start error", err) + return + } else { + if err = ioutil.WriteFile("configJsonA.json", []byte(configJsonE), 0644); err != nil { + t.Error(err) + return + } + if err = d.ReloadConfigFile("configJsonA.json"); err != nil { + t.Error(err) + return + } - // test SIGHUP via the kill command - // Would not work on windows as kill is not available. - // TODO: Implement an alternative test for windows. - if runtime.GOOS != "windows" { - sigHup() - // did config update? - if _, err := grepTestlog("Configuration was reloaded", 0); err != nil { + d.Shutdown() + + if _, err := grepTestlog("configuration was reloaded", 0); err != nil { t.Error("config didn't update") } + + // reverted to old gateway config? + if _, err := grepTestlog("reverted to old gateway config", 0); err != nil { + t.Error("Did not revert to old gateway config") + } + // did the pidfile change as expected? - if _, err := grepTestlog("pid_file (./pidfile2.pid) written", 0); err != nil { + if _, err := grepTestlog("msg=\"pid_file written\" file=./pidfile2.pid", 0); err != nil { t.Error("pid_file (./pidfile2.pid) not written") } if _, err := os.Stat("./pidfile2.pid"); os.IsNotExist(err) { t.Error("pidfile not changed after sighup SIGHUP", err) } - } - // send kill signal and wait for exit - d.Shutdown() - - // did backend started as expected? - if _, err := grepTestlog("reverted to old backend config", 0); err != nil { - t.Error("did not revert to old backend config") - t.FailNow() } } diff --git a/cmd/guerrillad/version.go b/cmd/guerrillad/version.go index 3cb121c8..28af5b76 100644 --- a/cmd/guerrillad/version.go +++ b/cmd/guerrillad/version.go @@ -20,7 +20,9 @@ func init() { } func logVersion() { - mainlog.Infof("guerrillad %s", guerrilla.Version) - mainlog.Debugf("Build Time: %s", guerrilla.BuildTime) - mainlog.Debugf("Commit: %s", guerrilla.Commit) + mainlog.Fields( + "version", guerrilla.Version, + "buildTime", guerrilla.BuildTime, + "commit", guerrilla.Commit). + Info("guerrillad") } diff --git a/config.go b/config.go index b93a3026..6ae4fc04 100644 --- a/config.go +++ b/config.go @@ -30,7 +30,7 @@ type AppConfig struct { // "info", "debug", "error", "panic". Default "info" LogLevel string `json:"log_level,omitempty"` // BackendConfig configures the email envelope processing backend - BackendConfig backends.BackendConfig `json:"backend_config"` + BackendConfig backends.BackendConfig `json:"backend,omitempty"` } // ServerConfig specifies config options for a single server @@ -60,6 +60,8 @@ type ServerConfig struct { // XClientOn when using a proxy such as Nginx, XCLIENT command is used to pass the // original client's IP address & client's HELO XClientOn bool `json:"xclient_on,omitempty"` + // Gateway specifies which backend to use + Gateway string `json:"backend,omitempty"` } type ServerTLSConfig struct { @@ -166,7 +168,7 @@ func (c *AppConfig) Load(jsonBytes []byte) error { if err = c.setDefaults(); err != nil { return err } - if err = c.setBackendDefaults(); err != nil { + if err = c.BackendConfig.ConfigureDefaults(); err != nil { return err } @@ -190,7 +192,7 @@ func (c *AppConfig) Load(jsonBytes []byte) error { func (c *AppConfig) EmitChangeEvents(oldConfig *AppConfig, app Guerrilla) { // has backend changed? if !reflect.DeepEqual((*c).BackendConfig, (*oldConfig).BackendConfig) { - app.Publish(EventConfigBackendConfig, c) + c.emitBackendChangeEvents(oldConfig, app) } // has config changed, general check if !reflect.DeepEqual(oldConfig, c) { @@ -234,13 +236,27 @@ func (c *AppConfig) EmitChangeEvents(oldConfig *AppConfig, app Guerrilla) { } // EmitLogReopen emits log reopen events using existing config -func (c *AppConfig) EmitLogReopenEvents(app Guerrilla) { +func (c *AppConfig) emitLogReopenEvents(app Guerrilla) { app.Publish(EventConfigLogReopen, c) for _, sc := range c.getServers() { app.Publish(EventConfigServerLogReopen, sc) } } +func (c *AppConfig) emitBackendChangeEvents(oldConfig *AppConfig, app Guerrilla) { + // check what's changed + changed, added, removed := c.BackendConfig.Changes(oldConfig.BackendConfig) + for b := range changed { + app.Publish(EventConfigBackendConfigChanged, c, b) + } + for b := range added { + app.Publish(EventConfigBackendConfigAdded, c, b) + } + for b := range removed { + app.Publish(EventConfigBackendConfigRemoved, c, b) + } +} + // gets the servers in a map (key by interface) for easy lookup func (c *AppConfig) getServers() map[string]*ServerConfig { servers := make(map[string]*ServerConfig, len(c.Servers)) @@ -288,6 +304,7 @@ func (c *AppConfig) setDefaults() error { sc.MaxClients = defaultMaxClients sc.Timeout = defaultTimeout sc.MaxSize = defaultMaxSize + sc.Gateway = backends.DefaultGateway c.Servers = append(c.Servers, sc) } else { // make sure each server has defaults correctly configured @@ -310,6 +327,9 @@ func (c *AppConfig) setDefaults() error { if c.Servers[i].LogFile == "" { c.Servers[i].LogFile = c.LogFile } + if c.Servers[i].Gateway == "" { + c.Servers[i].Gateway = backends.DefaultGateway + } // validate the server config err = c.Servers[i].Validate() if err != nil { @@ -320,44 +340,6 @@ func (c *AppConfig) setDefaults() error { return nil } -// setBackendDefaults sets default values for the backend config, -// if no backend config was added before starting, then use a default config -// otherwise, see what required values were missed in the config and add any missing with defaults -func (c *AppConfig) setBackendDefaults() error { - - if len(c.BackendConfig) == 0 { - h, err := os.Hostname() - if err != nil { - return err - } - c.BackendConfig = backends.BackendConfig{ - "log_received_mails": true, - "save_workers_size": 1, - "save_process": "HeadersParser|Header|Debugger", - "primary_mail_host": h, - } - } else { - if _, ok := c.BackendConfig["save_process"]; !ok { - c.BackendConfig["save_process"] = "HeadersParser|Header|Debugger" - } - if _, ok := c.BackendConfig["primary_mail_host"]; !ok { - h, err := os.Hostname() - if err != nil { - return err - } - c.BackendConfig["primary_mail_host"] = h - } - if _, ok := c.BackendConfig["save_workers_size"]; !ok { - c.BackendConfig["save_workers_size"] = 1 - } - - if _, ok := c.BackendConfig["log_received_mails"]; !ok { - c.BackendConfig["log_received_mails"] = false - } - } - return nil -} - // Emits any configuration change events on the server. // All events are fired and run synchronously func (sc *ServerConfig) emitChangeEvents(oldServer *ServerConfig, app Guerrilla) { @@ -402,6 +384,10 @@ func (sc *ServerConfig) emitChangeEvents(oldServer *ServerConfig, app Guerrilla) app.Publish(EventConfigServerMaxClients, sc) } + if _, ok := changes["Gateway"]; ok { + app.Publish(EventConfigServerGatewayConfig, sc) + } + if len(tlsChanges) > 0 { app.Publish(EventConfigServerTLSConfig, sc) } diff --git a/config_test.go b/config_test.go index bec6983b..c3c5385d 100644 --- a/config_test.go +++ b/config_test.go @@ -20,10 +20,19 @@ var configJsonA = ` "log_level" : "debug", "pid_file" : "tests/go-guerrilla.pid", "allowed_hosts": ["spam4.me","grr.la"], - "backend_config" : - { - "log_received_mails" : true - }, + "backend": { + "processors": { + "debugger": { + "log_received_mails": true + } + }, + "gateways" : { + "default" : { + "save_workers_size": 1, + "save_process": "HeadersParser|Header|Hasher|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -81,8 +90,8 @@ var configJsonA = ` "start_tls_on":false, "tls_always_on":false } - } - ] + } + ] } ` @@ -97,10 +106,19 @@ var configJsonB = ` "log_level" : "debug", "pid_file" : "tests/different-go-guerrilla.pid", "allowed_hosts": ["spam4.me","grr.la","newhost.com"], - "backend_config" : - { - "log_received_mails" : true - }, + "backend" : { + "processors" : { + "debugger": { + "log_received_mails" : true + } + }, + "gateways" : { + "default" : { + "save_workers_size": 1, + "save_process": "HeadersParser|Header|Hasher|Debugger" + } + } + }, "servers" : [ { "is_enabled" : true, @@ -230,12 +248,22 @@ func TestConfigChangeEvents(t *testing.T) { t.Error(err) } logger, _ := log.GetLogger(oldconf.LogFile, oldconf.LogLevel) - bcfg := backends.BackendConfig{"log_received_mails": true} - backend, err := backends.New(bcfg, logger) + + oldconf.BackendConfig = backends.BackendConfig{ + backends.ConfigProcessors: {"debugger": {"log_received_mails": true}}, + backends.ConfigGateways: { + "default": { + "save_process": "HeadersParser|Header|Hasher|Debugger", + }, + }, + } + + backend, err := backends.New("default", oldconf.BackendConfig, logger) if err != nil { - t.Error("cannot create backend", err) + t.Error("failed to create backend", err) + return } - app, err := New(oldconf, backend, logger) + app, err := New(oldconf, logger, backend) if err != nil { t.Error("cannot create daemon", err) } diff --git a/event.go b/event.go index 3c355b92..ffb2cd38 100644 --- a/event.go +++ b/event.go @@ -20,7 +20,11 @@ const ( // when log level changed EventConfigLogLevel // when the backend's config changed - EventConfigBackendConfig + EventConfigBackendConfigChanged + // when a gateway was added + EventConfigBackendConfigAdded + // when a gateway was removed + EventConfigBackendConfigRemoved // when a new server was added EventConfigServerNew // when an existing server was removed @@ -41,6 +45,8 @@ const ( EventConfigServerMaxClients // when a server's TLS config changed EventConfigServerTLSConfig + // when the server's backend config changed + EventConfigServerGatewayConfig ) var eventList = [...]string{ @@ -50,7 +56,9 @@ var eventList = [...]string{ "config_change:log_file", "config_change:reopen_log_file", "config_change:log_level", - "config_change:backend_config", + "backend_change:backend", + "backend_change:backend_config_added", + "backend_change:backend_removed", "server_change:new_server", "server_change:remove_server", "server_change:update_config", @@ -61,6 +69,7 @@ var eventList = [...]string{ "server_change:timeout", "server_change:max_clients", "server_change:tls_config", + "server_change:gateway", } func (e Event) String() string { diff --git a/goguerrilla.conf.sample b/goguerrilla.conf.sample index ee3d8d87..57ed80ec 100644 --- a/goguerrilla.conf.sample +++ b/goguerrilla.conf.sample @@ -14,8 +14,8 @@ "save_workers_size": 1, "save_process" : "HeadersParser|Header|Debugger", "primary_mail_host" : "mail.example.com", - "gw_save_timeout" : "30s", - "gw_val_rcpt_timeout" : "3s" + "save_timeout" : "30s", + "val_rcpt_timeout" : "3s" }, "servers" : [ { diff --git a/guerrilla.go b/guerrilla.go index d2df3411..92a882ef 100644 --- a/guerrilla.go +++ b/guerrilla.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "strings" "sync" "sync/atomic" @@ -52,19 +53,20 @@ type guerrilla struct { state int8 EventHandler logStore - backendStore + + beGuard sync.Mutex + backends BackendContainer } type logStore struct { atomic.Value } -type backendStore struct { - atomic.Value -} +type BackendContainer map[string]backends.Backend type daemonEvent func(c *AppConfig) type serverEvent func(sc *ServerConfig) +type backendEvent func(c *AppConfig, gateway string) // Get loads the log.logger in an atomic operation. Returns a stderr logger if not able to load func (ls *logStore) mainlog() log.Logger { @@ -80,13 +82,48 @@ func (ls *logStore) setMainlog(log log.Logger) { ls.Store(log) } +// makeConfiguredBackends makes backends from the config +func (g *guerrilla) makeConfiguredBackends(l log.Logger) ([]backends.Backend, error) { + var list []backends.Backend + config := g.Config.BackendConfig[backends.ConfigGateways] + if len(config) == 0 { + return list, errors.New("no backends configured") + } + list = make([]backends.Backend, 0) + for name := range config { + if b, err := backends.New(name, g.Config.BackendConfig, l); err != nil { + return nil, err + } else { + list = append(list, b) + } + } + return list, nil +} + +// New creates a new Guerrilla instance configured with backends and a logger // Returns a new instance of Guerrilla with the given config, not yet running. Backend started. -func New(ac *AppConfig, b backends.Backend, l log.Logger) (Guerrilla, error) { +// b can be nil. If nil. then it will use the config to make the backends +func New(ac *AppConfig, l log.Logger, b ...backends.Backend) (Guerrilla, error) { g := &guerrilla{ Config: *ac, // take a local copy servers: make(map[string]*server, len(ac.Servers)), } - g.backendStore.Store(b) + if 0 == len(b) { + var err error + b, err = g.makeConfiguredBackends(l) + if err != nil { + return g, err + } + } + if g.backends == nil { + g.backends = make(BackendContainer) + } + for i := range b { + if b[i] == nil { + return g, errors.New("cannot use a nil backend") + } + g.storeBackend(b[i]) + } g.setMainlog(l) if ac.LogLevel != "" { @@ -106,8 +143,11 @@ func New(ac *AppConfig, b backends.Backend, l log.Logger) (Guerrilla, error) { return g, err } - // start backend for processing email - err = g.backend().Start() + // start backends for processing email + _, err = g.mapBackends(func(b backends.Backend) error { + return b.Start() + }) + if err != nil { return g, err } @@ -122,20 +162,22 @@ func New(ac *AppConfig, b backends.Backend, l log.Logger) (Guerrilla, error) { func (g *guerrilla) makeServers() error { g.mainlog().Debug("making servers") var errs Errors - for _, sc := range g.Config.Servers { + for serverID, sc := range g.Config.Servers { if _, ok := g.servers[sc.ListenInterface]; ok { // server already instantiated continue } if err := sc.Validate(); err != nil { - g.mainlog().WithError(errs).Errorf("Failed to create server [%s]", sc.ListenInterface) + g.mainlog().Fields("error", errs, "iface", sc.ListenInterface). + Error("failed to create server") errs = append(errs, err) continue } else { sc := sc // pin! - server, err := newServer(&sc, g.backend(), g.mainlog()) + server, err := newServer(&sc, g.backend(sc.Gateway), g.mainlog(), serverID) if err != nil { - g.mainlog().WithError(err).Errorf("Failed to create server [%s]", sc.ListenInterface) + g.mainlog().Fields("error", err, "iface", sc.ListenInterface). + Error("failed to create server") errs = append(errs, err) } if server != nil { @@ -197,6 +239,31 @@ func (g *guerrilla) mapServers(callback func(*server)) map[string]*server { return g.servers } +type mapBackendErrors []error + +func (e mapBackendErrors) Error() string { + data := make([]string, len(e)) + for i, s := range e { + data[i] = fmt.Sprint(s) + } + return strings.Join(data, ",") +} + +func (g *guerrilla) mapBackends(callback func(backend backends.Backend) error) (BackendContainer, error) { + defer g.beGuard.Unlock() + g.beGuard.Lock() + var e mapBackendErrors + for name := range g.backends { + if err := callback(g.backends[name]); err != nil { + e = append(e, err) + } + } + if len(e) == 0 { + return g.backends, nil + } + return g.backends, e +} + // subscribeEvents subscribes event handlers for configuration change events func (g *guerrilla) subscribeEvents() { @@ -209,8 +276,9 @@ func (g *guerrilla) subscribeEvents() { events[EventConfigAllowedHosts] = daemonEvent(func(c *AppConfig) { g.mapServers(func(server *server) { server.setAllowedHosts(c.AllowedHosts) + g.mainlog().Fields("serverID", server.serverID, "event", EventConfigAllowedHosts). + Info("allowed_hosts config changed, a new list was set") }) - g.mainlog().Infof("allowed_hosts config changed, a new list was set") }) // the main log file changed @@ -223,9 +291,11 @@ func (g *guerrilla) subscribeEvents() { // it will change server's logger when the next client gets accepted server.mainlogStore.Store(l) }) - g.mainlog().Infof("main log for new clients changed to [%s]", c.LogFile) + g.mainlog().Fields("file", c.LogFile). + Info("main log for new clients changed") } else { - g.mainlog().WithError(err).Errorf("main logging change failed [%s]", c.LogFile) + g.mainlog().Fields("error", err, "file", c.LogFile). + Error("main logging change failed") } }) @@ -234,10 +304,11 @@ func (g *guerrilla) subscribeEvents() { events[EventConfigLogReopen] = daemonEvent(func(c *AppConfig) { err := g.mainlog().Reopen() if err != nil { - g.mainlog().WithError(err).Errorf("main log file [%s] failed to re-open", c.LogFile) + g.mainlog().Fields("error", err, "file", c.LogFile). + Error("main log file failed to re-open") return } - g.mainlog().Infof("re-opened main log file [%s]", c.LogFile) + g.mainlog().Fields("file", c.LogFile).Info("re-opened main log file") }) // when log level changes, apply to mainlog and server logs @@ -248,7 +319,7 @@ func (g *guerrilla) subscribeEvents() { g.mapServers(func(server *server) { server.logStore.Store(l) }) - g.mainlog().Infof("log level changed to [%s]", c.LogLevel) + g.mainlog().Fields("level", c.LogLevel).Info("log level changed") } }) @@ -260,39 +331,51 @@ func (g *guerrilla) subscribeEvents() { // server config was updated events[EventConfigServerConfig] = serverEvent(func(sc *ServerConfig) { g.setServerConfig(sc) - g.mainlog().Infof("server %s config change event, a new config has been saved", sc.ListenInterface) + g.mainlog().Fields("iface", sc.ListenInterface). + Info("server config change event, a new config has been saved") }) // add a new server to the config & start events[EventConfigServerNew] = serverEvent(func(sc *ServerConfig) { - g.mainlog().Debugf("event fired [%s] %s", EventConfigServerNew, sc.ListenInterface) + values := []interface{}{"iface", sc.ListenInterface, "event", EventConfigServerNew} + g.mainlog().Fields(values...). + Debug("event fired") if _, err := g.findServer(sc.ListenInterface); err != nil { + // not found, lets add it - // if err := g.makeServers(); err != nil { - g.mainlog().WithError(err).Errorf("cannot add server [%s]", sc.ListenInterface) + g.mainlog().Fields(append(values, "error", err)...). + Error("cannot add server") return } - g.mainlog().Infof("New server added [%s]", sc.ListenInterface) + g.mainlog().Fields(values...).Info("new server added") if g.state == daemonStateStarted { err := g.Start() if err != nil { - g.mainlog().WithError(err).Info("Event server_change:new_server returned errors when starting") + g.mainlog().Fields(append(values, "error", err)...). + Error("new server errors when starting") } } } else { - g.mainlog().Debugf("new event, but server already fund") + g.mainlog().Fields(values...). + Debug("new event, but server already fund") } }) // start a server that already exists in the config and has been enabled events[EventConfigServerStart] = serverEvent(func(sc *ServerConfig) { if server, err := g.findServer(sc.ListenInterface); err == nil { + fields := []interface{}{ + "iface", server.listenInterface, + "serverID", server.serverID, + "event", EventConfigServerStart} if server.state == ServerStateStopped || server.state == ServerStateNew { - g.mainlog().Infof("Starting server [%s]", server.listenInterface) + g.mainlog().Fields(fields...). + Info("starting server") err := g.Start() if err != nil { - g.mainlog().WithError(err).Info("Event server_change:start_server returned errors when starting") + g.mainlog().Fields(append(fields, "error", err)...). + Info("event server_change:start_server returned errors when starting") } } } @@ -303,7 +386,11 @@ func (g *guerrilla) subscribeEvents() { if server, err := g.findServer(sc.ListenInterface); err == nil { if server.state == ServerStateRunning { server.Shutdown() - g.mainlog().Infof("Server [%s] stopped.", sc.ListenInterface) + g.mainlog().Fields( + "event", EventConfigServerStop, + "server", sc.ListenInterface, + "serverID", server.serverID). + Info("server stopped.") } } }) @@ -313,24 +400,40 @@ func (g *guerrilla) subscribeEvents() { if server, err := g.findServer(sc.ListenInterface); err == nil { server.Shutdown() g.removeServer(sc.ListenInterface) - g.mainlog().Infof("Server [%s] removed from config, stopped it.", sc.ListenInterface) + g.mainlog().Fields( + "event", EventConfigServerRemove, + "server", sc.ListenInterface, + "serverID", server.serverID). + Info("server removed from config, stopped it") } }) // TLS changes events[EventConfigServerTLSConfig] = serverEvent(func(sc *ServerConfig) { if server, err := g.findServer(sc.ListenInterface); err == nil { + fields := []interface{}{ + "iface", server.listenInterface, + "serverID", server.serverID, + "event", EventConfigServerTLSConfig} if err := server.configureTLS(); err == nil { - g.mainlog().Infof("Server [%s] new TLS configuration loaded", sc.ListenInterface) + g.mainlog().Fields(fields...).Info("server new TLS configuration loaded") } else { - g.mainlog().WithError(err).Errorf("Server [%s] failed to load the new TLS configuration", sc.ListenInterface) + g.mainlog().Fields(append(fields, "error", err)...). + Error("Server failed to load the new TLS configuration") } } }) // when server's timeout change. events[EventConfigServerTimeout] = serverEvent(func(sc *ServerConfig) { g.mapServers(func(server *server) { + fields := []interface{}{ + "iface", server.listenInterface, + "serverID", server.serverID, + "event", EventConfigServerTimeout, + "timeout", sc.Timeout, + } server.setTimeout(sc.Timeout) + g.mainlog().Fields(fields...).Info("server timeout set") }) }) // when server's max clients change. @@ -345,93 +448,179 @@ func (g *guerrilla) subscribeEvents() { var err error var l log.Logger level := g.mainlog().GetLevel() + fields := []interface{}{ + "iface", server.listenInterface, + "serverID", server.serverID, + "event", EventConfigServerLogFile, + "file", sc.LogFile, + } if l, err = log.GetLogger(sc.LogFile, level); err == nil { g.setMainlog(l) backends.Svc.SetMainlog(l) // it will change to the new logger on the next accepted client server.logStore.Store(l) - g.mainlog().Infof("Server [%s] changed, new clients will log to: [%s]", + g.mainlog().Fields(fields...).Info("server log changed", sc.ListenInterface, sc.LogFile, ) } else { - g.mainlog().WithError(err).Errorf( - "Server [%s] log change failed to: [%s]", - sc.ListenInterface, - sc.LogFile, - ) + g.mainlog().Fields(append(fields, "error", err)...).Error( + "server log change failed") } } }) // when the daemon caught a sighup, event for individual server events[EventConfigServerLogReopen] = serverEvent(func(sc *ServerConfig) { if server, err := g.findServer(sc.ListenInterface); err == nil { + fields := []interface{}{"file", sc.LogFile, + "iface", sc.ListenInterface, + "serverID", server.serverID, + "file", sc.LogFile, + "event", EventConfigServerLogReopen} if err = server.log().Reopen(); err != nil { - g.mainlog().WithError(err).Errorf("server [%s] log file [%s] failed to re-open", sc.ListenInterface, sc.LogFile) + g.mainlog().Fields( + append(fields, "error", err)...). + Error("server log file failed to re-open") return } - g.mainlog().Infof("Server [%s] re-opened log file [%s]", sc.ListenInterface, sc.LogFile) + g.mainlog().Fields(fields).Info("server re-opened log file") } }) - // when the backend changes - events[EventConfigBackendConfig] = daemonEvent(func(appConfig *AppConfig) { - logger, _ := log.GetLogger(appConfig.LogFile, appConfig.LogLevel) - // shutdown the backend first. - var err error - if err = g.backend().Shutdown(); err != nil { - logger.WithError(err).Warn("Backend failed to shutdown") + + // when the server's gateway setting changed + events[EventConfigServerGatewayConfig] = serverEvent(func(sc *ServerConfig) { + b := g.backend(sc.Gateway) + if b == nil { + g.mainlog().Fields("gateway", sc.Gateway, "event", EventConfigServerGatewayConfig). + Error("could not change to gateway, not configured") return } - // init a new backend, Revert to old backend config if it fails - if newBackend, newErr := backends.New(appConfig.BackendConfig, logger); newErr != nil { - logger.WithError(newErr).Error("Error while loading the backend") - err = g.backend().Reinitialize() + g.storeBackend(b) + }) + + revertIfError := func(err error, name string, logger log.Logger, g *guerrilla) { + if err != nil { + logger.Fields("error", err, "gateway", name, "event", EventConfigServerGatewayConfig). + Error("cannot change gateway config, reverting to old config") + err = g.backend(name).Reinitialize() if err != nil { - logger.WithError(err).Fatal("failed to revert to old backend config") + logger.Fields("error", err, "gateway", name, "event", EventConfigServerGatewayConfig). + Error("failed to revert to old gateway config") return } - err = g.backend().Start() + err = g.backend(name).Start() if err != nil { - logger.WithError(err).Fatal("failed to start backend with old config") + logger.Fields("error", err, "gateway", name, "event", EventConfigServerGatewayConfig). + Error("failed to start gateway with old config") + return + } + logger.Fields("gateway", name, "event", EventConfigServerGatewayConfig). + Info("reverted to old gateway config") + } + } + + events[EventConfigBackendConfigChanged] = backendEvent(func(appConfig *AppConfig, name string) { + logger, _ := log.GetLogger(appConfig.LogFile, appConfig.LogLevel) + var err error + // shutdown the backend first. + if err = g.backend(name).Shutdown(); err != nil { + logger.Fields("error", err, "gateway", name, "event", EventConfigBackendConfigChanged). + Error("gateway failed to shutdown") + return // we can't do anything then + } + if newBackend, newErr := backends.New(name, appConfig.BackendConfig, logger); newErr != nil { + err = newErr + revertIfError(newErr, name, logger, g) // revert to old backend + return + } else { + if err = newBackend.Start(); err != nil { + logger.Fields("error", err, "gateway", name, "event", EventConfigBackendConfigChanged). + Error("gateway could not start") + revertIfError(err, name, logger, g) // revert to old backend return + } else { + logger.Fields("gateway", name, "event", EventConfigBackendConfigChanged). + Info("gateway with new config started") + g.storeBackend(newBackend) } - logger.Info("reverted to old backend config") + } + }) + + // a new gateway was added + events[EventConfigBackendConfigAdded] = backendEvent(func(appConfig *AppConfig, name string) { + logger, _ := log.GetLogger(appConfig.LogFile, appConfig.LogLevel) + // shutdown any old backend first. + if newBackend, newErr := backends.New(name, appConfig.BackendConfig, logger); newErr != nil { + logger.Fields("error", newErr, "gateway", name, "event", EventConfigBackendConfigAdded). + Error("cannot add new gateway") } else { - // swap to the bew backend (assuming old backend was shutdown so it can be safely swapped) + // swap to the bew gateway (assuming old gateway was shutdown so it can be safely swapped) if err := newBackend.Start(); err != nil { - logger.WithError(err).Error("backend could not start") + logger.Fields("error", err, "gateway", name, "event", EventConfigBackendConfigAdded). + Error("cannot start new gateway") } - logger.Info("new backend started") + logger.Fields("gateway", name).Info("new gateway started") g.storeBackend(newBackend) } }) + + // remove a gateway (shut it down) + events[EventConfigBackendConfigRemoved] = backendEvent(func(appConfig *AppConfig, name string) { + logger, _ := log.GetLogger(appConfig.LogFile, appConfig.LogLevel) + // shutdown the backend first. + var err error + // revert + defer revertIfError(err, name, logger, g) + if err = g.backend(name).Shutdown(); err != nil { + logger.Fields("error", err, "gateway", name, "event", EventConfigBackendConfigRemoved). + Error("gateway failed to shutdown") + return + } + g.removeBackend(g.backend(name)) + logger.Fields("gateway", name, "event", EventConfigBackendConfigRemoved).Info("gateway removed") + }) + + // subscribe all of the above events var err error for topic, fn := range events { - switch f := fn.(type) { - case daemonEvent: - err = g.Subscribe(topic, f) - case serverEvent: - err = g.Subscribe(topic, f) - } + err = g.Subscribe(topic, fn) if err != nil { - g.mainlog().WithError(err).Errorf("failed to subscribe on topic [%s]", topic) + g.mainlog().Fields("error", err, "event", topic). + Error("failed to subscribe on topic") break } } +} + +func (g *guerrilla) removeBackend(b backends.Backend) { + g.beGuard.Lock() + defer g.beGuard.Unlock() + delete(g.backends, b.Name()) } func (g *guerrilla) storeBackend(b backends.Backend) { - g.backendStore.Store(b) + g.beGuard.Lock() + defer g.beGuard.Unlock() + g.backends[b.Name()] = b g.mapServers(func(server *server) { - server.setBackend(b) + sc := server.configStore.Load().(ServerConfig) + if b.Name() == sc.Gateway { + server.setBackend(b) + } }) } -func (g *guerrilla) backend() backends.Backend { - if b, ok := g.backendStore.Load().(backends.Backend); ok { +func (g *guerrilla) backend(name string) backends.Backend { + g.beGuard.Lock() + defer g.beGuard.Unlock() + if b, ok := g.backends[name]; ok { return b } + // if not found, return a random one + for b := range g.backends { + return g.backends[b] + } return nil } @@ -448,10 +637,12 @@ func (g *guerrilla) Start() error { } if g.state == daemonStateStopped { // when a backend is shutdown, we need to re-initialize before it can be started again - if err := g.backend().Reinitialize(); err != nil { - startErrors = append(startErrors, err) - } - if err := g.backend().Start(); err != nil { + if _, err := g.mapBackends(func(b backends.Backend) error { + if err := b.Reinitialize(); err != nil { + return err + } + return b.Start() + }); err != nil { startErrors = append(startErrors, err) } } @@ -471,7 +662,8 @@ func (g *guerrilla) Start() error { } startWG.Add(1) go func(s *server) { - g.mainlog().Infof("Starting: %s", s.listenInterface) + g.mainlog().Fields("iface", s.listenInterface, "serverID", s.serverID). + Info("starting server") if err := s.Start(&startWG); err != nil { errs <- err } @@ -499,7 +691,7 @@ func (g *guerrilla) Shutdown() { g.mapServers(func(s *server) { if s.state == ServerStateRunning { s.Shutdown() - g.mainlog().Infof("shutdown completed for [%s]", s.listenInterface) + g.mainlog().Fields("iface", s.listenInterface, "serverID", s.serverID).Info("shutdown completed") } }) @@ -508,10 +700,14 @@ func (g *guerrilla) Shutdown() { g.state = daemonStateStopped defer g.guard.Unlock() }() - if err := g.backend().Shutdown(); err != nil { - g.mainlog().WithError(err).Warn("Backend failed to shutdown") + + if _, err := g.mapBackends(func(b backends.Backend) error { + return b.Shutdown() + }); err != nil { + fmt.Println(err) + g.mainlog().Fields("error", err).Error("backend failed to shutdown") } else { - g.mainlog().Infof("Backend shutdown completed") + g.mainlog().Info("backend shutdown completed") } } @@ -532,7 +728,7 @@ func (g *guerrilla) writePid() (err error) { } } if err != nil { - g.mainlog().WithError(err).Errorf("error while writing pidFile (%s)", g.Config.PidFile) + g.mainlog().Fields("error", err, "file", g.Config.PidFile).Error("error while writing pidFile") } }() if len(g.Config.PidFile) > 0 { @@ -546,7 +742,7 @@ func (g *guerrilla) writePid() (err error) { if err = f.Sync(); err != nil { return err } - g.mainlog().Infof("pid_file (%s) written with pid:%v", g.Config.PidFile, pid) + g.mainlog().Fields("file", g.Config.PidFile, "pid", pid).Info("pid_file written") } return nil } diff --git a/log/log.go b/log/log.go index 79146b25..d1ae8ffa 100644 --- a/log/log.go +++ b/log/log.go @@ -1,7 +1,8 @@ package log import ( - log "github.com/sirupsen/logrus" + "fmt" + loglib "github.com/sirupsen/logrus" "io" "io/ioutil" "net" @@ -52,14 +53,15 @@ func (level Level) String() string { } type Logger interface { - log.FieldLogger - WithConn(conn net.Conn) *log.Entry + loglib.FieldLogger + WithConn(conn net.Conn) *loglib.Entry Reopen() error GetLogDest() string SetLevel(level string) GetLevel() string IsDebug() bool - AddHook(h log.Hook) + AddHook(h loglib.Hook) + Fields(fields ...interface{}) *loglib.Entry } // Implements the Logger interface @@ -67,7 +69,7 @@ type Logger interface { type HookedLogger struct { // satisfy the log.FieldLogger interface - *log.Logger + *loglib.Logger h LoggerHook @@ -143,8 +145,8 @@ func GetLogger(dest string, level string) (Logger, error) { return l, nil } -func newLogrus(o OutputOption, level string) (*log.Logger, error) { - logLevel, err := log.ParseLevel(level) +func newLogrus(o OutputOption, level string) (*loglib.Logger, error) { + logLevel, err := loglib.ParseLevel(level) if err != nil { return nil, err } @@ -163,10 +165,10 @@ func newLogrus(o OutputOption, level string) (*log.Logger, error) { out = ioutil.Discard } - logger := &log.Logger{ + logger := &loglib.Logger{ Out: out, - Formatter: new(log.TextFormatter), - Hooks: make(log.LevelHooks), + Formatter: new(loglib.TextFormatter), + Hooks: make(loglib.LevelHooks), Level: logLevel, } @@ -174,22 +176,22 @@ func newLogrus(o OutputOption, level string) (*log.Logger, error) { } // AddHook adds a new logrus hook -func (l *HookedLogger) AddHook(h log.Hook) { - log.AddHook(h) +func (l *HookedLogger) AddHook(h loglib.Hook) { + loglib.AddHook(h) } func (l *HookedLogger) IsDebug() bool { - return l.GetLevel() == log.DebugLevel.String() + return l.GetLevel() == loglib.DebugLevel.String() } // SetLevel sets a log level, one of the LogLevels func (l *HookedLogger) SetLevel(level string) { - var logLevel log.Level + var logLevel loglib.Level var err error - if logLevel, err = log.ParseLevel(level); err != nil { + if logLevel, err = loglib.ParseLevel(level); err != nil { return } - log.SetLevel(logLevel) + loglib.SetLevel(logLevel) } // GetLevel gets the current log level @@ -211,10 +213,33 @@ func (l *HookedLogger) GetLogDest() string { } // WithConn extends logrus to be able to log with a net.Conn -func (l *HookedLogger) WithConn(conn net.Conn) *log.Entry { +func (l *HookedLogger) WithConn(conn net.Conn) *loglib.Entry { var addr = "unknown" if conn != nil { addr = conn.RemoteAddr().String() } return l.WithField("addr", addr) } + +// Fields accepts an even number of arguments in the format of ([ )1* +func (l *HookedLogger) Fields(spec ...interface{}) *loglib.Entry { + size := len(spec) + if size < 2 || size%2 != 0 { + return l.WithField("oops", "wrong fields specified") + } + fields := make(map[string]interface{}, size/2) + for i := range spec { + if i%2 != 0 { + continue + } + if key, ok := spec[i].(string); ok { + fields[key] = spec[i+1] + } else if key, ok := spec[i].(fmt.Stringer); ok { + fields[key.String()] = spec[i+1] + } else { + fields[fmt.Sprintf("%d", i)] = spec[i+1] + } + + } + return l.WithFields(fields) +} diff --git a/mail/envelope.go b/mail/envelope.go index 30e1da78..b077125c 100644 --- a/mail/envelope.go +++ b/mail/envelope.go @@ -1,11 +1,13 @@ package mail import ( - "bufio" "bytes" - "crypto/md5" + "encoding/binary" + "encoding/hex" "errors" "fmt" + "hash" + "hash/fnv" "io" "mime" "net" @@ -14,7 +16,8 @@ import ( "sync" "time" - "github.com/flashmob/go-guerrilla/mail/rfc5321" + "github.com/flashmob/go-guerrilla/mail/mimeparse" + "github.com/flashmob/go-guerrilla/mail/smtp" ) // A WordDecoder decodes MIME headers containing RFC 2047 encoded-words. @@ -27,10 +30,10 @@ var Dec mime.WordDecoder func init() { // use the default decoder, without Gnu inconv. Import the mail/inconv package to use iconv. Dec = mime.WordDecoder{} + // for QueuedID generation + hasher.h = fnv.New128a() } -const maxHeaderChunk = 1 + (4 << 10) // 4KB - // Address encodes an email address of the form `` type Address struct { // User is local part @@ -40,7 +43,7 @@ type Address struct { // ADL is at-domain list if matched ADL []string // PathParams contains any ESTMP parameters that were matched - PathParams [][]string + PathParams []smtp.PathParam // NullPath is true if <> was received NullPath bool // Quoted indicates if the local-part needs quotes @@ -99,7 +102,8 @@ func (a *Address) IsPostmaster() bool { // NewAddress takes a string of an RFC 5322 address of the // form "Gogh Fir " or "foo@example.com". func NewAddress(str string) (*Address, error) { - var ap rfc5321.RFC5322 + + var ap smtp.RFC5322 l, err := ap.Address([]byte(str)) if err != nil { return nil, err @@ -119,48 +123,99 @@ func NewAddress(str string) (*Address, error) { return a, nil } +type Hash128 [16]byte + +func (h Hash128) String() string { + return fmt.Sprintf("%x", h[:]) +} + +// FromHex converts the, string must be 32 bytes +func (h *Hash128) FromHex(s string) { + if len(s) != 32 { + panic("hex string must be 32 bytes") + } + _, _ = hex.Decode(h[:], []byte(s)) +} + +// Bytes returns the raw bytes +func (h Hash128) Bytes() []byte { return h[:] } + // Envelope of Email represents a single SMTP message. type Envelope struct { - // Remote IP address - RemoteIP string - // Message sent in EHLO command - Helo string - // Sender - MailFrom Address - // Recipients - RcptTo []Address - // Data stores the header and message body + // Data stores the header and message body (when using the non-streaming processor) Data bytes.Buffer // Subject stores the subject of the email, extracted and decoded after calling ParseHeaders() Subject string - // TLS is true if the email was received using a TLS connection - TLS bool // Header stores the results from ParseHeaders() Header textproto.MIMEHeader // Values hold the values generated when processing the envelope by the backend Values map[string]interface{} // Hashes of each email on the rcpt Hashes []string - // additional delivery header that may be added + // DeliveryHeader stores additional delivery header that may be added (used by non-streaming processor) DeliveryHeader string + // Size is the length of message, after being written + Size int64 + // MimeParts contain the information about the mime-parts after they have been parsed + MimeParts *mimeparse.Parts + // MimeError contains any error encountered when parsing mime using the mimeanalyzer + MimeError error + // MessageID contains the id of the message after it has been written + MessageID uint64 + // Remote IP address + RemoteIP string + // Message sent in EHLO command + Helo string + // Sender + MailFrom Address + // Recipients + RcptTo []Address + // TLS is true if the email was received using a TLS connection + TLS bool // Email(s) will be queued with this id - QueuedId string + QueuedId Hash128 + // TransportType indicates whenever 8BITMIME extension has been signaled + TransportType smtp.TransportType // ESMTP: true if EHLO was used ESMTP bool + // ServerID records the server's index in the configuration + ServerID int + // When locked, it means that the envelope is being processed by the backend + sync.WaitGroup +} + +type queuedIDGenerator struct { + h hash.Hash + n [24]byte sync.Mutex } -func NewEnvelope(remoteAddr string, clientID uint64) *Envelope { +var hasher queuedIDGenerator + +func NewEnvelope(remoteAddr string, clientID uint64, serverID int) *Envelope { return &Envelope{ RemoteIP: remoteAddr, Values: make(map[string]interface{}), - QueuedId: queuedID(clientID), + ServerID: serverID, + QueuedId: QueuedID(clientID, serverID), } } -func queuedID(clientID uint64) string { - return fmt.Sprintf("%x", md5.Sum([]byte(string(time.Now().Unix())+string(clientID)))) +func QueuedID(clientID uint64, serverID int) Hash128 { + hasher.Lock() + defer func() { + hasher.h.Reset() + hasher.Unlock() + }() + h := Hash128{} + // pack the seeds and hash'em + binary.BigEndian.PutUint64(hasher.n[0:8], uint64(time.Now().UnixNano())) + binary.BigEndian.PutUint64(hasher.n[8:16], clientID) + binary.BigEndian.PutUint64(hasher.n[16:24], uint64(serverID)) + hasher.h.Write(hasher.n[:]) + copy(h[:], hasher.h.Sum([]byte{})) + return h } // ParseHeaders parses the headers into Header field of the Envelope struct. @@ -168,31 +223,17 @@ func queuedID(clientID uint64) string { // It assumes that at most 30kb of email data can be a header // Decoding of encoding to UTF is only done on the Subject, where the result is assigned to the Subject field func (e *Envelope) ParseHeaders() error { - var err error - if e.Header != nil { - return errors.New("headers already parsed") + if e.Header == nil { + return errors.New("headers not parsed") } - buf := e.Data.Bytes() - // find where the header ends, assuming that over 30 kb would be max - if len(buf) > maxHeaderChunk { - buf = buf[:maxHeaderChunk] + if len(e.Header) == 0 { + return errors.New("header not found") } - - headerEnd := bytes.Index(buf, []byte{'\n', '\n'}) // the first two new-lines chars are the End Of Header - if headerEnd > -1 { - header := buf[0 : headerEnd+2] - headerReader := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(header))) - e.Header, err = headerReader.ReadMIMEHeader() - if err == nil || err == io.EOF { - // decode the subject - if subject, ok := e.Header["Subject"]; ok { - e.Subject = MimeHeaderDecode(subject[0]) - } - } - } else { - err = errors.New("header not found") + // decode the subject + if subject, ok := e.Header["Subject"]; ok { + e.Subject = MimeHeaderDecode(subject[0]) } - return err + return nil } // Len returns the number of bytes that would be in the reader returned by NewReader() @@ -218,9 +259,7 @@ func (e *Envelope) String() string { func (e *Envelope) ResetTransaction() { // ensure not processing by the backend, will only get lock if finished, otherwise block - e.Lock() - // got the lock, it means processing finished - e.Unlock() + e.Wait() e.MailFrom = Address{} e.RcptTo = []Address{} @@ -232,13 +271,18 @@ func (e *Envelope) ResetTransaction() { e.Header = nil e.Hashes = make([]string, 0) e.DeliveryHeader = "" + e.Size = 0 + e.MessageID = 0 + e.MimeParts = nil + e.MimeError = nil e.Values = make(map[string]interface{}) } // Reseed is called when used with a new connection, once it's accepted -func (e *Envelope) Reseed(remoteIP string, clientID uint64) { +func (e *Envelope) Reseed(remoteIP string, clientID uint64, serverID int) { e.RemoteIP = remoteIP - e.QueuedId = queuedID(clientID) + e.ServerID = serverID + e.QueuedId = QueuedID(clientID, serverID) e.Helo = "" e.TLS = false e.ESMTP = false @@ -256,10 +300,68 @@ func (e *Envelope) PopRcpt() Address { return ret } +func (e *Envelope) Protocol() Protocol { + protocol := ProtocolSMTP + switch { + case !e.ESMTP && !e.TLS: + protocol = ProtocolSMTP + case !e.ESMTP && e.TLS: + protocol = ProtocolSMTPS + case e.ESMTP && !e.TLS: + protocol = ProtocolESMTP + case e.ESMTP && e.TLS: + protocol = ProtocolESMTPS + } + return protocol +} + +type Protocol int + +const ( + ProtocolSMTP Protocol = iota + ProtocolSMTPS + ProtocolESMTP + ProtocolESMTPS + ProtocolLTPS + ProtocolUnknown +) + +func (p Protocol) String() string { + switch p { + case ProtocolSMTP: + return "SMTP" + case ProtocolSMTPS: + return "SMTPS" + case ProtocolESMTP: + return "ESMTP" + case ProtocolESMTPS: + return "ESMTPS" + case ProtocolLTPS: + return "LTPS" + } + return "unknown" +} + +func ParseProtocolType(str string) Protocol { + switch { + case str == "SMTP": + return ProtocolSMTP + case str == "SMTPS": + return ProtocolSMTPS + case str == "ESMTP": + return ProtocolESMTP + case str == "ESMTPS": + return ProtocolESMTPS + case str == "LTPS": + return ProtocolLTPS + } + + return ProtocolUnknown +} + const ( statePlainText = iota stateStartEncodedWord - stateEncodedWord stateEncoding stateCharset statePayload @@ -432,14 +534,14 @@ func NewPool(poolSize int) *Pool { } } -func (p *Pool) Borrow(remoteAddr string, clientID uint64) *Envelope { +func (p *Pool) Borrow(remoteAddr string, clientID uint64, serverID int) *Envelope { var e *Envelope p.sem <- true // block the envelope until more room select { case e = <-p.pool: - e.Reseed(remoteAddr, clientID) + e.Reseed(remoteAddr, clientID, serverID) default: - e = NewEnvelope(remoteAddr, clientID) + e = NewEnvelope(remoteAddr, clientID, serverID) } return e } @@ -456,3 +558,22 @@ func (p *Pool) Return(e *Envelope) { // take a value off the semaphore to make room for more envelopes <-p.sem } + +const MostCommonCharset = "ISO-8859-1" + +var supportedEncodingsCharsets map[string]bool + +func SupportsCharset(charset string) bool { + if supportedEncodingsCharsets == nil { + supportedEncodingsCharsets = make(map[string]bool) + } else if ok, result := supportedEncodingsCharsets[charset]; ok { + return result + } + _, err := Dec.CharsetReader(charset, bytes.NewReader([]byte{})) + if err != nil { + supportedEncodingsCharsets[charset] = false + return false + } + supportedEncodingsCharsets[charset] = true + return true +} diff --git a/mail/envelope_test.go b/mail/envelope_test.go index 06f0246f..e077ded1 100644 --- a/mail/envelope_test.go +++ b/mail/envelope_test.go @@ -1,6 +1,8 @@ package mail import ( + "bufio" + "bytes" "io" "io/ioutil" "strings" @@ -95,9 +97,9 @@ func TestAddressWithIP(t *testing.T) { } func TestEnvelope(t *testing.T) { - e := NewEnvelope("127.0.0.1", 22) + e := NewEnvelope("127.0.0.1", 22, 0) - e.QueuedId = "abc123" + e.QueuedId = QueuedID(2, 33) e.Helo = "helo.example.com" e.MailFrom = Address{User: "test", Host: "example.com"} e.TLS = true @@ -107,7 +109,18 @@ func TestEnvelope(t *testing.T) { if to.String() != "test@example.com" { t.Error("to does not equal test@example.com, it was:", to.String()) } - e.Data.WriteString("Subject: Test\n\nThis is a test nbnb nbnb hgghgh nnnbnb nbnbnb nbnbn.") + // we feed the input through the NewMineDotReader, it will parse the headers while reading the input + // the input has a single line header and ends with a line with a single . + in := "Subject: =?utf-8?B?55So5oi34oCcRXBpZGVtaW9sb2d5IGluIG51cnNpbmcgYW5kIGg=?=\n\nThis is a test nbnb nbnb hgghgh nnnbnb nbnbnb nbnbn.\n.\n" + mdr := NewMimeDotReader(bufio.NewReader(bytes.NewBufferString(in)), 1) + i, err := io.Copy(&e.Data, mdr) + if err != nil && err != io.EOF { + t.Error(err, "cannot copy buffer", i, err) + } + // pass the parsed headers to the envelope + if p := mdr.Parts(); p != nil && len(p) > 0 { + e.Header = p[0].Headers + } addHead := "Delivered-To: " + to.String() + "\n" addHead += "Received: from " + e.Helo + " (" + e.Helo + " [" + e.RemoteIP + "])\n" @@ -119,12 +132,13 @@ func TestEnvelope(t *testing.T) { if len(data) != e.Len() { t.Error("e.Len() is incorrect, it shown ", e.Len(), " but we wanted ", len(data)) } + if err := e.ParseHeaders(); err != nil && err != io.EOF { t.Error("cannot parse headers:", err) return } - if e.Subject != "Test" { - t.Error("Subject expecting: Test, got:", e.Subject) + if e.Subject != "用户“Epidemiology in nursing and h" { + t.Error("Subject expecting: 用户“Epidemiology in nursing and h, got:", e.Subject) } } @@ -146,3 +160,26 @@ func TestEncodedWordAhead(t *testing.T) { } } + +func TestQueuedID(t *testing.T) { + h := QueuedID(5550000000, 1) + + if len(h) != 16 { // silly comparison, but there in case of refactoring + t.Error("queuedID needs to be 16 bytes in length") + } + + str := h.String() + if len(str) != 32 { + t.Error("queuedID string should be 32 bytes in length") + } + + h2 := QueuedID(5550000000, 1) + if bytes.Equal(h[:], h2[:]) { + t.Error("hashes should not be equal") + } + + h2.FromHex("5a4a2f08784334de5148161943111ad3") + if h2.String() != "5a4a2f08784334de5148161943111ad3" { + t.Error("hex conversion didnt work") + } +} diff --git a/mail/mimeparse/mime.go b/mail/mimeparse/mime.go new file mode 100644 index 00000000..cbaf803a --- /dev/null +++ b/mail/mimeparse/mime.go @@ -0,0 +1,1211 @@ +package mimeparse + +/* + +Mime is a simple MIME scanner for email-message byte streams. +It builds a data-structure that represents a tree of all the mime parts, +recording their headers, starting and ending positions, while processioning +the message efficiently, slice by slice. It avoids the use of regular expressions, +doesn't back-track or multi-scan. + +*/ +import ( + "bytes" + "fmt" + "io" + "net/textproto" + "strconv" + "strings" + "sync" +) + +var ( + MaxNodesErr *Error + NotMineErr *Error +) + +func init() { + NotMineErr = &Error{ + err: ErrorNotMime, + } + MaxNodesErr = &Error{ + err: ErrorMaxNodes, + } +} + +const ( + // maxBoundaryLen limits the length of the content-boundary. + // Technically the limit is 79, but here we are more liberal + maxBoundaryLen = 70 + 10 + + // doubleDash is the prefix for a content-boundary string. It is also added + // as a postfix to a content-boundary string to signal the end of content parts. + doubleDash = "--" + + // startPos assigns the pos property when the buffer is set. + // The reason why -1 is because peek() implementation becomes simpler + startPos = -1 + + // headerErrorThreshold how many errors in the header + headerErrorThreshold = 4 + + multipart = "multipart" + contentTypeHeader = "Content-Type" + dot = "." + first = "1" + + // MaxNodes limits the number of items in the Parts array. Effectively limiting + // the number of nested calls the parser may make. + MaxNodes = 512 +) + +type MimeError int + +const ( + ErrorNotMime MimeError = iota + ErrorMaxNodes + ErrorBoundaryTooShort + ErrorBoundaryLineExpected + ErrorUnexpectedChar + ErrorHeaderFieldTooShort + ErrorBoundaryExceededLength + ErrorHeaderParseError + ErrorMissingSubtype + ErrorUnexpectedTok + ErrorUnexpectedCommentToken + ErrorInvalidToken + ErrorUnexpectedQuotedStrToken + ErrorParameterExpectingEquals + ErrorNoHeader +) + +func (e MimeError) Error() string { + switch e { + case ErrorNotMime: + return "not Mime" + case ErrorMaxNodes: + return "too many mime part nodes" + case ErrorBoundaryTooShort: + return "content boundary too short" + case ErrorBoundaryLineExpected: + return "boundary new line expected" + case ErrorUnexpectedChar: + return "unexpected char" + case ErrorHeaderFieldTooShort: + return "header field too short" + case ErrorBoundaryExceededLength: + return "boundary exceeded max length" + case ErrorHeaderParseError: + return "header parse error" + case ErrorMissingSubtype: + return "missing subtype" + case ErrorUnexpectedTok: + return "unexpected tok" + case ErrorUnexpectedCommentToken: + return "unexpected comment token" + case ErrorInvalidToken: + return "invalid token" + case ErrorUnexpectedQuotedStrToken: + return "unexpected token" + case ErrorParameterExpectingEquals: + return "expecting =" + case ErrorNoHeader: + return "parse error, no header" + } + return "unknown mime error" +} + +func (e *MimeError) UnmarshalJSON(b []byte) error { + v, err := strconv.ParseInt(string(b), 10, 32) + if err != nil { + return err + } + *e = MimeError(v) + return nil +} + +// MarshalJSON implements json.Marshaler +func (e MimeError) MarshalJSON() ([]byte, error) { + return []byte(strconv.Itoa(int(e))), nil +} + +// Error implements the error interface +type Error struct { + err error + char byte + peek byte + pos uint // msgPos +} + +func (e Error) Error() string { + if e.char == 0 { + return e.err.Error() + } + return e.err.Error() + " char:[" + string(e.char) + "], peek:[" + + string(e.peek) + "], pos:" + strconv.Itoa(int(e.pos)) +} + +func (e Error) Unwrap() error { + return e.err +} + +func (e *Error) ParseError() bool { + if e.err != io.EOF && error(e.err) != NotMineErr && error(e.err) != MaxNodesErr { + return true + } + return false +} + +func (p *Parser) newParseError(e MimeError) *Error { + var peek byte + offset := 1 + for { + // reached the end? (don't wait for more bytes to consume) + if p.pos+offset >= len(p.buf) { + peek = 0 + break + } + // peek the next byte + peek := p.buf[p.pos+offset] + if peek == '\r' { + // ignore \r + offset++ + continue + } + break + } + return &Error{ + err: e, + char: p.ch, + peek: peek, + pos: p.msgPos, + } +} + +type captureBuffer struct { + bytes.Buffer + upper bool // flag used by acceptHeaderName(), if true, the next accepted chr will be uppercase'd +} + +type Parser struct { + + // related to the state of the parser + + buf []byte // input buffer + pos int // position in the input buffer + peekOffset int // peek() ignores \r so we must keep count of how many \r were ignored + ch byte // value of byte at current pos in buf[]. At EOF, ch == 0 + gotNewSlice, consumed chan bool // flags that control the synchronisation of reads + accept captureBuffer // input is captured to this buffer to build strings + boundaryMatched int // an offset. Used in cases where the boundary string is split over multiple buffers + count uint // counts how many times Parse() was called + result chan parserMsg // used to pass the result back to the main goroutine + mux sync.Mutex // ensure calls to Parse() and Close() are synchronized + + // Parts is the mime parts tree. The parser builds the parts as it consumes the input + // In order to represent the tree in an array, we use Parts.Node to store the name of + // each node. The name of the node is the *path* of the node. The root node is always + // "1". The child would be "1.1", the next sibling would be "1.2", while the child of + // "1.2" would be "1.2.1" + Parts Parts + + msgPos uint // global position in the message + + lastBoundaryPos uint // the last msgPos where a boundary was detected + + maxNodes int // the desired number of maximum nodes the parser is limited to + + w io.Writer // underlying io.Writer + + temp string +} + +type Parts []*Part + +type Part struct { + + // Headers contain the header names and values in a map data-structure + Headers textproto.MIMEHeader + + // Node stores the name for the node that is a part of the resulting mime tree + Node string + // StartingPos is the starting position, including header (after boundary, 0 at the top) + StartingPos uint + // StartingPosBody is the starting position of the body, after header \n\n + StartingPosBody uint + // EndingPos is the ending position for the part, including the boundary line + EndingPos uint + // EndingPosBody is the ending position for the body, excluding boundary. + // I.e EndingPos - len(Boundary Line) + EndingPosBody uint + + // Charset holds the character-set the part is encoded in, eg. us-ascii + Charset string + // TransferEncoding holds the transfer encoding that was used to pack the message eg. base64 + TransferEncoding string + // ContentBoundary holds the unique string that was used to delimit multi-parts, eg. --someboundary123 + ContentBoundary string + // ContentType holds the mime content type, eg text/html + ContentType *contentType + // ContentBase is typically a url + ContentBase string + // DispositionFileName what file-nme to use for the part, eg. image.jpeg + DispositionFileName string + // ContentDisposition describes how to display the part, eg. attachment + ContentDisposition string + // ContentName as name implies + ContentName string +} + +type parameter struct { + name string + value string +} + +type contentType struct { + superType string + subType string + parameters []parameter + b bytes.Buffer +} + +type parserMsg struct { + err error +} + +var isTokenSpecial = [128]bool{ + '(': true, + ')': true, + '<': true, + '>': true, + '@': true, + ',': true, + ';': true, + ':': true, + '\\': true, + '"': true, + '/': true, + '[': true, + ']': true, + '?': true, + '=': true, +} + +func (c *contentType) params() (ret string) { + defer func() { + c.b.Reset() + }() + for k := range c.parameters { + if c.parameters[k].value == "" { + c.b.WriteString("; " + c.parameters[k].name) + continue + } + c.b.WriteString("; " + c.parameters[k].name + "=\"" + c.parameters[k].value + "\"") + } + return c.b.String() +} + +// String returns the contentType type as a string +func (c *contentType) String() (ret string) { + ret = fmt.Sprintf("%s/%s%s", c.superType, c.subType, + c.params()) + return +} + +// Charset returns the charset value specified by the content type +func (c *contentType) Charset() (ret string) { + if c.superType == "" { + return "" + } + for i := range c.parameters { + if c.parameters[i].name == "charset" { + return c.parameters[i].value + } + } + return "" +} + +func (c *contentType) Supertype() (ret string) { + return c.superType +} + +func newPart() *Part { + mh := new(Part) + mh.Headers = make(textproto.MIMEHeader, 1) + return mh +} + +func (p *Parser) addPart(mh *Part, id string) { + mh.Node = id + p.Parts = append(p.Parts, mh) +} + +// more waits for more input, returns false if there is no more +func (p *Parser) more() bool { + p.consumed <- true // signal that we've reached the end of available input + gotMore := <-p.gotNewSlice + return gotMore +} + +// next reads the next byte and advances the pointer +// returns 0 if no more input can be read +// blocks if at the end of the buffer +func (p *Parser) next() byte { + for { + // wait for more bytes if reached the end + if p.pos+1 >= len(p.buf) { + if !p.more() { + p.ch = 0 + return 0 + } + } + if p.pos > -1 || p.msgPos != 0 { + // dont incr on first call to next() + p.msgPos++ + } + p.pos++ + if p.buf[p.pos] == '\r' { + // ignore \r + continue + } + p.ch = p.buf[p.pos] + + return p.ch + } +} + +// peek does not advance the pointer, but will block if there's no more +// input in the buffer +func (p *Parser) peek() byte { + p.peekOffset = 1 + for { + // reached the end? Wait for more bytes to consume + if p.pos+p.peekOffset >= len(p.buf) { + if !p.more() { + return 0 + } + } + // peek the next byte + ret := p.buf[p.pos+p.peekOffset] + if ret == '\r' { + // ignore \r + p.peekOffset++ + continue + } + return ret + } +} + +// inject is used for testing, to simulate a byte stream +func (p *Parser) inject(input ...[]byte) { + p.msgPos = 0 + p.set(input[0]) + p.pos = 0 + p.ch = p.buf[0] + go func() { + for i := 1; i < len(input); i++ { + <-p.consumed + p.set(input[i]) + p.gotNewSlice <- true + } + <-p.consumed + p.gotNewSlice <- false // no more data + }() +} + +// Set the buffer and reset p.pos to startPos, which is typically -1 +// The reason why -1 is because peek() implementation becomes more +// simple, as it only needs to add 1 to p.pos for all cases. +// We don't read the buffer when we set, only when next() is called. +// This allows us to peek in to the next buffer while still being on +// the last element from the previous buffer +func (p *Parser) set(input []byte) { + if p.pos != startPos { + // rewind + p.pos = startPos + } + p.buf = input +} + +// skip advances the pointer n bytes. It will block if not enough bytes left in +// the buffer, i.e. if bBytes > len(p.buf) - p.pos +func (p *Parser) skip(nBytes int) { + for { + if p.pos+nBytes < len(p.buf) { + p.pos += nBytes - 1 + p.msgPos = p.msgPos + uint(nBytes) - 1 + p.next() + return + } + remainder := len(p.buf) - p.pos + nBytes -= remainder + p.pos += remainder - 1 + p.msgPos += uint(remainder - 1) + p.next() + if p.ch == 0 { + return + } else if nBytes < 1 { + return + } + } +} + +// boundary scans until next boundary string, returns error if not found +// syntax specified https://tools.ietf.org/html/rfc2046 p21 +func (p *Parser) boundary(contentBoundary string) (end bool, err error) { + defer func() { + if err == nil { + if p.ch == '\n' { + p.next() + } + } + }() + + if len(contentBoundary) < 1 { + err = ErrorBoundaryTooShort + } + boundary := doubleDash + contentBoundary + p.boundaryMatched = 0 + for { + if i := bytes.Index(p.buf[p.pos:], []byte(boundary)); i > -1 { + p.skip(i) + p.lastBoundaryPos = p.msgPos + p.skip(len(boundary)) + if end, err = p.boundaryEnd(); err != nil { + return + } + if err = p.transportPadding(); err != nil { + return + } + if p.ch != '\n' { + err = ErrorBoundaryLineExpected + } + return + } else { + // search the tail for partial match + // if one is found, load more data and continue the match + // if matched, advance buffer in same way as above + start := len(p.buf) - len(boundary) + 1 + if start < 0 { + start = 0 + } + subject := p.buf[start:] + + for i := 0; i < len(subject); i++ { + if subject[i] == boundary[p.boundaryMatched] { + p.boundaryMatched++ + } else { + p.boundaryMatched = 0 + } + } + + p.skip(len(p.buf) - p.pos) // discard the remaining data + + if p.ch == 0 { + return false, io.EOF + } else if p.boundaryMatched > 0 { + // check for a match by joining the match from the end of the last buf + // & the beginning of this buf + if bytes.Compare( + p.buf[0:len(boundary)-p.boundaryMatched], + []byte(boundary[p.boundaryMatched:])) == 0 { + + // advance the pointer + p.skip(len(boundary) - p.boundaryMatched) + + p.lastBoundaryPos = p.msgPos - uint(len(boundary)) + end, err = p.boundaryEnd() + if err != nil { + return + } + if err = p.transportPadding(); err != nil { + return + } + if p.ch != '\n' { + err = ErrorBoundaryLineExpected + } + return + } + p.boundaryMatched = 0 + } + } + } +} + +// is it the end of a boundary? +func (p *Parser) boundaryEnd() (result bool, err error) { + if p.ch == '-' && p.peek() == '-' { + p.next() + p.next() + result = true + } + if p.ch == 0 { + err = io.EOF + } + return +} + +// *LWSP-char +// = *(WSP / CRLF WSP) +func (p *Parser) transportPadding() (err error) { + for { + if p.ch == ' ' || p.ch == '\t' { + p.next() + } else if c := p.peek(); p.ch == '\n' && (c == ' ' || c == '\t') { + p.next() + p.next() + } else { + if c == 0 { + err = io.EOF + } + return + } + } +} + +// acceptHeaderName builds the header name in the buffer while ensuring that +// that the case is normalized. Ie. Content-type is written as Content-Type +func (p *Parser) acceptHeaderName() { + if p.accept.upper && p.ch >= 'a' && p.ch <= 'z' { + p.ch -= 32 + } + if !p.accept.upper && p.ch >= 'A' && p.ch <= 'Z' { + p.ch += 32 + } + p.accept.upper = p.ch == '-' + _ = p.accept.WriteByte(p.ch) +} + +func (p *Parser) header(mh *Part) (err error) { + var ( + state int + name string + errorCount int + ) + + defer func() { + p.accept.Reset() + if val := mh.Headers.Get("Content-Transfer-Encoding"); val != "" { + mh.TransferEncoding = val + } + if val := mh.Headers.Get("Content-Disposition"); val != "" { + mh.ContentDisposition = val + } + }() + + for { + switch state { + case 0: // header name + if (p.ch >= 33 && p.ch <= 126) && p.ch != ':' { + // capture + p.acceptHeaderName() + } else if p.ch == ':' { + state = 1 + } else if p.ch == ' ' && p.peek() == ':' { // tolerate a SP before the : + p.next() + state = 1 + } else { + if errorCount < headerErrorThreshold { + state = 2 // tolerate this error + continue + } + err = p.newParseError(ErrorUnexpectedChar) + return + } + if state == 1 { + if p.accept.Len() < 2 { + err = p.newParseError(ErrorHeaderFieldTooShort) + return + } + p.accept.upper = true + name = p.accept.String() + p.accept.Reset() + if c := p.peek(); c == ' ' { + // skip the space + p.next() + } + p.next() + continue + } + case 1: // header value + if name == contentTypeHeader { + var err error + contentType, err := p.contentType() + if err != nil { + return err + } + mh.ContentType = &contentType + for i := range contentType.parameters { + switch { + case contentType.parameters[i].name == "boundary": + mh.ContentBoundary = contentType.parameters[i].value + if len(mh.ContentBoundary) >= maxBoundaryLen { + return p.newParseError(ErrorBoundaryExceededLength) + } + case contentType.parameters[i].name == "charset": + mh.Charset = strings.ToUpper(contentType.parameters[i].value) + case contentType.parameters[i].name == "name": + mh.ContentName = contentType.parameters[i].value + } + } + mh.Headers.Add(contentTypeHeader, contentType.String()) + state = 0 + } else { + if p.ch != '\n' || p.isWSP(p.ch) { + _ = p.accept.WriteByte(p.ch) + } else if p.ch == '\n' { + c := p.peek() + + if p.isWSP(c) { + break // skip \n + } else { + mh.Headers.Add(name, p.accept.String()) + p.accept.Reset() + + state = 0 + } + } else { + err = p.newParseError(ErrorHeaderParseError) + return + } + } + case 2: // header error, discard line + errorCount++ + // error recovery for header lines with parse errors - + // ignore the line, discard anything that was scanned, scan until the end-of-line + // then start a new line again (back to state 0) + p.accept.Reset() + for { + if p.ch != '\n' { + p.next() + } + if p.ch == 0 { + return io.EOF + } else if p.ch == '\n' { + state = 0 + break + } + } + } + if p.ch == '\n' && p.peek() == '\n' { + return nil + } + p.next() + + if p.ch == 0 { + return io.EOF + } + + } + +} + +func (p *Parser) isWSP(b byte) bool { + return b == ' ' || b == '\t' +} + +func (p *Parser) contentType() (result contentType, err error) { + result = contentType{} + + if result.superType, err = p.mimeType(); err != nil { + return + } + if p.ch != '/' { + return result, p.newParseError(ErrorMissingSubtype) + } + p.next() + + if result.subType, err = p.mimeSubType(); err != nil { + return + } + + for { + if p.ch == ';' { + p.next() + continue + } + if p.ch == '\n' { + c := p.peek() + if p.isWSP(c) { + p.next() // skip \n (FWS) + continue + } + if c == '\n' { // end of header + return + } + } + if p.isWSP(p.ch) { // skip WSP + p.next() + continue + } + if p.ch == '(' { + if err = p.comment(); err != nil { + return + } + continue + } + if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] { + if key, val, err := p.parameter(); err != nil { + return result, err + } else { + if key == "charset" { + val = strings.ToUpper(val) + } + // add the new parameter + result.parameters = append(result.parameters, parameter{key, val}) + } + } else { + break + } + } + + return +} + +func (p *Parser) mimeType() (str string, err error) { + + defer func() { + if p.accept.Len() > 0 { + str = p.accept.String() + p.accept.Reset() + } + }() + if p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch] { + for { + if p.ch >= 'A' && p.ch <= 'Z' { + p.ch += 32 // lowercase + } + _ = p.accept.WriteByte(p.ch) + p.next() + if !(p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch]) { + return + } + + } + } else { + err = p.newParseError(ErrorUnexpectedTok) + return + } +} + +func (p *Parser) mimeSubType() (str string, err error) { + return p.mimeType() +} + +// comment = "(" *(ctext / quoted-pair / comment) ")" +// +// ctext = may be folded +// ")", "\" & CR, & including +// linear-white-space> +// +// quoted-pair = "\" CHAR ; may quote any char +func (p *Parser) comment() (err error) { + // all header fields except for Content-Disposition + // can include RFC 822 comments + if p.ch != '(' { + err = p.newParseError(ErrorUnexpectedCommentToken) + } + for { + p.next() + if p.ch == ')' { + p.next() + return + } + } +} + +func (p *Parser) token(lower bool) (str string, err error) { + defer func() { + if err == nil { + str = p.accept.String() + } + if p.accept.Len() > 0 { + p.accept.Reset() + } + }() + var once bool // must match at least 1 good char + for { + if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] { + if lower && p.ch >= 'A' && p.ch <= 'Z' { + p.ch += 32 // lowercase it + } + _ = p.accept.WriteByte(p.ch) + once = true + } else if !once { + err = p.newParseError(ErrorInvalidToken) + return + } else { + return + } + p.next() + } +} + +// quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) +// quoted-pair = "\" CHAR +// CHAR = +// qdtext = > +// TEXT = +func (p *Parser) quotedString() (str string, err error) { + defer func() { + if err == nil { + str = p.accept.String() + } + if p.accept.Len() > 0 { + p.accept.Reset() + } + }() + + if p.ch != '"' { + err = p.newParseError(ErrorUnexpectedQuotedStrToken) + return + } + p.next() + state := 0 + for { + switch state { + case 0: // inside quotes + if p.ch == '"' { + p.next() + return + } + if p.ch == '\\' { + state = 1 + break + } + if (p.ch < 127 && p.ch > 32) || p.isWSP(p.ch) { + _ = p.accept.WriteByte(p.ch) + } else { + err = p.newParseError(ErrorUnexpectedQuotedStrToken) + return + } + case 1: + // escaped () + if p.ch != 0 && p.ch <= 127 { + _ = p.accept.WriteByte(p.ch) + state = 0 + } else { + err = p.newParseError(ErrorUnexpectedQuotedStrToken) + return + } + } + p.next() + } +} + +// parameter := attribute "=" value +// attribute := token +// token := 1* +// value := token / quoted-string +// CTL := %x00-1F / %x7F +// quoted-string : <"> <"> +func (p *Parser) parameter() (attribute, value string, err error) { + defer func() { + p.accept.Reset() + }() + + if attribute, err = p.token(true); err != nil { + return "", "", err + } + if p.ch != '=' { + if len(attribute) > 0 { + return + } + return "", "", p.newParseError(ErrorParameterExpectingEquals) + } + p.next() + if p.ch == '"' { + if value, err = p.quotedString(); err != nil { + return + } + return + } else { + if value, err = p.token(false); err != nil { + return + } + return + } +} + +// mime scans the mime content and builds the mime-part tree in +// p.Parts on-the-fly, as more bytes get fed in. +func (p *Parser) mime(part *Part, cb string) (err error) { + if len(p.Parts) >= p.maxNodes { + for { + // skip until the end of the stream (we've stopped parsing due to max nodes) + p.skip(len(p.buf) + 1) + if p.ch == 0 { + break + } + } + if p.maxNodes == 1 { + // in this case, only one header item, so assume the end of message is + // the ending position of the header + p.Parts[0].EndingPos = p.msgPos + p.Parts[0].EndingPosBody = p.msgPos + } else { + err = MaxNodesErr + } + return + } + count := 1 + root := part == nil + if root { + part = newPart() + p.addPart(part, first) + defer func() { + // err is io.EOF if nothing went with parsing + if err == io.EOF { + err = nil + } + if err != MaxNodesErr { + part.EndingPosBody = p.lastBoundaryPos + part.EndingPos = p.msgPos + } else { + // remove the unfinished node (edge case) + var parts []*Part + p.Parts = append(parts, p.Parts[:p.maxNodes]...) + } + // not a mime email (but is an rfc5322 message) + if len(p.Parts) == 1 && + len(part.Headers) > 0 && + part.Headers.Get("MIME-Version") == "" && + err == nil { + err = NotMineErr + } + }() + } + + // read the header + if p.ch >= 33 && p.ch <= 126 { + err = p.header(part) + if err != nil { + return err + } + } else if root { + return p.newParseError(ErrorNoHeader) + } + if p.ch == '\n' && p.peek() == '\n' { + p.next() + p.next() + } + part.StartingPosBody = p.msgPos + ct := part.ContentType + if ct != nil && ct.superType == "message" && ct.subType == "rfc822" { + var subPart *Part + subPart = newPart() + subPartId := part.Node + dot + strconv.Itoa(count) + subPart.StartingPos = p.msgPos + count++ + p.addPart(subPart, subPartId) + err = p.mime(subPart, part.ContentBoundary) + subPart.EndingPosBody = p.lastBoundaryPos + subPart.EndingPos = p.msgPos + return + } + if ct != nil && ct.superType == multipart && + part.ContentBoundary != "" && + part.ContentBoundary != cb { /* content-boundary must be different to previous */ + var subPart *Part + subPart = newPart() + subPart.ContentBoundary = part.ContentBoundary + for { + subPartId := part.Node + dot + strconv.Itoa(count) + if end, bErr := p.boundary(part.ContentBoundary); bErr != nil { + // there was an error with parsing the boundary + err = bErr + if subPart.StartingPos == 0 { + subPart.StartingPos = p.msgPos + } else { + subPart.EndingPos = p.msgPos + subPart.EndingPosBody = p.lastBoundaryPos + subPart, count = p.split(subPart, count) + } + return + } else if end { + // reached the terminating boundary (ends with double dash --) + subPart.EndingPosBody = p.lastBoundaryPos + subPart.EndingPos = p.msgPos + break + } else { + // process the part boundary + if subPart.StartingPos == 0 { + subPart.StartingPos = p.msgPos + count++ + p.addPart(subPart, subPartId) + err = p.mime(subPart, part.ContentBoundary) + if err != nil { + return + } + subPartId = part.Node + dot + strconv.Itoa(count) + } else { + subPart.EndingPosBody = p.lastBoundaryPos + subPart.EndingPos = p.msgPos + subPart, count = p.split(subPart, count) + p.addPart(subPart, subPartId) + err = p.mime(subPart, part.ContentBoundary) + if err != nil { + return + } + } + } + } + } else if part.ContentBoundary == "" { + for { + p.skip(len(p.buf)) + if p.ch == 0 { + if part.StartingPosBody > 0 { + part.EndingPosBody = p.msgPos + part.EndingPos = p.msgPos + } + err = io.EOF + return + } + } + + } + return + +} + +func (p *Parser) split(subPart *Part, count int) (*Part, int) { + cb := subPart.ContentBoundary + subPart = nil + count++ + subPart = newPart() + subPart.StartingPos = p.msgPos + subPart.ContentBoundary = cb + return subPart, count +} + +func (p *Parser) reset() { + p.lastBoundaryPos = 0 + p.pos = startPos + p.msgPos = 0 + p.count = 0 + p.ch = 0 +} + +// Open prepares the parser for accepting input +func (p *Parser) Open() { + p.Parts = make([]*Part, 0) +} + +// Close tells the MIME Parser there's no more data & waits for it to return a result +// it will return an io.EOF error if no error with parsing MIME was detected +func (p *Parser) Close() error { + p.mux.Lock() + defer func() { + p.reset() + p.mux.Unlock() + }() + if p.count == 0 { + return nil + } + for { + select { + // we need to repeat sending a false signal because peek() / next() could be + // called a few times before a result is returned + case p.gotNewSlice <- false: + select { + + case <-p.consumed: // more() was called, there's nothing to consume + + case r := <-p.result: + return r.err + } + case r := <-p.result: + + return r.err + } + + } + +} + +func (p *Parser) Write(buf []byte) (int, error) { + p.temp = p.temp + string(buf) + + if err := p.Parse(buf); err != nil { + return len(buf), err + } + if p.w != nil { + return p.w.Write(buf) + } + return len(buf), nil +} + +// Parse takes a byte stream, and feeds it to the MIME Parser, then +// waits for the Parser to consume all input before returning. +// The parser will build a parse tree in p.Parts +// The parser doesn't decode any input. All it does +// is collect information about where the different MIME parts +// start and end, and other meta-data. This data can be used +// later down the stack to determine how to store/decode/display +// the messages +// returns error if there's a parse error, except io.EOF when no +// error occurred. +func (p *Parser) Parse(buf []byte) error { + defer func() { + p.mux.Unlock() + }() + p.mux.Lock() + + // Feed the new slice. Assumes that the parser is blocked now, waiting + // for new data, or not started yet. + p.set(buf) + + if p.count == 0 { + // initial step - start the mime parser + go func() { + p.next() + err := p.mime(nil, "") + p.result <- parserMsg{err} + }() + } else { + // tell the parser to resume consuming + p.gotNewSlice <- true + } + p.count++ + + select { + case <-p.consumed: // wait for prev buf to be consumed + return nil + case r := <-p.result: + // mime() has returned with a result (it finished consuming) + p.reset() + return r.err + } +} + +// Error returns true if the type of error was a parse error +// Returns false if it was an io.EOF or the email was not mime, or exceeded maximum nodes +func (p *Parser) ParseError(err error) bool { + if err != nil && err != io.EOF && err != NotMineErr && err != MaxNodesErr { + return true + } + return false +} + +// NewMimeParser returns a mime parser. See MaxNodes for how many nodes it's limited to +func NewMimeParser() *Parser { + p := new(Parser) + p.consumed = make(chan bool) + p.gotNewSlice = make(chan bool) + p.result = make(chan parserMsg, 1) + p.maxNodes = MaxNodes + return p +} + +func NewMimeParserWriter(w io.Writer) *Parser { + p := NewMimeParser() + p.w = w + return p +} + +// NewMimeParser returns a mime parser with a custom MaxNodes value +func NewMimeParserLimited(maxNodes int) *Parser { + p := NewMimeParser() + p.maxNodes = maxNodes + return p +} diff --git a/mail/mimeparse/mime_test.go b/mail/mimeparse/mime_test.go new file mode 100644 index 00000000..9da0c592 --- /dev/null +++ b/mail/mimeparse/mime_test.go @@ -0,0 +1,744 @@ +package mimeparse + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "testing" + "time" +) + +var p *Parser + +func init() { + +} +func TestInject(t *testing.T) { + p = NewMimeParser() + var b bytes.Buffer + + // it should read from both slices + // as if it's a continuous stream + p.inject([]byte("abcd"), []byte("efgh"), []byte("ijkl")) + for i := 0; i < 12; i++ { + b.WriteByte(p.ch) + if p.pos == 3 && p.msgPos < 4 { + if c := p.peek(); c != 101 { + t.Error("next() expecting e, got:", string(c)) + } + } + p.next() + if p.ch == 0 { + break + } + } + if b.String() != "abcdefghijkl" { + t.Error("expecting abcdefghijkl, got:", b.String()) + } +} +func TestMimeType(t *testing.T) { + p = NewMimeParser() + if isTokenSpecial['-'] { + t.Error("- should not be in the set") + } + + p.inject([]byte("text/plain; charset=us-ascii")) + str, err := p.mimeType() + if err != nil { + t.Error(err) + } + if str != "text" { + t.Error("mime type should be: text") + } + +} + +func TestMimeContentType(t *testing.T) { + p = NewMimeParser() + go func() { + <-p.consumed + p.gotNewSlice <- false + }() + + // what happens if we call Charset with empty content type? + empty := contentType{} + blank := empty.Charset() + if blank != "" { + t.Error("expecting charset to be blank") + } + + subject := "text/plain; charset=\"us-aScii\"; moo; boundary=\"foo\"" + p.inject([]byte(subject)) + contentType, err := p.contentType() + if err != nil { + t.Error(err) + } + + if charset := contentType.Charset(); charset != "US-ASCII" { + t.Error("charset is not US-ASCII") + } + + // test the stringer (note it will canonicalize us-aScii to US-ASCII + subject = strings.Replace(subject, "us-aScii", "US-ASCII", 1) + if ct := contentType.String(); contentType.String() != subject { + t.Error("\n[" + ct + "]\ndoes not equal\n[" + subject + "]") + } + + // what happens if we don't use quotes for the param? + subject = "text/plain; charset=us-aScii; moo; boundary=\"foo\"" + p.inject([]byte(subject)) + contentType, err = p.contentType() + if err != nil { + t.Error(err) + } + + if contentType.subType != "plain" { + t.Error("contentType.subType expecting 'plain', got:", contentType.subType) + } + + if contentType.superType != "text" { + t.Error("contentType.subType expecting 'text', got:", contentType.superType) + } + +} + +func TestEmailHeader(t *testing.T) { + p = NewMimeParser() + in := `Wong ignore me +From: Al Gore +To: White House Transportation Coordinator +Subject: [Fwd: Map of Argentina with Description] +MIME-Version: 1.0 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=ncr424; d=reliancegeneral.co.in; + h=List-Unsubscribe:MIME-Version:From:To:Reply-To:Date:Subject:Content-Type:Content-Transfer-Encoding:Message-ID; i=prospects@prospects.reliancegeneral.co.in; + bh=F4UQPGEkpmh54C7v3DL8mm2db1QhZU4gRHR1jDqffG8=; + b=MVltcq6/I9b218a370fuNFLNinR9zQcdBSmzttFkZ7TvV2mOsGrzrwORT8PKYq4KNJNOLBahswXf + GwaMjDKT/5TXzegdX/L3f/X4bMAEO1einn+nUkVGLK4zVQus+KGqm4oP7uVXjqp70PWXScyWWkbT + 1PGUwRfPd/HTJG5IUqs= +Content-Type: multipart/mixed; + boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" + +This is a multi-part message in MIME format. +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Fred, + +Fire up Air Force One! We\'re going South! + +Thanks, +Al +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +This +` + p.inject([]byte(in)) + h := newPart() + err := p.header(h) + if err != nil { + t.Error(err) + } + if _, err := p.boundary(h.ContentBoundary); err != nil { + t.Error(err) + } +} + +func TestBoundary(t *testing.T) { + p = NewMimeParser() + var err error + part := newPart() + part.ContentBoundary = "-wololo-" + + // in the middle of the string + test := "The quick brown fo---wololo-\nx jumped over the lazy dog" + p.inject([]byte(test)) + + _, err = p.boundary(part.ContentBoundary) + if err != nil && err != io.EOF { + t.Error(err) + } + body := string(test[:p.lastBoundaryPos]) + if body != "The quick brown fo" { + t.Error("p.lastBoundaryPos seems incorrect") + } + + // at the end (with the -- postfix) + p.inject([]byte("The quick brown fox jumped over the lazy dog---wololo---\n")) + _, err = p.boundary(part.ContentBoundary) + if err != nil && err != io.EOF { + t.Error(err) + } + + // the boundary with an additional buffer in between + p.inject([]byte("The quick brown fox jumped over the lazy dog"), + []byte("this is the middle"), + []byte("and thats the end---wololo---\n")) + + _, err = p.boundary(part.ContentBoundary) + if err != nil && err != io.EOF { + t.Error(err) + } + +} + +func TestBoundarySplit(t *testing.T) { + p = NewMimeParser() + var err error + part := newPart() + + part.ContentBoundary = "-wololo-" + // boundary is split over multiple slices + p.inject( + []byte("The quick brown fox jumped ov---wolo"), + []byte("lo---\ner the lazy dog")) + _, err = p.boundary(part.ContentBoundary) + if err != nil && err != io.EOF { + t.Error(err) + } + + body := string([]byte("The quick brown fox jumped ov---wolo")[:p.lastBoundaryPos]) + if body != "The quick brown fox jumped ov" { + t.Error("p.lastBoundaryPos value seems incorrect") + } + + // boundary has a space, pointer advanced before, and is split over multiple slices + part.ContentBoundary = "XXXXboundary text" // 17 chars + p.inject( + []byte("The quick brown fox jumped ov--X"), + []byte("XXXboundary text\ner the lazy dog")) + p.next() // here the pointer is advanced before the boundary is searched + _, err = p.boundary(part.ContentBoundary) + if err != nil && err != io.EOF { + t.Error(err) + return + } + +} + +func TestSkip(t *testing.T) { + p = NewMimeParser() + p.inject( + []byte("you cant touch this"), + []byte("stop, hammer time")) + + p.skip(3) + + if p.pos != 3 { + t.Error("position should be 3 after skipping 3 bytes, it is:", p.pos) + } + + p.pos = 0 + + // after we used next() to advance + p.next() + p.skip(3) + if p.pos != 4 { + t.Error("position should be 4 after skipping 3 bytes, it is:", p.pos) + } + + // advance to the 2nd buffer + p.pos = 0 + p.msgPos = 0 + p.skip(19) + if p.pos != 0 && p.buf[0] != 's' { + t.Error("position should be 0 and p.buf[0] should be 's'") + } + +} + +func TestHeaderNormalization(t *testing.T) { + p = NewMimeParser() + p.inject([]byte("ConTent-type")) + p.accept.upper = true + for { + p.acceptHeaderName() + p.next() + if p.ch == 0 { + break + } + } + if p.accept.String() != "Content-Type" { + t.Error("header name not normalized, expecting Content-Type") + } +} + +func TestMimeContentQuotedParams(t *testing.T) { + p = NewMimeParser() + // quoted + p.inject([]byte("text/plain; charset=\"us-ascii\"")) + contentType, err := p.contentType() + if err != nil { + t.Error(err) + } + + // with whitespace & tab + p.inject([]byte("text/plain; charset=\"us-ascii\" \tboundary=\"D7F------------D7FD5A0B8AB9C65CCDBFA872\"")) + contentType, err = p.contentType() + if err != nil { + t.Error(err) + } + + // with comment (ignored) + p.inject([]byte("text/plain; charset=\"us-ascii\" (a comment) \tboundary=\"D7F------------D7FD5A0B8AB9C65CCDBFA872\"")) + contentType, err = p.contentType() + + if contentType.subType != "plain" { + t.Error("contentType.subType expecting 'plain', got:", contentType.subType) + } + + if contentType.superType != "text" { + t.Error("contentType.subType expecting 'text', got:", contentType.superType) + } + + if len(contentType.parameters) != 2 { + t.Error("expecting 2 elements in parameters") + } else { + m := make(map[string]string) + for _, e := range contentType.parameters { + m[e.name] = e.value + } + if _, ok := m["charset"]; !ok { + t.Error("charset parameter not present") + } + if b, ok := m["boundary"]; !ok { + t.Error("charset parameter not present") + } else { + if b != "D7F------------D7FD5A0B8AB9C65CCDBFA872" { + t.Error("boundary should be: D7F------------D7FD5A0B8AB9C65CCDBFA872") + } + } + + } + +} + +var email = `From: Al Gore +To: White House Transportation Coordinator +Subject: [Fwd: Map of Argentina with Description] +MIME-Version: 1.0 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed; s=ncr424; d=reliancegeneral.co.in; + h=List-Unsubscribe:MIME-Version:From:To:Reply-To:Date:Subject:Content-Type:Content-Transfer-Encoding:Message-ID; i=prospects@prospects.reliancegeneral.co.in; + bh=F4UQPGEkpmh54C7v3DL8mm2db1QhZU4gRHR1jDqffG8=; + b=MVltcq6/I9b218a370fuNFLNinR9zQcdBSmzttFkZ7TvV2mOsGrzrwORT8PKYq4KNJNOLBahswXf + GwaMjDKT/5TXzegdX/L3f/X4bMAEO1einn+nUkVGLK4zVQus+KGqm4oP7uVXjqp70PWXScyWWkbT + 1PGUwRfPd/HTJG5IUqs= +Content-Type: multipart/mixed; + boundary="D7F------------D7FD5A0B8AB9C65CCDBFA872" + +This is a multi-part message in MIME format. +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Fred, + +Fire up Air Force One! We're going South! + +Thanks, +Al +--D7F------------D7FD5A0B8AB9C65CCDBFA872 +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Return-Path: +Received: from mailhost.whitehouse.gov ([192.168.51.200]) + by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453 + for ; + Mon, 13 Aug 1998 l8:14:23 +1000 +Received: from the_big_box.whitehouse.gov ([192.168.51.50]) + by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366 + for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000 + Date: Mon, 13 Aug 1998 17:42:41 +1000 +Message-Id: <199804130742.RAA20366@mai1host.whitehouse.gov> +From: Bill Clinton +To: A1 (The Enforcer) Gore +Subject: Map of Argentina with Description +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="DC8------------DC8638F443D87A7F0726DEF7" + +This is a multi-part message in MIME format. +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Hi A1, + +I finally figured out this MIME thing. Pretty cool. I'll send you +some sax music in .au files next week! + +Anyway, the attached image is really too small to get a good look at +Argentina. Try this for a much better map: + +http://www.1one1yp1anet.com/dest/sam/graphics/map-arg.htm + +Then again, shouldn't the CIA have something like that? + +Bill +--DC8------------DC8638F443D87A7F0726DEF7 +Content-Type: image/gif; name="map_of_Argentina.gif" +Content-Transfer-Encoding: base64 +Content-Disposition: inline; filename="map_of_Argentina.gif" + +R01GOD1hJQA1AKIAAP/////78P/omn19fQAAAAAAAAAAAAAAACwAAAAAJQA1AAAD7Qi63P5w +wEmjBCLrnQnhYCgM1wh+pkgqqeC9XrutmBm7hAK3tP31gFcAiFKVQrGFR6kscnonTe7FAAad +GugmRu3CmiBt57fsVq3Y0VFKnpYdxPC6M7Ze4crnnHum4oN6LFJ1bn5NXTN7OF5fQkN5WYow +BEN2dkGQGWJtSzqGTICJgnQuTJN/WJsojad9qXMuhIWdjXKjY4tenjo6tjVssk2gaWq3uGNX +U6ZGxseyk8SasGw3J9GRzdTQky1iHNvcPNNI4TLeKdfMvy0vMqLrItvuxfDW8ubjueDtJufz +7itICBxISKDBgwgTKjyYAAA7 +--DC8------------DC8638F443D87A7F0726DEF7-- + +--D7F------------D7FD5A0B8AB9C65CCDBFA872-- + +` + +var email2 = `From: abc@def.de +Content-Type: multipart/mixed; + boundary="----_=_NextPart_001_01CBE273.65A0E7AA" +To: ghi@def.de + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01CBE273.65A0E7AA +Content-Type: multipart/alternative; + boundary="----_=_NextPart_002_01CBE273.65A0E7AA" + + +------_=_NextPart_002_01CBE273.65A0E7AA +Content-Type: text/plain; + charset="UTF-8" +Content-Transfer-Encoding: base64 + +[base64-content] +------_=_NextPart_002_01CBE273.65A0E7AA +Content-Type: text/html; + charset="UTF-8" +Content-Transfer-Encoding: base64 + +[base64-content] +------_=_NextPart_002_01CBE273.65A0E7AA-- +------_=_NextPart_001_01CBE273.65A0E7AA +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +X-MimeOLE: Produced By Microsoft Exchange V6.5 +Content-class: urn:content-classes:message +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----_=_NextPart_003_01CBE272.13692C80" +From: bla@bla.de +To: xxx@xxx.de + +This is a multi-part message in MIME format. + +------_=_NextPart_003_01CBE272.13692C80 +Content-Type: multipart/alternative; + boundary="----_=_NextPart_004_01CBE272.13692C80" + + +------_=_NextPart_004_01CBE272.13692C80 +Content-Type: text/plain; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +=20 + +Viele Gr=FC=DFe + +------_=_NextPart_004_01CBE272.13692C80 +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +... +------_=_NextPart_004_01CBE272.13692C80-- +------_=_NextPart_003_01CBE272.13692C80 +Content-Type: application/x-zip-compressed; + name="abc.zip" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="abc.zip" + +[base64-content] + +------_=_NextPart_003_01CBE272.13692C80-- +------_=_NextPart_001_01CBE273.65A0E7AA--` + +// note: this mime has an error: the boundary for multipart/alternative is re-used. +// it should use a new unique boundary marker, which then needs to be terminated after +// the text/html part. +var email3 = `MIME-Version: 1.0 +X-Mailer: MailBee.NET 8.0.4.428 +Subject: test subject +To: kevinm@datamotion.com +Content-Type: multipart/mixed; + boundary="XXXXboundary text" + +--XXXXboundary text +Content-Type: multipart/alternative; + boundary="XXXXboundary text" + +--XXXXboundary text +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +This is the body text of a sample message. +--XXXXboundary text +Content-Type: text/html; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +
This is the body text of a sample message.
+ +--XXXXboundary text +Content-Type: text/plain; + name="log_attachment.txt" +Content-Disposition: attachment; + filename="log_attachment.txt" +Content-Transfer-Encoding: base64 + +TUlNRS1WZXJzaW9uOiAxLjANClgtTWFpbGVyOiBNYWlsQmVlLk5FVCA4LjAuNC40MjgNClN1Ympl +Y3Q6IHRlc3Qgc3ViamVjdA0KVG86IGtldmlubUBkYXRhbW90aW9uLmNvbQ0KQ29udGVudC1UeXBl +OiBtdWx0aXBhcnQvYWx0ZXJuYXRpdmU7DQoJYm91bmRhcnk9Ii0tLS09X05leHRQYXJ0XzAwMF9B +RTZCXzcyNUUwOUFGLjg4QjdGOTM0Ig0KDQoNCi0tLS0tLT1fTmV4dFBhcnRfMDAwX0FFNkJfNzI1 +RTA5QUYuODhCN0Y5MzQNCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsNCgljaGFyc2V0PSJ1dGYt +OCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KdGVzdCBi +b2R5DQotLS0tLS09X05leHRQYXJ0XzAwMF9BRTZCXzcyNUUwOUFGLjg4QjdGOTM0DQpDb250ZW50 +LVR5cGU6IHRleHQvaHRtbDsNCgljaGFyc2V0PSJ1dGYtOCINCkNvbnRlbnQtVHJhbnNmZXItRW5j +b2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KPHByZT50ZXN0IGJvZHk8L3ByZT4NCi0tLS0tLT1f +TmV4dFBhcnRfMDAwX0FFNkJfNzI1RTA5QUYuODhCN0Y5MzQtLQ0K +--XXXXboundary text-- +` + +/* + +email 1 +Array +( + [0] => 1 + [1] => 1.1 + [2] => 1.2 + [3] => 1.2.1 + [4] => 1.2.1.1 + [5] => 1.2.1.2 +) +0 =>744 to 3029 +1 =>907 to 968 +2 =>1101 to 3029 +3 =>1889 to 3029 +4 =>2052 to 2402 +5 =>2594 to 2983 + +email 2 + +1 0 121 1763 1803 +1.1 207 302 628 668 +1.1.1 343 428 445 485 +1.1.2 485 569 586 628 +1.2 668 730 1763 1803 +1.2.1 730 959 1721 1763 +1.2.1.1 1045 1140 1501 1541 +1.2.1.1.1 1181 1281 1303 1343 +1.2.1.1.2 1343 1442 1459 1501 +1.2.1.2 1541 1703 1721 1763 +*/ +func TestNestedEmail(t *testing.T) { + p = NewMimeParser() + email = email3 + //email = strings.Replace(string(email), "\n", "\r\n", -1) + p.inject([]byte(email)) + + go func() { + time.Sleep(time.Second * 15) + + // for debugging deadlocks + //pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + //os.Exit(1) + }() + + if err := p.mime(nil, ""); err != nil { + t.Error(err) + } + //output := email + for part := range p.Parts { + //output = replaceAtIndex(output, '#', p.Parts[part].StartingPos) + //output = replaceAtIndex(output, '&', p.Parts[part].StartingPosBody) + //output = replaceAtIndex(output, '*', p.Parts[part].EndingPosBody) + fmt.Println(p.Parts[part].Node + + " " + strconv.Itoa(int(p.Parts[part].StartingPos)) + + " " + strconv.Itoa(int(p.Parts[part].StartingPosBody)) + + " " + strconv.Itoa(int(p.Parts[part].EndingPosBody)) + + " " + strconv.Itoa(int(p.Parts[part].EndingPos))) + } + //fmt.Print(output) + //fmt.Println(strings.Index(output, "--D7F------------D7FD5A0B8AB9C65CCDBFA872--")) + i := 1 + _ = i + //fmt.Println("[" + output[p.Parts[i].StartingPosBody:p.Parts[i].EndingPosBody] + "]") + //i := 2 + //fmt.Println("**********{" + output[p.parts[i].startingPosBody:p.parts[i].endingPosBody] + "}**********") + + //p.Close() + //p.inject([]byte(email)) + //if err := p.mime("", 1, ""); err != nil && err != io.EOF { + // t.Error(err) + //} + //p.Close() +} + +func replaceAtIndex(str string, replacement rune, index uint) string { + return str[:index] + string(replacement) + str[index+1:] +} + +var email4 = `Subject: test subject +To: kevinm@datamotion.com + +This is not a an MIME email +` + +func TestNonMineEmail(t *testing.T) { + p = NewMimeParser() + p.inject([]byte(email4)) + if err := p.mime(nil, ""); err != nil && err != NotMineErr { + t.Error(err) + } else { + // err should be NotMime + for part := range p.Parts { + fmt.Println(p.Parts[part].Node + " " + strconv.Itoa(int(p.Parts[part].StartingPos)) + " " + strconv.Itoa(int(p.Parts[part].StartingPosBody)) + " " + strconv.Itoa(int(p.Parts[part].EndingPosBody))) + } + } + err := p.Close() + if err != nil { + t.Error(err) + } + + // what if we pass an empty string? + p.inject([]byte{' '}) + if err := p.mime(nil, ""); err == nil || err == NotMineErr { + t.Error("unexpected error", err) + } + +} + +var email6 = `Delivered-To: nevaeh@sharklasers.com +Received: from bb_dyn_pb-146-88-38-36.violin.co.th (bb_dyn_pb-146-88-38-36.violin.co.th [146.88.38.36]) + by sharklasers.com with SMTP id d0e961595a207a79ab84603750372de8@sharklasers.com; + Tue, 17 Sep 2019 01:13:00 +0000 +Received: from mx03.listsystemsf.net [100.20.38.85] by mxs.perenter.com with SMTP; Tue, 17 Sep 2019 04:57:59 +0500 +Received: from mts.locks.grgtween.net ([Tue, 17 Sep 2019 04:52:27 +0500]) + by webmail.halftomorrow.com with LOCAL; Tue, 17 Sep 2019 04:52:27 +0500 +Received: from mail.naihautsui.co.kr ([61.220.30.1]) by mtu67.syds.piswix.net with ASMTP; Tue, 17 Sep 2019 04:47:25 +0500 +Received: from unknown (HELO mx03.listsystemsf.net) (Tue, 17 Sep 2019 04:41:45 +0500) + by smtp-server1.cfdenselr.com with LOCAL; Tue, 17 Sep 2019 04:41:45 +0500 +Message-ID: <78431AF2.E9B20F56@violin.co.th> +Date: Tue, 17 Sep 2019 04:14:56 +0500 +Reply-To: "Richard" +From: "Rick" +User-Agent: Mozilla 4.73 [de]C-CCK-MCD DT (Win98; U) +X-Accept-Language: en-us +MIME-Version: 1.0 +To: "Nevaeh" +Subject: Cesść tereska +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: base64 + +iVBORw0KGgoAAAANSUhEUgAAAG4AAAAyCAIAAAAydXkgAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA +B3RJTUUH1gYEExgGfYkXIAAAAAd0RVh0QXV0aG9yAKmuzEgAAAAMdEVYdERlc2NyaXB0aW9uABMJ +ISMAAAAKdEVYdENvcHlyaWdodACsD8w6AAAADnRFWHRDcmVhdGlvbiB0aW1lADX3DwkAAAAJdEVY +dFNvZnR3YXJlAF1w/zoAAAALdEVYdERpc2NsYWltZXIAt8C0jwAAAAh0RVh0V2FybmluZwDAG+aH +AAAAB3RFWHRTb3VyY2UA9f+D6wAAAAh0RVh0Q29tbWVudAD2zJa/AAAABnRFWHRUaXRsZQCo7tIn +AAABAElEQVR4nO2ZUY6DIBCG66YH88FGvQLHEI+hHsMriPFw7AMJIYAwoO269v+eSDPDmKn5HOXx +AAAAAAAAAPxblmWRZJZlSU3RCCE451Z1IUQ00c1ScM7p15zHT1J0URSpwUkpmrquh2HY60uA1+vl +/b2qKkp63tUCcA8otrK8k+dKr7+I1V0tEEUppRRCZDcnzZUZHLdP6g6uFomiBACYeHUTTnF9ZwV4 +3dp1HaW0V5dRUR6ZJU3e7kqLaK+9ZpymKamKOV3uTZrhigCAU1wZhV7aRE2IlKn2tq60WNeVHtz3 +vV7Xdc05b5pmL0ADVwLg5QOu3BNZhhxVwH1cmYoluwDqX2zbj2bPFgAAAMdJREFUNnUruBIALxmu +dF1mBXhlSimtPzW6O5hfIQOJB7mcK72NSzrk2bYt+ku0IvhL8PCKwxhTi3meT9s06aBGOSjjpduF +Ut1UnlnUUmG4kDtj6j5aa5c3noOfhX4ND1eXhvJMOYZFGYYxNs8zY6wsS73O3u2rUY1jjOkOBlp5 +uSf4NTn/fsw4Bz/oSnMMCm9laU4FuzMj5ZpN6K58JrVSfnAEW9d127ZxHInVLZM2TSOlpL/C72He +j2c+wQEAAAAAAAAAfB2/3ihTGANzPd8AAAAASUVORK5CYII= +` + +func TestNonMineEmailBigBody(t *testing.T) { + p = NewMimeParser() + b := []byte(email6) + to := 74 + var in [][]byte + for i := 0; ; i += to { + if to+i > len(b) { + in = append(in, b[i:]) + break + } + in = append(in, b[i:to+i]) + } + p.inject(in...) + if err := p.mime(nil, ""); err != nil && err != NotMineErr { + t.Error(err) + } else { + for part := range p.Parts { + fmt.Println(p.Parts[part].Node + " " + strconv.Itoa(int(p.Parts[part].StartingPos)) + " " + strconv.Itoa(int(p.Parts[part].StartingPosBody)) + " " + strconv.Itoa(int(p.Parts[part].EndingPosBody))) + } + } + err := p.Close() + if err != nil { + t.Error(err) + } + + // what if we pass an empty string? + p.inject([]byte{' '}) + if err := p.mime(nil, ""); err == nil || err == NotMineErr { + t.Error("unexpected error", err) + } + +} + +func TestMimeErr(t *testing.T) { + p := NewMimeParser() + p.Open() + // the error is missing subtype + data := + + `To "moo": j m +Subject: and a predicate +MIME-Version: 1.0 +Content-Type: text; +Content-Transfer-Encoding: 1 + +Rock the microphone and then I’m gone + +` + i, err := p.Write([]byte(data)) + + if err != nil { + if mimeErr, ok := err.(*Error); !ok { + t.Error("not a *MimeError type") + return + } else { + b, err := json.Marshal(mimeErr.Unwrap()) + if err != nil { + t.Error(err) + return + } + if string(b) != "8" { + t.Error("expecting error be 8") + return + } + var parsedErr MimeError + json.Unmarshal(b, &parsedErr) + if parsedErr != ErrorMissingSubtype { + t.Error("expecting error to be ErrorMissingSubtype, got:", parsedErr) + return + } + } + } + if i != 148 { + t.Error("test was expecting to read 148 bytes, got", i) + } + err = p.Close() + +} diff --git a/mail/reader.go b/mail/reader.go new file mode 100644 index 00000000..0a434333 --- /dev/null +++ b/mail/reader.go @@ -0,0 +1,74 @@ +package mail + +import ( + "bufio" + "io" + "net/textproto" + + "github.com/flashmob/go-guerrilla/mail/mimeparse" +) + +// MimeDotReader parses the mime structure while reading using the underlying reader +type MimeDotReader struct { + R io.Reader + p *mimeparse.Parser + mimeErr error +} + +// Read parses the mime structure wile reading. Results are immediately available in +// the data-structure returned from Parts() after each read. +func (r *MimeDotReader) Read(p []byte) (n int, err error) { + n, err = r.R.Read(p) + if n > 0 { + if r.mimeErr == nil { + r.mimeErr = r.p.Parse(p) + } + } + + return +} + +// Close closes the underlying reader if it's a ReadCloser and closes the mime parser +func (r MimeDotReader) Close() (err error) { + defer func() { + if err == nil && r.mimeErr != nil { + err = r.mimeErr + } + }() + if rc, t := r.R.(io.ReadCloser); t { + err = rc.Close() + } + // parser already closed? + if r.mimeErr != nil { + return + } + // close the parser + r.mimeErr = r.p.Close() + return +} + +// Parts returns the mime-header parts built by the parser +func (r *MimeDotReader) Parts() mimeparse.Parts { + return r.p.Parts +} + +// Returns the underlying io.Reader (which is a dotReader from textproto) +// useful for reading from directly if mime parsing is not desirable. +func (r *MimeDotReader) DotReader() io.Reader { + return r.R +} + +// NewMimeDotReader returns a pointer to a new MimeDotReader +// br is the underlying reader it will read from +// maxNodes limits the number of nodes can be added to the mime tree before the mime-parser aborts +func NewMimeDotReader(br *bufio.Reader, maxNodes int) *MimeDotReader { + r := new(MimeDotReader) + r.R = textproto.NewReader(br).DotReader() + if maxNodes > 0 { + r.p = mimeparse.NewMimeParserLimited(maxNodes) + } else { + r.p = mimeparse.NewMimeParser() + } + r.p.Open() + return r +} diff --git a/mail/rfc5321/address.go b/mail/smtp/address.go similarity index 99% rename from mail/rfc5321/address.go rename to mail/smtp/address.go index fd8dde02..80ce259d 100644 --- a/mail/rfc5321/address.go +++ b/mail/smtp/address.go @@ -1,4 +1,4 @@ -package rfc5321 +package smtp import ( "errors" diff --git a/mail/rfc5321/address_test.go b/mail/smtp/address_test.go similarity index 99% rename from mail/rfc5321/address_test.go rename to mail/smtp/address_test.go index 7d227d2d..e33045df 100644 --- a/mail/rfc5321/address_test.go +++ b/mail/smtp/address_test.go @@ -1,4 +1,4 @@ -package rfc5321 +package smtp import ( "testing" diff --git a/mail/rfc5321/parse.go b/mail/smtp/parse.go similarity index 89% rename from mail/rfc5321/parse.go rename to mail/smtp/parse.go index 1d22b8a4..ea78a703 100644 --- a/mail/rfc5321/parse.go +++ b/mail/smtp/parse.go @@ -1,4 +1,4 @@ -package rfc5321 +package smtp // Parse RFC5321 productions, no regex @@ -22,21 +22,81 @@ const ( LimitRecipients = 100 ) +type PathParam []string + +// A TransportType specifies the message transport according to https://tools.ietf.org/html/rfc6152 +type TransportType int + +const ( + TransportType7bit TransportType = iota + TransportType8bit + TransportTypeUnspecified + TransportTypeInvalid +) + +func (t TransportType) String() string { + switch t { + case TransportType7bit: + return "7bit" + case TransportType8bit: + return "8bit" + case TransportTypeUnspecified: + return "unknown" + case TransportTypeInvalid: + return "invalid" + } + return "invalid" +} + +func ParseTransportType(str string) TransportType { + switch { + case str == "7bit": + return TransportType7bit + case str == "8bit": + return TransportType8bit + case str == "unknown": + return TransportTypeUnspecified + case str == "invalid": + return TransportTypeInvalid + } + return TransportTypeInvalid +} + +// is8BitMime checks for the BODY parameter as +func (p PathParam) Transport() TransportType { + if len(p) != 2 { + return TransportTypeUnspecified + } + if strings.ToUpper(p[0]) != "BODY" { + // this is not a 'BODY' param + return TransportTypeUnspecified + } + if strings.ToUpper(p[1]) == "8BITMIME" { + return TransportType8bit + } else if strings.ToUpper(p[1]) == "7BIT" { + return TransportType7bit + } + return TransportTypeInvalid +} + var atExpected = errors.New("@ expected as part of mailbox") // Parse Email Addresses according to https://tools.ietf.org/html/rfc5321 type Parser struct { - accept bytes.Buffer - buf []byte - PathParams [][]string - ADL []string + NullPath bool LocalPart string - LocalPartQuotes bool // does the local part need quotes? - Domain string // can be an ip-address, enclosed in square brackets if it is + LocalPartQuotes bool // does the local part need quotes? + Domain string IP net.IP - pos int - NullPath bool - ch byte + + ADL []string + PathParams []PathParam + + pos int + ch byte + + buf []byte + accept bytes.Buffer } func NewParser(buf []byte) *Parser { @@ -153,8 +213,8 @@ func (s *Parser) RcptTo(input []byte) (err error) { } // esmtp-param *(SP esmtp-param) -func (s *Parser) parameters() ([][]string, error) { - params := make([][]string, 0) +func (s *Parser) parameters() ([]PathParam, error) { + params := make([]PathParam, 0) for { if result, err := s.param(); err != nil { return params, err diff --git a/mail/rfc5321/parse_test.go b/mail/smtp/parse_test.go similarity index 95% rename from mail/rfc5321/parse_test.go rename to mail/smtp/parse_test.go index 4311371f..c3b75a06 100644 --- a/mail/rfc5321/parse_test.go +++ b/mail/smtp/parse_test.go @@ -1,4 +1,4 @@ -package rfc5321 +package smtp import ( "strings" @@ -646,3 +646,30 @@ func TestHelo(t *testing.T) { t.Error("expecting domain exam_ple.com to be invalid") } } + +func TestTransport(t *testing.T) { + + path := PathParam([]string{"BODY", "8bitmime"}) + transport := path.Transport() + if transport != TransportType8bit { + t.Error("transport was not 8bit") + } + + path = []string{"BODY", "7bit"} + transport = path.Transport() + if transport != TransportType7bit { + t.Error("transport was not 7bit") + } + + path = []string{"BODY", "invalid"} + transport = path.Transport() + if transport != TransportTypeInvalid { + t.Error("transport was not invalid") + } + + path = []string{} + transport = path.Transport() + if transport != TransportTypeUnspecified { + t.Error("transport was not unspecified") + } +} diff --git a/pool.go b/pool.go index 6ee2ce87..94265df7 100644 --- a/pool.go +++ b/pool.go @@ -127,7 +127,7 @@ func (p *Pool) GetActiveClientsCount() int { } // Borrow a Client from the pool. Will block if len(activeClients) > maxClients -func (p *Pool) Borrow(conn net.Conn, clientID uint64, logger log.Logger, ep *mail.Pool) (Poolable, error) { +func (p *Pool) Borrow(conn net.Conn, clientID uint64, logger log.Logger, ep *mail.Pool, serverID int) (Poolable, error) { p.poolGuard.Lock() defer p.poolGuard.Unlock() @@ -142,7 +142,7 @@ func (p *Pool) Borrow(conn net.Conn, clientID uint64, logger log.Logger, ep *mai case c = <-p.pool: c.init(conn, clientID, ep) default: - c = NewClient(conn, clientID, logger, ep) + c = NewClient(conn, clientID, logger, ep, serverID) } p.activeClientsAdd(c) diff --git a/server.go b/server.go index 1ab96b5b..786b1dd5 100644 --- a/server.go +++ b/server.go @@ -6,7 +6,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "github.com/sirupsen/logrus" "io" "io/ioutil" "net" @@ -19,7 +18,7 @@ import ( "github.com/flashmob/go-guerrilla/backends" "github.com/flashmob/go-guerrilla/log" "github.com/flashmob/go-guerrilla/mail" - "github.com/flashmob/go-guerrilla/mail/rfc5321" + "github.com/flashmob/go-guerrilla/mail/smtp" "github.com/flashmob/go-guerrilla/response" ) @@ -47,6 +46,7 @@ type server struct { tlsConfigStore atomic.Value timeout atomic.Value // stores time.Duration listenInterface string + serverID int clientPool *Pool wg sync.WaitGroup // for waiting to shutdown listener net.Listener @@ -84,28 +84,29 @@ var ( ) func (c command) match(in []byte) bool { - return bytes.Index(in, []byte(c)) == 0 + return bytes.Index(in, c) == 0 } // Creates and returns a new ready-to-run Server from a ServerConfig configuration -func newServer(sc *ServerConfig, b backends.Backend, mainlog log.Logger) (*server, error) { +func newServer(sc *ServerConfig, b backends.Backend, mainlog log.Logger, serverID int) (*server, error) { server := &server{ clientPool: NewPool(sc.MaxClients), closedListener: make(chan bool, 1), listenInterface: sc.ListenInterface, + serverID: serverID, state: ServerStateNew, - envelopePool: mail.NewPool(sc.MaxClients), + envelopePool: mail.NewPool(sc.MaxClients * 2), } server.mainlogStore.Store(mainlog) server.backendStore.Store(b) if sc.LogFile == "" { // none set, use the mainlog for the server log server.logStore.Store(mainlog) - server.log().Info("server [" + sc.ListenInterface + "] did not configure a separate log file, so using the main log") + server.log().Fields("iface", sc.ListenInterface).Info("server did not configure a separate log file, so using the main log") } else { // set level to same level as mainlog level if l, logOpenError := log.GetLogger(sc.LogFile, server.mainlog().GetLevel()); logOpenError != nil { - server.log().WithError(logOpenError).Errorf("Failed creating a logger for server [%s]", sc.ListenInterface) + server.log().Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for server") return server, logOpenError } else { server.logStore.Store(l) @@ -158,7 +159,8 @@ func (s *server) configureTLS() error { if len(sConfig.TLS.RootCAs) > 0 { caCert, err := ioutil.ReadFile(sConfig.TLS.RootCAs) if err != nil { - s.log().WithError(err).Errorf("failed opening TLSRootCAs file [%s]", sConfig.TLS.RootCAs) + s.log().Fields("error", err, "file", sConfig.TLS.RootCAs).Error("failed opening TLSRootCAs file") + return err } else { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -239,29 +241,30 @@ func (s *server) Start(startWG *sync.WaitGroup) error { if err != nil { startWG.Done() // don't wait for me s.state = ServerStateStartError - return fmt.Errorf("[%s] Cannot listen on port: %s ", s.listenInterface, err.Error()) + return fmt.Errorf("[%s] cannot listen on port: %s ", s.listenInterface, err.Error()) } - s.log().Infof("Listening on TCP %s", s.listenInterface) + s.log().Fields("iface", s.listenInterface, "serverID", s.serverID).Info("listening on TCP") s.state = ServerStateRunning startWG.Done() // start successful, don't wait for me for { - s.log().Debugf("[%s] Waiting for a new client. Next Client ID: %d", s.listenInterface, clientID+1) + s.log().Fields("serverID", s.serverID, "nextSeq", clientID+1, "iface", s.listenInterface). + Debug("waiting for a new client") conn, err := listener.Accept() clientID++ if err != nil { if e, ok := err.(net.Error); ok && !e.Temporary() { - s.log().Infof("Server [%s] has stopped accepting new clients", s.listenInterface) + s.log().Fields("iface", s.listenInterface, "serverID", s.serverID).Info("server has stopped accepting new clients") // the listener has been closed, wait for clients to exit - s.log().Infof("shutting down pool [%s]", s.listenInterface) + s.log().Fields("iface", s.listenInterface, "serverID", s.serverID).Info("shutting down pool") s.clientPool.ShutdownState() s.clientPool.ShutdownWait() s.state = ServerStateStopped s.closedListener <- true return nil } - s.mainlog().WithError(err).Info("Temporary error accepting client") + s.mainlog().Fields("error", err, "serverID", s.serverID).Error("temporary error accepting client") continue } go func(p Poolable, borrowErr error) { @@ -271,15 +274,13 @@ func (s *server) Start(startWG *sync.WaitGroup) error { s.envelopePool.Return(c.Envelope) s.clientPool.Return(c) } else { - s.log().WithError(borrowErr).Info("couldn't borrow a new client") + s.log().Fields("error", borrowErr, "serverID", s.serverID).Error("couldn't borrow a new client") // we could not get a client, so close the connection. _ = conn.Close() - } // intentionally placed Borrow in args so that it's called in the // same main goroutine. - }(s.clientPool.Borrow(conn, clientID, s.log(), s.envelopePool)) - + }(s.clientPool.Borrow(conn, clientID, s.log(), s.envelopePool, s.serverID)) } } @@ -360,11 +361,33 @@ func (s *server) isShuttingDown() bool { return s.clientPool.IsShuttingDown() } -// Handles an entire client SMTP exchange +const advertisePipelining = "250-PIPELINING\r\n" +const advertiseStartTLS = "250-STARTTLS\r\n" +const advertiseEnhancedStatusCodes = "250-ENHANCEDSTATUSCODES\r\n" +const advertise8BitMime = "250-8BITMIME\r\n" + +// The last line doesn't need \r\n since string will be printed as a new line. +// Also, Last line has no dash - +const advertiseHelp = "250 HELP" + +// handleClient handles an entire client SMTP exchange func (s *server) handleClient(client *client) { - defer client.closeConn() + defer func() { + s.log().Fields( + "peer", client.RemoteIP, + "event", "disconnect", + "id", client.ID, + "queuedID", client.QueuedId, + ).Info("Disconnect client") + client.closeConn() + }() sc := s.configStore.Load().(ServerConfig) - s.log().Infof("Handle client [%s], id: %d", client.RemoteIP, client.ID) + s.log().Fields( + "peer", client.RemoteIP, + "id", client.ID, + "event", "connect", + "queuedID", client.QueuedId, + ).Info("handle client") // Initial greeting greeting := fmt.Sprintf("220 %s SMTP Guerrilla(%s) #%d (%d) %s", @@ -377,12 +400,9 @@ func (s *server) handleClient(client *client) { // Extended feature advertisements messageSize := fmt.Sprintf("250-SIZE %d\r\n", sc.MaxSize) - pipelining := "250-PIPELINING\r\n" - advertiseTLS := "250-STARTTLS\r\n" - advertiseEnhancedStatusCodes := "250-ENHANCEDSTATUSCODES\r\n" + advertiseTLS := advertiseStartTLS // The last line doesn't need \r\n since string will be printed as a new line. // Also, Last line has no dash - - help := "250 HELP" if sc.TLS.AlwaysOn { tlsConfig, ok := s.tlsConfigStore.Load().(*tls.Config) @@ -391,7 +411,7 @@ func (s *server) handleClient(client *client) { } else if err := client.upgradeToTLS(tlsConfig); err == nil { advertiseTLS = "" } else { - s.log().WithError(err).Warnf("[%s] Failed TLS handshake", client.RemoteIP) + s.log().Fields("error", err, "peer", client.RemoteIP).Warn("failed TLS handshake") // server requires TLS, but can't handshake client.kill() } @@ -409,19 +429,19 @@ func (s *server) handleClient(client *client) { case ClientCmd: client.bufin.setLimit(CommandLineMaxLength) input, err := s.readCommand(client) - s.log().Debugf("Client sent: %s", input) + s.log().Fields("input", string(input)).Debug("client said") if err == io.EOF { - s.log().WithError(err).Warnf("Client closed the connection: %s", client.RemoteIP) + s.log().Fields("error", err, "peer", client.RemoteIP).Warn("client closed the connection") return } else if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - s.log().WithError(err).Warnf("Timeout: %s", client.RemoteIP) + s.log().Fields("error", err, "peer", client.RemoteIP).Warn("timeout") return } else if err == LineLimitExceeded { client.sendResponse(r.FailLineTooLong) client.kill() break } else if err != nil { - s.log().WithError(err).Warnf("Read error: %s", client.RemoteIP) + s.log().Fields("error", err, "peer", client.RemoteIP).Warn("read error") client.kill() break } @@ -440,7 +460,7 @@ func (s *server) handleClient(client *client) { if h, err := client.parser.Helo(input[4:]); err == nil { client.Helo = h } else { - s.log().WithFields(logrus.Fields{"helo": h, "client": client.ID}).Warn("invalid helo") + s.log().Fields("helo", h, "seq", client.ID).Warn("invalid helo") client.sendResponse(r.FailSyntaxError) break } @@ -452,7 +472,7 @@ func (s *server) handleClient(client *client) { client.Helo = h } else { client.sendResponse(r.FailSyntaxError) - s.log().WithFields(logrus.Fields{"ehlo": h, "client": client.ID}).Warn("invalid ehlo") + s.log().Fields("ehlo", h, "seq", client.ID).Warn("invalid ehlo") client.sendResponse(r.FailSyntaxError) break } @@ -460,10 +480,11 @@ func (s *server) handleClient(client *client) { client.resetTransaction() client.sendResponse(ehlo, messageSize, - pipelining, + advertisePipelining, advertiseTLS, advertiseEnhancedStatusCodes, - help) + advertise8BitMime, + advertiseHelp) case cmdHELP.match(cmd): quote := response.GetQuote() @@ -494,23 +515,42 @@ func (s *server) handleClient(client *client) { } client.MailFrom, err = client.parsePath(input[10:], client.parser.MailFrom) if err != nil { - s.log().WithError(err).Error("MAIL parse error", "["+string(input[10:])+"]") + s.log().Fields("error", err, "raw", string(input[10:])).Error("MAIL parse error") client.sendResponse(err) break } else if client.parser.NullPath { // bounce has empty from address client.MailFrom = mail.Address{} + } else { + s.log().Fields( + "event", "mailfrom", + "helo", client.Helo, + "domain", client.MailFrom.Host, + "address", client.RemoteIP, + "id", client.ID, + "queuedID", client.QueuedId, + ).Info("mail from") + } + client.TransportType = smtp.TransportTypeUnspecified + for i := range client.MailFrom.PathParams { + if tt := client.MailFrom.PathParams[i].Transport(); tt != smtp.TransportTypeUnspecified { + client.TransportType = tt + if tt == smtp.TransportTypeInvalid { + continue + } + break + } } client.sendResponse(r.SuccessMailCmd) case cmdRCPT.match(cmd): - if len(client.RcptTo) > rfc5321.LimitRecipients { + if len(client.RcptTo) > smtp.LimitRecipients { client.sendResponse(r.ErrorTooManyRecipients) break } to, err := client.parsePath(input[8:], client.parser.RcptTo) if err != nil { - s.log().WithError(err).Error("RCPT parse error", "["+string(input[8:])+"]") + s.log().Fields("error", err, "raw", string(input[8:])).Error("RCPT parse error") client.sendResponse(err.Error()) break } @@ -563,17 +603,57 @@ func (s *server) handleClient(client *client) { client.sendResponse(r.FailUnrecognizedCmd) } } - case ClientData: - // intentionally placed the limit 1MB above so that reading does not return with an error // if the client goes a little over. Anything above will err client.bufin.setLimit(sc.MaxSize + 1024000) // This a hard limit. - - n, err := client.Data.ReadFrom(client.smtpReader.DotReader()) - if n > sc.MaxSize { - err = fmt.Errorf("maximum DATA size exceeded (%d)", sc.MaxSize) + be := s.backend() + var ( + n int64 + err error + res backends.Result + ) + fields := []interface{}{ + "event", "data", + "id", client.ID, + "queuedID", client.QueuedId, + "messageID", client.MessageID, + "peer", client.RemoteIP, + "serverID", s.serverID, } + s.log().Fields(fields...).Info("receive DATA") + if be.StreamOn() { + // process the message as a stream + res, n, err = be.ProcessStream(client.smtpReader.DotReader(), client.Envelope) + if err == nil && res.Code() < 300 { + e := s.envelopePool.Borrow( + client.Envelope.RemoteIP, + client.ID, + client.Envelope.ServerID, + ) + s.copyEnvelope(client.Envelope, e) + // process in the background then return the envelope + go func() { + be.ProcessBackground(e) + s.envelopePool.Return(e) + }() + } + } else { + // or buffer the entire message (parse headers & mime structure as we go along) + n, err = client.Data.ReadFrom(client.smtpReader) + if n > sc.MaxSize { + err = fmt.Errorf("maximum DATA size exceeded (%d)", sc.MaxSize) + } else { + if p := client.smtpReader.Parts(); p != nil && len(p) > 0 { + client.Envelope.Header = p[0].Headers + } + } + // All done. we can close the smtpReader, the protocol will reset the transaction, expecting a new message + if closeErr := client.smtpReader.Close(); closeErr != nil { + s.log().WithError(closeErr).Error("could not close DATA reader") + } + } + if err != nil { if err == LineLimitExceeded { client.sendResponse(r.FailReadLimitExceededDataCmd, " ", LineLimitExceeded.Error()) @@ -585,14 +665,18 @@ func (s *server) handleClient(client *client) { client.sendResponse(r.FailReadErrorDataCmd, " ", err.Error()) client.kill() } - s.log().WithError(err).Warn("Error reading data") + s.log().Fields(append(fields, "error", err)...).Error("error reading DATA") client.resetTransaction() break } - res := s.backend().Process(client.Envelope) + if !be.StreamOn() { + res = be.Process(client.Envelope) + } + if res.Code() < 300 { client.messagesSent++ + s.log().Fields(append(fields, "length", n)...).Info("received message DATA") } client.sendResponse(res) client.state = ClientCmd @@ -605,12 +689,13 @@ func (s *server) handleClient(client *client) { if !client.TLS && sc.TLS.StartTLSOn { tlsConfig, ok := s.tlsConfigStore.Load().(*tls.Config) if !ok { - s.mainlog().Error("Failed to load *tls.Config") + s.mainlog().Fields("iface", s.listenInterface).Error("failed to load *tls.Config") } else if err := client.upgradeToTLS(tlsConfig); err == nil { advertiseTLS = "" client.resetTransaction() } else { - s.log().WithError(err).Warnf("[%s] Failed TLS handshake", client.RemoteIP) + s.log().Fields("error", err, "iface", s.listenInterface, "ip", client.RemoteIP). + Warn("failed TLS handshake") // Don't disconnect, let the client decide if it wants to continue } } @@ -629,7 +714,7 @@ func (s *server) handleClient(client *client) { // flush the response buffer if client.bufout.Buffered() > 0 { if s.log().IsDebug() { - s.log().Debugf("Writing response to client: \n%s", client.response.String()) + s.log().Fields("out", client.response.String()).Debug("writing response to client") } err := s.flushResponse(client) if err != nil { @@ -675,9 +760,18 @@ func (s *server) defaultHost(a *mail.Address) { sc := s.configStore.Load().(ServerConfig) a.Host = sc.Hostname if !s.allowsHost(a.Host) { - s.log().WithFields( - logrus.Fields{"hostname": sc.Hostname}). - Warn("the hostname is not present in AllowedHosts config setting") + s.log().Fields("hostname", sc.Hostname). + Warn("the hostname is not present in the AllowedHosts config setting") } } } + +func (s *server) copyEnvelope(src *mail.Envelope, dest *mail.Envelope) { + dest.TLS = src.TLS + dest.Helo = src.Helo + dest.ESMTP = src.ESMTP + dest.RcptTo = src.RcptTo + dest.MailFrom = src.MailFrom + dest.MessageID = src.MessageID + dest.TransportType = src.TransportType +} diff --git a/server_test.go b/server_test.go index f1067836..81e051fd 100644 --- a/server_test.go +++ b/server_test.go @@ -1,18 +1,16 @@ package guerrilla import ( - "os" - "testing" - "bufio" - "net/textproto" - "strings" - "sync" - "crypto/tls" "fmt" "io/ioutil" "net" + "net/textproto" + "os" + "strings" + "sync" + "testing" "github.com/flashmob/go-guerrilla/backends" "github.com/flashmob/go-guerrilla/log" @@ -48,15 +46,25 @@ func getMockServerConn(sc *ServerConfig, t *testing.T) (*mocks.Conn, *server) { var mainlog log.Logger mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") + } + + bcfg := backends.BackendConfig{ + backends.ConfigProcessors: { + "debugger": {"log_received_mails": true}, + }, + backends.ConfigGateways: { + backends.DefaultGateway: {"save_workers_size": 1}, + }, } - backend, err := backends.New( - backends.BackendConfig{"log_received_mails": true, "save_workers_size": 1}, + + backend, err := backends.New(backends.DefaultGateway, + bcfg, mainlog) if err != nil { t.Error("new dummy backend failed because:", err) } - server, err := newServer(sc, backend, mainlog) + server, err := newServer(sc, backend, mainlog, 0) if err != nil { //t.Error("new server failed because:", err) } else { @@ -269,11 +277,11 @@ func TestHandleClient(t *testing.T) { sc := getMockServerConfig() mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) // call the serve.handleClient() func in a goroutine. - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -309,7 +317,7 @@ func TestGithubIssue197(t *testing.T) { sc := getMockServerConfig() mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) server.backend().Start() @@ -317,7 +325,7 @@ func TestGithubIssue197(t *testing.T) { // [2001:DB8::FF00:42:8329] is an address literal server.setAllowedHosts([]string{"1.1.1.1", "[2001:DB8::FF00:42:8329]"}) - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -411,11 +419,15 @@ func TestGithubIssue198(t *testing.T) { backends.Svc.AddProcessor("custom", customBackend) if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) - be, err := backends.New(map[string]interface{}{ - "save_process": "HeadersParser|Header|custom", "primary_mail_host": "example.com"}, + cfg := backends.BackendConfig{} + cfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_process", "HeadersParser|Header|debugger|custom") + cfg.SetValue(backends.ConfigProcessors, "header", "primary_mail_host", "example.com") + cfg.SetValue(backends.ConfigProcessors, "debugger", "log_received_mails", true) + + be, err := backends.New("default", cfg, mainlog) if err != nil { t.Error(err) @@ -429,7 +441,7 @@ func TestGithubIssue198(t *testing.T) { server.setAllowedHosts([]string{"1.1.1.1", "[2001:DB8::FF00:42:8329]"}) - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) client.RemoteIP = "127.0.0.1" var wg sync.WaitGroup @@ -504,6 +516,9 @@ func sendMessage(greet string, TLS bool, w *textproto.Writer, t *testing.T, line t.Error(err) } } + if r.R.Buffered() > 0 { + line, _ = r.ReadLine() + } if err := w.PrintfLine("MAIL FROM: test@example.com>"); err != nil { t.Error(err) @@ -514,17 +529,25 @@ func sendMessage(greet string, TLS bool, w *textproto.Writer, t *testing.T, line t.Error(err) } line, _ = r.ReadLine() - client.Hashes = append(client.Hashes, "abcdef1526777763") + client.Hashes = append(client.Hashes, "abcdef1526777763"+greet) client.TLS = TLS + client.QueuedId = mail.QueuedID(1, 1) if err := w.PrintfLine("DATA"); err != nil { t.Error(err) } line, _ = r.ReadLine() + if greet == "EHLO" { + } - if err := w.PrintfLine("Subject: Test subject\r\n\r\nHello Sir,\nThis is a test.\r\n."); err != nil { + if err := w.PrintfLine("Subject: Test subject" + greet + "\r\n\r\nHello Sir,\nThis is a test.\r\n."); err != nil { t.Error(err) } + if r.R.Buffered() > 0 { + line, _ = r.ReadLine() + } + line, _ = r.ReadLine() + return line } @@ -535,14 +558,14 @@ func TestGithubIssue199(t *testing.T) { sc := getMockServerConfig() mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) server.backend().Start() server.setAllowedHosts([]string{"grr.la", "fake.com", "[1.1.1.1]", "[2001:db8::8a2e:370:7334]", "saggydimes.test.com"}) - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -714,13 +737,13 @@ func TestGithubIssue200(t *testing.T) { sc := getMockServerConfig() mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) server.backend().Start() server.setAllowedHosts([]string{"1.1.1.1", "[2001:DB8::FF00:42:8329]"}) - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -766,7 +789,7 @@ func TestGithubIssue201(t *testing.T) { sc := getMockServerConfig() mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) server.backend().Start() @@ -774,7 +797,7 @@ func TestGithubIssue201(t *testing.T) { // it will be used for rcpt to: which does not specify a host server.setAllowedHosts([]string{"a.com", "saggydimes.test.com"}) - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -854,11 +877,11 @@ func TestXClient(t *testing.T) { sc.XClientOn = true mainlog, logOpenError = log.GetLogger(sc.LogFile, "debug") if logOpenError != nil { - mainlog.WithError(logOpenError).Errorf("Failed creating a logger for mock conn [%s]", sc.ListenInterface) + mainlog.Fields("error", logOpenError, "iface", sc.ListenInterface).Error("Failed creating a logger for mock conn") } conn, server := getMockServerConn(sc, t) // call the serve.handleClient() func in a goroutine. - client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5)) + client := NewClient(conn.Server, 1, mainlog, mail.NewPool(5), 0) var wg sync.WaitGroup wg.Add(1) go func() { @@ -911,15 +934,15 @@ func TestXClient(t *testing.T) { // a second transaction func TestGatewayTimeout(t *testing.T) { defer cleanTestArtifacts(t) - bcfg := backends.BackendConfig{ - "save_workers_size": 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": true, - "primary_mail_host": "example.com", - "gw_save_timeout": "1s", - "gw_val_rcpt_timeout": "1s", - "sleep_seconds": 2, - } + + bcfg := backends.BackendConfig{} + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_timeout", "1s") + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "val_rcpt_timeout", "1s") + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_workers_size", 1) + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_process", "HeadersParser|Debugger") + bcfg.SetValue(backends.ConfigProcessors, "header", "primary_mail_host", "example.com") + bcfg.SetValue(backends.ConfigProcessors, "debugger", "log_received_mails", true) + bcfg.SetValue(backends.ConfigProcessors, "debugger", "sleep_seconds", 2) cfg := &AppConfig{ LogFile: log.OutputOff.String(), @@ -951,7 +974,7 @@ func TestGatewayTimeout(t *testing.T) { // perform 2 transactions // both should panic. for i := 0; i < 2; i++ { - if _, err := fmt.Fprint(conn, "MAIL FROM:r\r\n"); err != nil { + if _, err := fmt.Fprint(conn, "MAIL FROM:\r\n"); err != nil { t.Error(err) } if str, err = in.ReadString('\n'); err != nil { @@ -998,15 +1021,15 @@ func TestGatewayTimeout(t *testing.T) { // The processor will panic and gateway should recover from it func TestGatewayPanic(t *testing.T) { defer cleanTestArtifacts(t) - bcfg := backends.BackendConfig{ - "save_workers_size": 1, - "save_process": "HeadersParser|Debugger", - "log_received_mails": true, - "primary_mail_host": "example.com", - "gw_save_timeout": "2s", - "gw_val_rcpt_timeout": "2s", - "sleep_seconds": 1, - } + + bcfg := backends.BackendConfig{} + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_timeout", "2s") + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "val_rcpt_timeout", "2s") + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_workers_size", 1) + bcfg.SetValue(backends.ConfigGateways, backends.DefaultGateway, "save_process", "HeadersParser|Debugger") + bcfg.SetValue(backends.ConfigProcessors, "header", "primary_mail_host", "example.com") + bcfg.SetValue(backends.ConfigProcessors, "debugger", "log_received_mails", true) + bcfg.SetValue(backends.ConfigProcessors, "debugger", "sleep_seconds", 1) cfg := &AppConfig{ LogFile: log.OutputOff.String(), @@ -1041,7 +1064,7 @@ func TestGatewayPanic(t *testing.T) { // sure that the client waits until processing finishes, and the // timeout event is captured. for i := 0; i < 2; i++ { - if _, err := fmt.Fprint(conn, "MAIL FROM:r\r\n"); err != nil { + if _, err := fmt.Fprint(conn, "MAIL FROM:\r\n"); err != nil { t.Error(err) } if _, err = in.ReadString('\n'); err != nil { diff --git a/tests/guerrilla_test.go b/tests/guerrilla_test.go index d6ba5b77..6b4c4639 100644 --- a/tests/guerrilla_test.go +++ b/tests/guerrilla_test.go @@ -16,7 +16,7 @@ package test import ( "encoding/json" - "github.com/flashmob/go-guerrilla/mail/rfc5321" + "github.com/flashmob/go-guerrilla/mail/smtp" "testing" "time" @@ -41,7 +41,7 @@ import ( type TestConfig struct { guerrilla.AppConfig - BackendConfig map[string]interface{} `json:"backend_config"` + BackendConfig backends.BackendConfig `json:"backend"` } var ( @@ -68,7 +68,7 @@ func init() { return } backend, _ := getBackend(config.BackendConfig, logger) - app, initErr = guerrilla.New(&config.AppConfig, backend, logger) + app, initErr = guerrilla.New(&config.AppConfig, logger, backend) } } @@ -80,10 +80,14 @@ var configJson = ` "log_level" : "debug", "pid_file" : "go-guerrilla.pid", "allowed_hosts": ["spam4.me","grr.la"], - "backend_config" : - { - "log_received_mails" : true - }, + + "backend" : { + "processors" : { + "debugger" : { + "log_received_mails" : true + } + } + }, "servers" : [ { "is_enabled" : true, @@ -120,8 +124,9 @@ var configJson = ` } ` -func getBackend(backendConfig map[string]interface{}, l log.Logger) (backends.Backend, error) { - b, err := backends.New(backendConfig, l) +func getBackend(backendConfig backends.BackendConfig, l log.Logger) (backends.Backend, error) { + _ = backendConfig.ConfigureDefaults() + b, err := backends.New(backends.DefaultGateway, backendConfig, l) if err != nil { fmt.Println("backend init error", err) os.Exit(1) @@ -181,6 +186,39 @@ func cleanTestArtifacts(t *testing.T) { } +func TestMatchConfig(t *testing.T) { + str := ` +time="2020-07-20T14:14:17+09:00" level=info msg="pid_file written" file=tests/go-guerrilla.pid pid=15247 +time="2020-07-20T14:14:17+09:00" level=debug msg="making servers" +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=default id=3 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=default id=2 +time="2020-07-20T14:14:17+09:00" level=info msg="starting server" iface="127.0.0.1:2526" serverID=0 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=default id=1 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=temp id=2 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=default id=4 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=temp id=3 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=temp id=4 +time="2020-07-20T14:14:17+09:00" level=info msg="processing worker started" gateway=temp id=1 +time="2020-07-20T14:14:17+09:00" level=info msg="listening on TCP" iface="127.0.0.1:2526" serverID=0 +time="2020-07-20T14:14:17+09:00" level=debug msg="waiting for a new client" nextSeq=1 serverID=0 + + +` + defer cleanTestArtifacts(t) + if !MatchLog(str, 1, "msg", "making servers") { + t.Error("making servers not matched") + } + + if MatchLog(str, 10, "msg", "making servers") { + t.Error("not expecting making servers matched") + } + + if !MatchLog(str, 1, "msg", "listening on TCP", "serverID", 0) { + t.Error("2 not pairs matched") + } + +} + // Testing start and stop of server func TestStart(t *testing.T) { if initErr != nil { @@ -196,40 +234,48 @@ func TestStart(t *testing.T) { app.Shutdown() if read, err := ioutil.ReadFile("./testlog"); err == nil { logOutput := string(read) - if i := strings.Index(logOutput, "Listening on TCP 127.0.0.1:4654"); i < 0 { + if !MatchLog(logOutput, 1, "msg", "listening on TCP", "iface", "127.0.0.1:2526") { t.Error("Server did not listen on 127.0.0.1:4654") } - if i := strings.Index(logOutput, "Listening on TCP 127.0.0.1:2526"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "listening on TCP", "iface", "127.0.0.1:2526") { t.Error("Server did not listen on 127.0.0.1:2526") } - if i := strings.Index(logOutput, "[127.0.0.1:4654] Waiting for a new client"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "waiting for a new client", "iface", "127.0.0.1:4654") { t.Error("Server did not wait on 127.0.0.1:4654") } - if i := strings.Index(logOutput, "[127.0.0.1:2526] Waiting for a new client"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "waiting for a new client", "iface", "127.0.0.1:2526") { t.Error("Server did not wait on 127.0.0.1:2526") } - if i := strings.Index(logOutput, "Server [127.0.0.1:4654] has stopped accepting new clients"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "server has stopped accepting new clients", "iface", "127.0.0.1:4654") { t.Error("Server did not stop on 127.0.0.1:4654") } - if i := strings.Index(logOutput, "Server [127.0.0.1:2526] has stopped accepting new clients"); i < 0 { + if !MatchLog(logOutput, 1, "msg", "server has stopped accepting new clients", "iface", "127.0.0.1:2526") { t.Error("Server did not stop on 127.0.0.1:2526") } - if i := strings.Index(logOutput, "shutdown completed for [127.0.0.1:4654]"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "shutdown completed", "iface", "127.0.0.1:4654") { t.Error("Server did not complete shutdown on 127.0.0.1:4654") } - if i := strings.Index(logOutput, "shutdown completed for [127.0.0.1:2526]"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "shutdown completed", "iface", "127.0.0.1:2526") { t.Error("Server did not complete shutdown on 127.0.0.1:2526") } - if i := strings.Index(logOutput, "shutting down pool [127.0.0.1:4654]"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "shutting down pool", "iface", "127.0.0.1:4654") { t.Error("Server did not shutdown pool on 127.0.0.1:4654") } - if i := strings.Index(logOutput, "shutting down pool [127.0.0.1:2526]"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "shutting down pool", "iface", "127.0.0.1:2526") { t.Error("Server did not shutdown pool on 127.0.0.1:2526") } - if i := strings.Index(logOutput, "Backend shutdown completed"); i < 0 { + + if !MatchLog(logOutput, 1, "msg", "backend shutdown completed") { t.Error("Backend didn't shut down") } - } } @@ -300,7 +346,7 @@ func TestGreeting(t *testing.T) { app.Shutdown() if read, err := ioutil.ReadFile("./testlog"); err == nil { logOutput := string(read) - if i := strings.Index(logOutput, "Handle client [127.0.0.1"); i < 0 { + if !MatchLog(logOutput, 1, "msg", "handle client", "peer", "127.0.0.1") { t.Error("Server did not handle any clients") } } @@ -358,9 +404,10 @@ func TestShutDown(t *testing.T) { if read, err := ioutil.ReadFile("./testlog"); err == nil { logOutput := string(read) // fmt.Println(logOutput) - if i := strings.Index(logOutput, "Handle client [127.0.0.1"); i < 0 { + if !MatchLog(logOutput, 1, "msg", "handle client", "peer", "127.0.0.1") { t.Error("Server did not handle any clients") } + } } @@ -433,7 +480,7 @@ func TestRFC2832LimitLocalPart(t *testing.T) { t.Error("Hello command failed", err.Error()) } // repeat > 64 characters in local part - response, err := Command(conn, bufin, fmt.Sprintf("RCPT TO:<%s@grr.la>", strings.Repeat("a", rfc5321.LimitLocalPart+1))) + response, err := Command(conn, bufin, fmt.Sprintf("RCPT TO:<%s@grr.la>", strings.Repeat("a", smtp.LimitLocalPart+1))) if err != nil { t.Error("rcpt command failed", err.Error()) } @@ -443,7 +490,7 @@ func TestRFC2832LimitLocalPart(t *testing.T) { } // what about if it's exactly 64? // repeat > 64 characters in local part - response, err = Command(conn, bufin, fmt.Sprintf("RCPT TO:<%s@grr.la>", strings.Repeat("a", rfc5321.LimitLocalPart-1))) + response, err = Command(conn, bufin, fmt.Sprintf("RCPT TO:<%s@grr.la>", strings.Repeat("a", smtp.LimitLocalPart-1))) if err != nil { t.Error("rcpt command failed", err.Error()) } @@ -867,7 +914,7 @@ func TestHeloEhlo(t *testing.T) { } } - expected = fmt.Sprintf("250-%s Hello\r\n250-SIZE 100017\r\n250-PIPELINING\r\n250-STARTTLS\r\n250-ENHANCEDSTATUSCODES\r\n250 HELP\r\n", hostname) + expected = fmt.Sprintf("250-%s Hello\r\n250-SIZE 100017\r\n250-PIPELINING\r\n250-STARTTLS\r\n250-ENHANCEDSTATUSCODES\r\n250-8BITMIME\r\n250 HELP\r\n", hostname) if fullresp != expected { t.Error("Server did not respond with [" + expected + "], it said [" + fullresp + "]") } diff --git a/tests/util.go b/tests/util.go new file mode 100644 index 00000000..dc8b4feb --- /dev/null +++ b/tests/util.go @@ -0,0 +1,52 @@ +package test + +import ( + "fmt" + "regexp" + "strings" +) + +// MatchLog looks for the key/val in the input (a log file) +func MatchLog(input string, startLine int, args ...interface{}) bool { + size := len(args) + if size < 2 || size%2 != 0 { + panic("args must be even") + } + lines := strings.Split(input, "\n") + if len(lines) < startLine { + panic("log too short, lines:" + fmt.Sprintf("%v", len(lines))) + } + re, _ := regexp.Compile(`[[:space:]:\\]`) + var lookFor string + // for each line + found := false + for i := startLine - 1; i < len(lines); i++ { + // for each pair + for j := 0; j < len(args); j++ { + if j%2 != 0 { + continue + } + key := args[j] + val := args[j+1] + lookFor = fmt.Sprintf("%v", val) + if re.MatchString(lookFor) { + // quote it + lookFor = fmt.Sprintf(`%s="%s"`, key, val) + } else { + lookFor = fmt.Sprintf(`%s=%v`, key, val) + } + + if pos := strings.Index(lines[i], lookFor); pos != -1 { + found = true + } else { + found = false + // short circuit + break + } + } + if found { + break + } + } + return found +}