From 1ea13705b253162f4a0ebeb4b93857f7f0a975fc Mon Sep 17 00:00:00 2001 From: SA6MWA Date: Wed, 15 Mar 2023 23:22:08 +0100 Subject: [PATCH] Add io.Reader and io.WriteCloser support to Stash and Unstash --- README.md | 57 +++++++++++++++- anystore.go | 13 +++- anystore_test.go | 19 ++++++ stash.go | 158 ++++++++++++++++++++++++++++++++++++++----- stash_test.go | 170 ++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 387 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9bfa217..a934708 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,60 @@ func main() { } ``` -You can generate a random AES-256 base64-encoded encryption key using... +The `Stash` and `Unstash` functions also support `io.Reader` and `io.Writer` +(`io.WriteCloser`). `Stash` will write to both a file and `io.Writer` if +configured with both via the `anystore.StashConfig` struct. `Unstash` will +prefer `io.Reader` over file if both are provided. `Unstash` will successfully +unstash from the `os.File` `io.Reader` of +`os.Open(previously_stashed_file_by_Stash)`. + +Example with only reader and writer... + +```go +greeting := "Hello world" +var receivedGreeting string + +reader, writer := io.Pipe() +defer reader.Close() // Stash closes the writer, it's an io.ReadCloser + +errch := make(chan error) + +go func() { + defer close(errch) + if err := anystore.Unstash(&anystore.StashConfig{ + Reader: reader, + Key: "secret", + Thing: &receivedGreeting, + }, nil); err != nil { + errch <- err + } + errch <- nil +}() + +if err := anystore.Stash(&anystore.StashConfig{ + Writer: writer, + Key: "secret", + Thing: &greeting, +}); err != nil { + log.Fatal(err) +} + +err := <-errch +if err != nil { + log.Fatal(err) +} + +fmt.Println(receivedGreeting) +``` + + + +## Encrypted by default + +There is a default encryption key constant (`anystore.DefaultEncryptionKey`) +that will be used if no user-defined key is provided. It is obviously not secure +to use the default asymmetric key as it is publicly known. You can generate your +own random AES-256 base64-encoded encryption key using `./cmd/newkey`... ```sh go run github.com/sa6mwa/anystore/cmd/newkey @@ -88,7 +141,7 @@ go run github.com/sa6mwa/anystore/cmd/newkey ## Persistence, not performance The persistence-feature is not designed for performance, but for simplicity, -durability and concurrent access by multiple processes/instances. The entire +durability, and concurrent access by multiple processes/instances. The entire key/value store (`map[any]any`) is loaded and persisted on retrieving or storing every key/value pair making it slow with many keys (can be sharded manually by managing several AnyStores). diff --git a/anystore.go b/anystore.go index 67c1865..9415e5a 100644 --- a/anystore.go +++ b/anystore.go @@ -172,6 +172,9 @@ type AnyStore interface { SetEncryptionKey(key string) (AnyStore, error) + // GetEncryptionKeyBytes returns a byte slice with the AES encryption key. + GetEncryptionKeyBytes() []byte + // HasKey tests if key exists in the store, returns true if it does, false if // not. Retrieval is atomic. HasKey(key any) bool @@ -214,7 +217,7 @@ type AnyStore interface { } type Options struct { - // Store and load the AnyStore from file? Set to true + // Store and load the AnyStore from file or io.ReadWriter? Set to true EnablePersistence bool // Can start with tilde for HOME resolution, will do os.MkdirAll on directory // path. Omit to use DefaultPersistenceFile @@ -336,6 +339,10 @@ func (a *anyStore) SetEncryptionKey(key string) (AnyStore, error) { return a, nil } +func (a *anyStore) GetEncryptionKeyBytes() []byte { + return a.key.Load().([]byte) +} + func (a *anyStore) HasKey(key any) bool { if a.persist.Load() { a.mutex.Lock() @@ -621,6 +628,10 @@ func (u *unsafeAnyStore) SetEncryptionKey(key string) (AnyStore, error) { return u, nil } +func (u *unsafeAnyStore) GetEncryptionKeyBytes() []byte { + return u.key.Load().([]byte) +} + func (u *unsafeAnyStore) HasKey(key any) bool { if u.persist.Load() { u.load() diff --git a/anystore_test.go b/anystore_test.go index 16efacd..8e98604 100644 --- a/anystore_test.go +++ b/anystore_test.go @@ -1,6 +1,8 @@ package anystore_test import ( + "bytes" + "encoding/base64" "errors" "fmt" "os" @@ -166,6 +168,23 @@ func TestAnyStore_Run(t *testing.T) { } } +func TestAnyStore_GetEncryptionKeyBytes(t *testing.T) { + expected, err := base64.RawStdEncoding.DecodeString(anystore.DefaultEncryptionKey) + if err != nil { + t.Fatal(err) + } + a, err := anystore.NewAnyStore(&anystore.Options{ + EnablePersistence: false, + }) + if err != nil { + t.Fatal(err) + } + obtained := a.GetEncryptionKeyBytes() + if !bytes.Equal(expected, obtained) { + t.Error("obtained bytes from AnyStore.GetEncryptionKeyBytes() and expected bytes do not match") + } +} + func ExampleAnyStore_Store_encrypt() { f, err := os.CreateTemp("", "anystore-example-*") if err != nil { diff --git a/stash.go b/stash.go index d91a141..1b17ba4 100644 --- a/stash.go +++ b/stash.go @@ -5,6 +5,7 @@ import ( "encoding/gob" "errors" "fmt" + "io" "reflect" ) @@ -14,16 +15,47 @@ var ( ErrNotAPointer error = errors.New("stash configuration does not point to thing, need a pointer") ErrThingNotFound error = errors.New("thing not found in stash") ErrTypeMismatch error = errors.New("type-mismatch between thing and default thing") + ErrMissingReader error = errors.New("missing filename (or io.Reader) to load persisted data from") + ErrMissingWriter error = errors.New("missing filename (or io.Writer) to persist data to") ) +// StashConfig instructs how functions anystore.Stash and anystore.Unstash +// should save/load a "stash". If Reader is not nil and File is not an empty +// string, Reader will be preferred over File when executing Unstash. If Writer +// is not nil and File is not an empty string when executing Stash, the file +// will be written first, then written to through the io.Writer (both will be +// written to). Writer.Close() is deferred early, Stash always closes the writer +// on success and failure. If File is an empty string (== "") and Writer is not +// nil, Stash will only write to the io.Writer. type StashConfig struct { - File string // AnyStore DB file - EncryptionKey string // 16, 24 or 32 byte long base64-encoded string - Key string // Key name where to store Thing - Thing any // Usually a struct with data, properties, configuration, etc - Editor string // Editor to use to edit Thing as JSON + File string // AnyStore DB file, if empty, use Reader/Writer + Reader io.Reader // If nil, use File for Unstash, if not, prefer Reader over File + Writer io.WriteCloser // If nil, use File for Stash, if not, write to both Writer and File (if File is not an empty string) + EncryptionKey string // 16, 24 or 32 byte long base64-encoded string + Key string // Key name where to store Thing + Thing any // Usually a struct with data, properties, configuration, etc + Editor string // Editor to use to edit Thing as JSON } +// "stash, verb. to put (something of future use or value) in a safe or secret +// place" +// +// Unstash loads a "Thing" from a place specified in a StashConfig, usually an +// AnyStore DB file, but the Stash and Unstash functions also support io.Reader +// and io.Writer (io.WriteCloser). Reader/writer is essentially an in-memory +// version of the physical DB file, Unstash does io.ReadAll into memory in order +// to decrypt and de-GOB the data. A previous file-Stash command can be +// Unstashed via the io.Reader. Unstash prefers io.Reader when both +// StashConfig.File and StashConfig.Reader are defined. +// +// StashConfig instructs how functions anystore.Stash and anystore.Unstash +// should save/load a "stash". If Reader is not nil and File is not an empty +// string, Reader will be preferred over File when executing Unstash. If Writer +// is not nil and File is not an empty string when executing Stash, the file +// will be written first, then written to through the io.Writer (both will be +// written to). Writer.Close() is deferred early, Stash always closes the writer +// on success and failure. If File is an empty string (== "") and Writer is not +// nil, Stash will only write to the io.Writer. func Unstash(conf *StashConfig, defaultThing any) error { if conf.Thing == nil { return ErrNilThing @@ -31,19 +63,52 @@ func Unstash(conf *StashConfig, defaultThing any) error { if conf.Key == "" { return ErrEmptyKey } - - a, err := NewAnyStore(&Options{ + if conf.File == "" && conf.Reader == nil { + return ErrMissingReader + } + options := Options{ EnablePersistence: true, PersistenceFile: conf.File, EncryptionKey: conf.EncryptionKey, - }) - if err != nil { - return err } - gobbedThing, err := a.Load(conf.Key) + // If we have an io.Reader, prefer it above File. + if conf.Reader != nil { + options.EnablePersistence = false + } + a, err := NewAnyStore(&options) if err != nil { return err } + var gobbedThing any + if conf.Reader != nil { + // Read encrypted anyMap + kv := make(anyMap) + data, err := io.ReadAll(conf.Reader) + if err != nil { + return err + } + decrypted, err := Decrypt(a.GetEncryptionKeyBytes(), data) + if err != nil { + return err + } + in := gob.NewDecoder(bytes.NewReader(decrypted)) + if err := in.Decode(&kv); err != nil { + return err + } + var ok bool + gobbedThing, ok = kv[conf.Key] + if !ok { + return ErrThingNotFound + } + } else { + // Load key from PersistenceFile instead. + var err error + gobbedThing, err = a.Load(conf.Key) + if err != nil { + return err + } + } + // GOB encoded thing came from either file or io.Reader. thing, ok := gobbedThing.([]byte) if !ok { if defaultThing != nil { @@ -68,7 +133,26 @@ func Unstash(conf *StashConfig, defaultThing any) error { // "stash, verb. to put (something of future use or value) in a safe or secret // place" +// +// Stash stores a "Thing" according to a StashConfig, usually an AnyStore DB +// file, but Stash and Unstash can also be used with an io.Writer +// (io.WriteCloser) and an io.Reader for arbitrary stashing/unstashing. Stash +// always closes the writer on exit (why it's an io.WriteCloser). The +// reader/writers are essentially in-memory versions of the physical DB file, +// Unstash does io.ReadAll into memory in order to decrypt and de-GOB it. +// +// StashConfig instructs how functions anystore.Stash and anystore.Unstash +// should save/load a "stash". If Reader is not nil and File is not an empty +// string, Reader will be preferred over File when executing Unstash. If Writer +// is not nil and File is not an empty string when executing Stash, the file +// will be written first, then written to through the io.Writer (both will be +// written to). Writer.Close() is deferred early, Stash always closes the writer +// on success and failure. If File is an empty string (== "") and Writer is not +// nil, Stash will only write to the io.Writer. func Stash(conf *StashConfig) error { + if conf.Writer != nil { + defer conf.Writer.Close() + } value := reflect.ValueOf(conf.Thing) if value.Type().Kind() != reflect.Pointer { return ErrNotAPointer @@ -76,14 +160,28 @@ func Stash(conf *StashConfig) error { if value.IsNil() { return ErrNilThing } - a, err := NewAnyStore(&Options{ - EnablePersistence: true, - PersistenceFile: conf.File, - EncryptionKey: conf.EncryptionKey, - }) + if conf.Key == "" { + return ErrEmptyKey + } + if conf.File == "" && conf.Writer == nil { + return ErrMissingWriter + } + + options := Options{ + PersistenceFile: conf.File, + EncryptionKey: conf.EncryptionKey, + } + if conf.File == "" { + options.EnablePersistence = false + } else { + options.EnablePersistence = true + } + + a, err := NewAnyStore(&options) if err != nil { return err } + // Use gob to store the struct (or other value) instead of re-inventing // dereference of all pointers. It is also unlikely that the interface stored // is registered with gob in the downstream anystore package. @@ -92,5 +190,31 @@ func Stash(conf *StashConfig) error { if err := g.Encode(conf.Thing); err != nil { return fmt.Errorf("gob.Encode of StashConfig.Thing: %w", err) } - return a.Store(conf.Key, thing.Bytes()) + // Persist to file if filename was not an empty string. + if conf.File != "" { + if err := a.Store(conf.Key, thing.Bytes()); err != nil { + return err + } + } + // If conf.Writer was given, also write to the io.Writer, but this has to be + // emulated (AnyStore does not implement io.Writer or io.Reader). + if conf.Writer != nil { + kv := make(anyMap) + kv[conf.Key] = thing.Bytes() + var gobOutput bytes.Buffer + out := gob.NewEncoder(&gobOutput) + if err := out.Encode(kv); err != nil { + return err + } + encrypted, err := Encrypt(a.GetEncryptionKeyBytes(), gobOutput.Bytes()) + if err != nil { + return err + } + if n, err := conf.Writer.Write(encrypted); err != nil { + return err + } else if n != len(encrypted) { + return ErrWroteTooLittle + } + } + return nil } diff --git a/stash_test.go b/stash_test.go index 1cf9b84..284a95b 100644 --- a/stash_test.go +++ b/stash_test.go @@ -2,6 +2,9 @@ package anystore_test import ( "errors" + "fmt" + "io" + "log" "os" "reflect" "testing" @@ -25,7 +28,7 @@ func strptr(s string) *string { return &s } -func doStash(file string, encryptionKey string) error { +func doStash(file string, writer io.WriteCloser, encryptionKey string) error { expectedThing := &Thing{ Name: strptr("Hello World"), Number: 32, @@ -36,31 +39,37 @@ func doStash(file string, encryptionKey string) error { {ID: 3, Name: "Component three"}, }, } - if err := anystore.Stash(&anystore.StashConfig{ + + stashconf := anystore.StashConfig{ File: file, EncryptionKey: encryptionKey, Key: "configuration", Thing: expectedThing, - }); err != nil { + Writer: writer, + } + + if err := anystore.Stash(&stashconf); err != nil { return err } return nil } -func doUnstash(file string, encryptionKey string) (Thing, error) { +func doUnstash(file string, reader io.Reader, encryptionKey string) (Thing, error) { var gotThing Thing + if err := anystore.Unstash(&anystore.StashConfig{ File: file, EncryptionKey: encryptionKey, Key: "configuration", Thing: &gotThing, + Reader: reader, }, nil); err != nil { return Thing{}, err } return gotThing, nil } -func doUnstashDefault(file string, encryptionKey string) (Thing, error) { +func doUnstashDefault(file string, reader io.Reader, encryptionKey string) (Thing, error) { defaultThing := &Thing{ Name: strptr("Hello World"), Number: 32, @@ -77,6 +86,7 @@ func doUnstashDefault(file string, encryptionKey string) (Thing, error) { EncryptionKey: encryptionKey, Key: "key_not_in_stash", Thing: &gotThing, + Reader: reader, }, defaultThing); err != nil { return Thing{}, err } @@ -176,10 +186,67 @@ func TestStash(t *testing.T) { {ID: 3, Name: "Component three"}, }, } - if err := doStash(tempfile, secret); err != nil { + if err := doStash(tempfile, nil, secret); err != nil { + t.Fatal(err) + } + gotThing, err := doUnstash(tempfile, nil, secret) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&gotThing, expectedThing) { + t.Errorf("got %s and expected %s does not match", reflect.TypeOf(gotThing), reflect.TypeOf(expectedThing)) + } +} + +func TestStash_doublestash(t *testing.T) { + secret := anystore.NewKey() + fl, err := os.CreateTemp("", "anystore-stash-double-test-*") + if err != nil { + t.Fatal(err) + } + tempfile := fl.Name() + fl.Close() + defer func() { + os.Remove(tempfile) + os.Remove(tempfile + ".lock") + }() + + expectedThing := &Thing{ + Name: strptr("Hello World"), + Number: 32, + Description: "There is not much to a Hello World thing.", + Components: []*Component{ + {ID: 1, Name: "Component one"}, + {ID: 2, Name: "Component two"}, + {ID: 3, Name: "Component three"}, + }, + } + + reader, writer := io.Pipe() + defer reader.Close() // writer will be closed by Stash + errch := make(chan error) + go func() { + defer close(errch) + gt, err := doUnstash("", reader, secret) + if err != nil { + errch <- err + return + } + if !reflect.DeepEqual(>, expectedThing) { + errch <- fmt.Errorf("got %s and expected %s does not match", reflect.TypeOf(gt), reflect.TypeOf(expectedThing)) + return + } + errch <- nil + }() + if err := doStash(tempfile, writer, secret); err != nil { + t.Fatal(err) + } + err = <-errch + if err != nil { t.Fatal(err) } - gotThing, err := doUnstash(tempfile, secret) + + gotThing, err := doUnstash(tempfile, nil, secret) if err != nil { t.Fatal(err) } @@ -210,17 +277,17 @@ func TestUnstash(t *testing.T) { {ID: 3, Name: "Component three"}, }, } - if err := doStash(tempfile, secret); err != nil { + if err := doStash(tempfile, nil, secret); err != nil { t.Fatal(err) } - gotThing, err := doUnstash(tempfile, secret) + gotThing, err := doUnstash(tempfile, nil, secret) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(&gotThing, expectedThing) { t.Errorf("got %s and expected %s does not match", reflect.TypeOf(gotThing), reflect.TypeOf(expectedThing)) } - gotThing, err = doUnstashDefault(tempfile, secret) + gotThing, err = doUnstashDefault(tempfile, nil, secret) if err != nil { t.Fatal(err) } @@ -228,6 +295,49 @@ func TestUnstash(t *testing.T) { t.Errorf("got %s and expected %s does not match", reflect.TypeOf(gotThing), reflect.TypeOf(expectedThing)) } + // Test io.Reader, should be the same as the expectedThing + if err := doStash(tempfile, nil, secret); err != nil { + t.Fatal(err) + } + tf, err := os.Open(tempfile) + if err != nil { + t.Fatal(err) + } + gotThing, err = doUnstash("", tf, secret) + tf.Close() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&gotThing, expectedThing) { + t.Errorf("got %s and expected %s does not match", reflect.TypeOf(gotThing), reflect.TypeOf(expectedThing)) + } + + // Test stashing to an io.Writer and unstash from an io.Reader using io.Pipe. + reader, writer := io.Pipe() + defer reader.Close() // writer will be closed by Stash + errch := make(chan error) + go func() { + defer close(errch) + gt, err := doUnstash("", reader, secret) + if err != nil { + errch <- err + return + } + if !reflect.DeepEqual(>, expectedThing) { + errch <- fmt.Errorf("got %s and expected %s does not match", reflect.TypeOf(gt), reflect.TypeOf(expectedThing)) + return + } + errch <- nil + }() + if err := doStash("", writer, secret); err != nil { + t.Fatal(err) + } + err = <-errch + if err != nil { + t.Fatal(err) + } + + // Test negative case (key not found) var gotThing2 Thing if err := anystore.Unstash(&anystore.StashConfig{ File: tempfile, @@ -242,3 +352,43 @@ func TestUnstash(t *testing.T) { t.Error("expected anystore.ErrThingNotFound") } } + +func ExampleStash_reader_writer() { + greeting := "Hello world" + var receivedGreeting string + + reader, writer := io.Pipe() + defer reader.Close() // Stash closes the writer, it's an io.ReadCloser + + errch := make(chan error) + + go func() { + defer close(errch) + if err := anystore.Unstash(&anystore.StashConfig{ + Reader: reader, + Key: "secret", + Thing: &receivedGreeting, + }, nil); err != nil { + errch <- err + } + errch <- nil + }() + + if err := anystore.Stash(&anystore.StashConfig{ + Writer: writer, + Key: "secret", + Thing: &greeting, + }); err != nil { + log.Fatal(err) + } + + err := <-errch + if err != nil { + log.Fatal(err) + } + + fmt.Println(receivedGreeting) + + // Output: + // Hello world +}