From 906695ab96cc95e8a2a4551b1b06bc778e548712 Mon Sep 17 00:00:00 2001 From: egibs Date: Fri, 29 Dec 2023 10:16:29 -0600 Subject: [PATCH] Add DeepSearch --- README.md | 91 +++++++++++++++++++++++-- VERSION | 2 +- deepsearch.go | 85 ++++++++++++++++++++++++ deepsearch_test.go | 143 ++++++++++++++++++++++++++++++++++++++++ deepwalk.go | 90 ++++--------------------- deepwalk_test.go | 4 +- test_util.go => util.go | 63 ++++++++++++++++++ 7 files changed, 392 insertions(+), 86 deletions(-) create mode 100644 deepsearch.go create mode 100644 deepsearch_test.go rename test_util.go => util.go (73%) diff --git a/README.md b/README.md index 991d688..4843cd2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ This project was mostly done to hack on some Go after spending awhile way from i Usage examples can be found in `deepwalk_test.go`, but because code is only as good as its documentation, examples will be added (and added to) below. +Additionally, a naive search method is also provided via `DeepSearch`. This method traverses a data structure and returns all values for the provided key. This method is useful when the structure of the data is not known beforehand or can change. + ## Installation To install `deepwalk`, run the following: @@ -88,6 +90,53 @@ if err != nil { fmt.Println(value) ``` +### Structs + +Structs can also be traversed with `DeepWalk`: +```go +type TestStruct struct { + Field1 string + Field2 int + NestedStruct struct { + NestedStruct2 struct { + NestedField1 string + NestedField2 int + } + NestedField1 string + NestedField2 int + } +} + +testStruct := TestStruct{ + Field1: "test", + Field2: 123, + NestedStruct: struct { + NestedStruct2 struct { + NestedField1 string + NestedField2 int + } + NestedField1 string + NestedField2 int + }{ + NestedStruct2: struct { + NestedField1 string + NestedField2 int + }{ + NestedField1: "nested", + NestedField2: 456, + }, + NestedField1: "nested", + NestedField2: 456, + }, +} + +values, err := DeepWalk(testStruct, []string{"NestedStruct", "NestedStruct2", "NestedField1"}, "", "all") +if err != nil { + fmt.Println(err) +} +fmt.Println(value) +``` + ## Testing Several categories of tests are included: 1. A manual JSON object @@ -99,7 +148,29 @@ Categories three and four run one-thousand iterations of each variant and each d Run all of the included tests by running `make test`: ```sh +❯ make test go test ./... -v +go: downloading github.com/wk8/go-ordered-map/v2 v2.1.8 +go: downloading gopkg.in/yaml.v3 v3.0.1 +go: downloading github.com/buger/jsonparser v1.1.1 +go: downloading github.com/mailru/easyjson v0.7.7 +go: downloading github.com/bahlo/generic-list-go v0.2.0 +=== RUN TestDeepSearch +=== RUN TestDeepSearch/Test_case_1_-_key_found_in_map +=== RUN TestDeepSearch/Test_case_2_-_key_not_found_in_map +=== RUN TestDeepSearch/Test_case_3_-_key_found_in_nested_map +=== RUN TestDeepSearch/Test_case_4_-_key_not_found_in_nested_map +=== RUN TestDeepSearch/Test_case_7_-_key_found_in_struct +=== RUN TestDeepSearch/Test_case_8_-_key_not_found_in_struct +=== RUN TestDeepSearch/Test_case_9_-_duplicate_key_found_in_map +--- PASS: TestDeepSearch (0.00s) + --- PASS: TestDeepSearch/Test_case_1_-_key_found_in_map (0.00s) + --- PASS: TestDeepSearch/Test_case_2_-_key_not_found_in_map (0.00s) + --- PASS: TestDeepSearch/Test_case_3_-_key_found_in_nested_map (0.00s) + --- PASS: TestDeepSearch/Test_case_4_-_key_not_found_in_nested_map (0.00s) + --- PASS: TestDeepSearch/Test_case_7_-_key_found_in_struct (0.00s) + --- PASS: TestDeepSearch/Test_case_8_-_key_not_found_in_struct (0.00s) + --- PASS: TestDeepSearch/Test_case_9_-_duplicate_key_found_in_map (0.00s) === RUN TestDeepwalkMinimalJSON --- PASS: TestDeepwalkMinimalJSON (0.00s) === RUN TestDeepwalkMinimalMap @@ -117,8 +188,6 @@ go test ./... -v === RUN TestDeepWalk/Test_case_10_-_array_of_maps_with_multiple_matching_keys,_return_last === RUN TestDeepWalk/Test_case_11_-_array_of_maps_with_multiple_matching_keys,_return_first === RUN TestDeepWalk/Test_case_12_-_array_of_maps_with_multiple_matching_keys,_return_default -=== RUN TestDeepWalk/Test_case_13_-_array_of_maps_with_multiple_matching_keys,_return_default -=== RUN TestDeepWalk/Test_case_14_-_array_of_maps_with_multiple_matching_keys,_return_default --- PASS: TestDeepWalk (0.00s) --- PASS: TestDeepWalk/Test_case_1_-_empty_object (0.00s) --- PASS: TestDeepWalk/Test_case_2_-_empty_keys (0.00s) @@ -132,14 +201,24 @@ go test ./... -v --- PASS: TestDeepWalk/Test_case_10_-_array_of_maps_with_multiple_matching_keys,_return_last (0.00s) --- PASS: TestDeepWalk/Test_case_11_-_array_of_maps_with_multiple_matching_keys,_return_first (0.00s) --- PASS: TestDeepWalk/Test_case_12_-_array_of_maps_with_multiple_matching_keys,_return_default (0.00s) - --- PASS: TestDeepWalk/Test_case_13_-_array_of_maps_with_multiple_matching_keys,_return_default (0.00s) - --- PASS: TestDeepWalk/Test_case_14_-_array_of_maps_with_multiple_matching_keys,_return_default (0.00s) === RUN TestDeepwalkRandomSuccess --- PASS: TestDeepwalkRandomSuccess (0.25s) === RUN TestDeepwalkRandomDefault ---- PASS: TestDeepwalkRandomDefault (3.22s) +--- PASS: TestDeepwalkRandomDefault (3.18s) +=== RUN TestDeepWalkWithStruct +=== RUN TestDeepWalkWithStruct/Test_Field1 +=== RUN TestDeepWalkWithStruct/Test_Field2 +=== RUN TestDeepWalkWithStruct/Test_NestedField1 +=== RUN TestDeepWalkWithStruct/Test_NestedField2 +=== RUN TestDeepWalkWithStruct/Test_Second-level_NestedField1 +--- PASS: TestDeepWalkWithStruct (0.00s) + --- PASS: TestDeepWalkWithStruct/Test_Field1 (0.00s) + --- PASS: TestDeepWalkWithStruct/Test_Field2 (0.00s) + --- PASS: TestDeepWalkWithStruct/Test_NestedField1 (0.00s) + --- PASS: TestDeepWalkWithStruct/Test_NestedField2 (0.00s) + --- PASS: TestDeepWalkWithStruct/Test_Second-level_NestedField1 (0.00s) PASS -ok github.com/egibs/deepwalk 3.600s +ok github.com/egibs/deepwalk 3.529s ``` ## Benchmarks diff --git a/VERSION b/VERSION index 3eefcb9..9084fa2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/deepsearch.go b/deepsearch.go new file mode 100644 index 0000000..aea0881 --- /dev/null +++ b/deepsearch.go @@ -0,0 +1,85 @@ +package deepwalk + +import "reflect" + +// DeepSearch traverses a data structure and returns the value of the specified key +// DeepSearch does not need to know the path to the specified key +// Like DeepWalk, DeepSearch will return either the first, last, or all encountered values +// associated with the provided `searchKey` +func DeepSearch( + obj interface{}, + searchKey string, + defaultVal string, + returnVal string, +) (interface{}, error) { + // Return the default value if the object is empty or if the keys or return value are invalid + if IsEmpty(obj) || !ValidKeys([]string{searchKey}) || !ValidReturnVal(returnVal) { + return defaultVal, nil + } + + // Return the object if there are no keys to traverse + if len(searchKey) == 0 { + return obj, nil + } + + var foundList []interface{} + + search(obj, searchKey, &foundList) + + return HandleReturnVal(&foundList, defaultVal, returnVal) +} + +// search traverses a data structure and returns the value of the specified key +func search(obj interface{}, searchKey string, foundList *[]interface{}) { + switch object := obj.(type) { + case map[string]interface{}: + deepSearchMap(object, searchKey, foundList) + case []interface{}: + deepSearchSlice(object, searchKey, foundList) + default: + deepSearchStruct(object, searchKey, foundList) + } +} + +// deepSearchMap handles the case where the object is a map +func deepSearchMap( + obj map[string]interface{}, + searchKey string, + foundList *[]interface{}, +) { + for key, value := range obj { + if key == searchKey { + *foundList = append(*foundList, value) + } + search(value, searchKey, foundList) + } +} + +// deepSearchSlice handles the case where the object is a slice +func deepSearchSlice( + obj []interface{}, + searchKey string, + foundList *[]interface{}, +) { + for _, item := range obj { + search(item, searchKey, foundList) + } +} + +// deepSearchStruct handles the case where the object is a struct +func deepSearchStruct( + obj interface{}, + searchKey string, + foundList *[]interface{}, +) { + if r := reflect.ValueOf(obj); r.Kind() == reflect.Struct { + for i := 0; i < r.NumField(); i++ { + f := r.Field(i) + if r.Type().Field(i).Name == searchKey { + *foundList = append(*foundList, f.Interface()) + } else { + search(f.Interface(), searchKey, foundList) + } + } + } +} diff --git a/deepsearch_test.go b/deepsearch_test.go new file mode 100644 index 0000000..41459ca --- /dev/null +++ b/deepsearch_test.go @@ -0,0 +1,143 @@ +package deepwalk + +import ( + "reflect" + "testing" +) + +func TestDeepSearch(t *testing.T) { + type args struct { + obj interface{} + searchKey string + defaultVal string + returnVal string + } + tests := []struct { + name string + args args + want interface{} + wantErr bool + }{ + { + name: "Test case 1 - key found in map", + args: args{ + obj: map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }, + searchKey: "key1", + defaultVal: "default", + returnVal: "first", + }, + want: "value1", + wantErr: false, + }, + { + name: "Test case 2 - key not found in map", + args: args{ + obj: map[string]interface{}{ + "key1": "value1", + "key2": "value2", + }, + searchKey: "key3", + defaultVal: "default", + returnVal: "first", + }, + want: "default", + wantErr: false, + }, + { + name: "Test case 3 - key found in nested map", + args: args{ + obj: map[string]interface{}{ + "key1": "value1", + "key2": map[string]interface{}{ + "key3": "value3", + }, + }, + searchKey: "key3", + defaultVal: "default", + returnVal: "first", + }, + want: "value3", + wantErr: false, + }, + { + name: "Test case 4 - key not found in nested map", + args: args{ + obj: map[string]interface{}{ + "key1": "value1", + "key2": map[string]interface{}{ + "key3": "value3", + }, + }, + searchKey: "key4", + defaultVal: "default", + returnVal: "first", + }, + want: "default", + wantErr: false, + }, + { + name: "Test case 7 - key found in struct", + args: args{ + obj: struct { + Key1 string + Key2 string + }{ + Key1: "value1", + Key2: "value2", + }, + searchKey: "Key1", + defaultVal: "default", + returnVal: "first", + }, + want: "value1", + wantErr: false, + }, + { + name: "Test case 8 - key not found in struct", + args: args{ + obj: struct { + Key1 string + Key2 string + }{ + Key1: "value1", + Key2: "value2", + }, + searchKey: "Key3", + defaultVal: "default", + returnVal: "first", + }, + want: "default", + wantErr: false, + }, + { + name: "Test case 9 - duplicate key found in map", + args: args{ + obj: map[string]interface{}{ + "key1": "value1", + "key2": map[string]interface{}{ + "key1": "value2", + }, + }, + searchKey: "key1", + defaultVal: "default", + returnVal: "all", + }, + want: []interface{}{"value1", "value2"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeepSearch(tt.args.obj, tt.args.searchKey, tt.args.defaultVal, tt.args.returnVal) + if (err != nil) != tt.wantErr { + t.Errorf("DeepSearch() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("DeepSearch() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deepwalk.go b/deepwalk.go index fb96e2c..76310d3 100644 --- a/deepwalk.go +++ b/deepwalk.go @@ -2,7 +2,6 @@ package deepwalk import ( "reflect" - "strings" orderedmap "github.com/wk8/go-ordered-map/v2" ) @@ -20,7 +19,7 @@ func DeepWalk( returnVal string, ) (interface{}, error) { // Return the default value if the object is empty or if the keys or return value are invalid - if isEmpty(obj) || !validKeys(keys) || !validReturnVal(returnVal) { + if IsEmpty(obj) || !ValidKeys(keys) || !ValidReturnVal(returnVal) { return defaultVal, nil } @@ -31,66 +30,29 @@ func DeepWalk( currentKey := keys[0] found := orderedmap.New[string, struct{}]() - var foundList []string + var foundList []interface{} r := reflect.ValueOf(obj) if r.Kind() == reflect.Struct { - return handleStruct(r, currentKey, keys, defaultVal, returnVal) + return deepWalkStruct(r, currentKey, keys, defaultVal, returnVal) } switch object := obj.(type) { case map[string]interface{}: - return handleMap(object, currentKey, keys, defaultVal, returnVal) + return deepWalkMap(object, currentKey, keys, defaultVal, returnVal) case []interface{}: - return handleSlice(object, keys, defaultVal, returnVal, found, foundList) + return deepWalkSlice(object, keys, defaultVal, returnVal, found, &foundList) } for kv := found.Oldest(); kv != nil; kv = kv.Next() { - foundList = append(foundList, kv.Key) + foundList = append(foundList, []interface{}{kv.Key}) } - return handleReturnVal(found, foundList, defaultVal, returnVal) -} - -// isEmpty checks if the specified object is empty -func isEmpty(subObj interface{}) bool { - switch subObject := subObj.(type) { - case string: - return false - case []interface{}: - for _, nextObj := range subObject { - if !isEmpty(nextObj) { - return false - } - } - return true - default: - return false - } -} - -// validKeys checks if the specified keys are valid -func validKeys(keys []string) bool { - for _, key := range keys { - if strings.TrimSpace(key) == "" { - return false - } - } - return true -} - -// validReturnVal checks if the specified return value is valid -func validReturnVal(returnVal string) bool { - switch returnVal { - case "first", "last", "all": - return true - default: - return false - } + return HandleReturnVal(&foundList, defaultVal, returnVal) } // handleStruct handles the case where the object is a struct -func handleStruct( +func deepWalkStruct( r reflect.Value, currentKey string, keys []string, @@ -105,7 +67,7 @@ func handleStruct( } // handleMap handles the case where the object is a map -func handleMap( +func deepWalkMap( object map[string]interface{}, currentKey string, keys []string, @@ -120,13 +82,13 @@ func handleMap( } // handleSlice handles the case where the object is a slice -func handleSlice( +func deepWalkSlice( object []interface{}, keys []string, defaultVal string, returnVal string, found *orderedmap.OrderedMap[string, struct{}], - foundList []string, + foundList *[]interface{}, ) (interface{}, error) { for _, item := range object { val, err := DeepWalk(item, keys, defaultVal, returnVal) @@ -135,35 +97,9 @@ func handleSlice( } if val != defaultVal { found.Set(val.(string), struct{}{}) - foundList = append(foundList, val.(string)) + *foundList = append(*foundList, val.(string)) } } - return handleReturnVal(found, foundList, defaultVal, returnVal) -} - -// handleReturnVal handles the the appropriate value to return based on the returnVal argument -func handleReturnVal( - found *orderedmap.OrderedMap[string, struct{}], - foundList []string, - defaultVal string, - returnVal string, -) (interface{}, error) { - if len(foundList) == 0 { - return defaultVal, nil - } - - switch returnVal { - case "first": - return foundList[0], nil - case "last": - return foundList[len(foundList)-1], nil - case "all": - if len(foundList) == 1 { - return foundList[0], nil - } - return foundList, nil - default: - return defaultVal, nil - } + return HandleReturnVal(foundList, defaultVal, returnVal) } diff --git a/deepwalk_test.go b/deepwalk_test.go index e915235..22a1170 100644 --- a/deepwalk_test.go +++ b/deepwalk_test.go @@ -89,7 +89,7 @@ func BenchmarkDeepwalkMinimalJSON(b *testing.B) { } want := map[int]interface{}{ 1: "foo", - 2: []string{"foo", "bar", "baz"}, + 2: []interface{}{"foo", "bar", "baz"}, 3: "", } for i := 0; i < b.N; i++ { @@ -232,7 +232,7 @@ func TestDeepWalk(t *testing.T) { defaultVal: "default", returnVal: "all", }, - want: []string{"value1", "value3"}, + want: []interface{}{"value1", "value3"}, wantErr: false, }, { diff --git a/test_util.go b/util.go similarity index 73% rename from test_util.go rename to util.go index bfe52a7..3155479 100644 --- a/test_util.go +++ b/util.go @@ -8,6 +8,69 @@ import ( var maxDepth = 10 +// IsEmpty checks if the specified object is empty +func IsEmpty(subObj interface{}) bool { + switch subObject := subObj.(type) { + case string: + return false + case []interface{}: + for _, nextObj := range subObject { + if !IsEmpty(nextObj) { + return false + } + } + return true + default: + return false + } +} + +// ValidKeys checks if the specified keys are valid +func ValidKeys(keys []string) bool { + for _, key := range keys { + if strings.TrimSpace(key) == "" { + return false + } + } + return true +} + +// ValidReturnVal checks if the specified return value is valid +func ValidReturnVal(returnVal string) bool { + switch returnVal { + case "first", "last", "all": + return true + default: + return false + } +} + +// HandleReturnVal handles the the appropriate value to return based on the returnVal argument +func HandleReturnVal( + foundList *[]interface{}, + defaultVal string, + returnVal string, +) (interface{}, error) { + switch { + case foundList == nil || (foundList != nil && len(*foundList) == 0): + return defaultVal, nil + } + + switch returnVal { + case "first": + return (*foundList)[0], nil + case "last": + return (*foundList)[len(*foundList)-1], nil + case "all": + if len(*foundList) == 1 { + return (*foundList)[0], nil + } + return *foundList, nil + default: + return defaultVal, nil + } +} + // RandomInt generate a cryptographically secure random integer func RandomInt(min, max int64) (int64, error) { if max-min <= 0 {