Feature/search-by (#331)

* Feature/search-by

* Update Searching

* FTS Fixes

* Update READMEs

* Go v1.19 changes

* Added Examples to README

* Spelling fix

Co-authored-by: Caleb Horst <chorst@infoblox.com>

Co-authored-by: Caleb Horst <chorst@infoblox.com>

gateway/field_presence.go

@@ -152,7 +152,7 @@ type presenceInterceptorOptionsDecorator struct {
 
 type presenceInterceptorOption func(*presenceInterceptorOptionsDecorator)
 
-//WithOverrideFieldMask represent an option to override field mask generated by grpc-gateway
+// WithOverrideFieldMask represent an option to override field mask generated by grpc-gateway
 func WithOverrideFieldMask(d *presenceInterceptorOptionsDecorator) {
 	d.overrideFieldMask = true
 }

gateway/fields.go

@@ -7,10 +7,10 @@ import (
 	"github.com/infobloxopen/atlas-app-toolkit/query"
 )
 
-//retainFields function extracts the configuration for fields that
-//need to be ratained either from gRPC response or from original testRequest
-//(in case when gRPC side didn't set any preferences) and retains only
-//this fields on outgoing response (dynmap).
+// retainFields function extracts the configuration for fields that
+// need to be ratained either from gRPC response or from original testRequest
+// (in case when gRPC side didn't set any preferences) and retains only
+// this fields on outgoing response (dynmap).
 func retainFields(ctx context.Context, req *http.Request, dynmap map[string]interface{}) {
 	fieldsStr := ""
 	if req != nil {

gateway/gateway.go

@@ -74,6 +74,15 @@ func ClientUnaryInterceptor(parentCtx context.Context, method string, req, reply
 			}
 		}
 
+		// extracts "_fts" parameters from request
+		if v := vals.Get(searchQueryKey); v != "" {
+			s := query.ParseSearching(v)
+			err = SetCollectionOps(req, s)
+			if err != nil {
+				return err
+			}
+		}
+
 		// extracts "_limit", "_offset",  "_page_token" parameters from request
 		var p *query.Pagination
 		l := vals.Get(limitQueryKey)

gateway/middleware.go

@@ -171,3 +171,12 @@ func GetFieldSelection(req proto.Message) (fieldName string, fs *query.FieldSele
 	}
 	return
 }
+
+func GetSearching(req proto.Message) (fieldName string, s *query.Searching, err error) {
+	s = new(query.Searching)
+	fieldName, err = getAndUnsetOp(req, s, false)
+	if fieldName == "" {
+		s = nil
+	}
+	return
+}

gateway/operator.go

@@ -17,6 +17,7 @@ const (
 	limitQueryKey            = "_limit"
 	offsetQueryKey           = "_offset"
 	pageTokenQueryKey        = "_page_token"
+	searchQueryKey           = "_fts"
 	pageInfoSizeMetaKey      = "status-page-info-size"
 	pageInfoOffsetMetaKey    = "status-page-info-offset"
 	pageInfoPageTokenMetaKey = "status-page-info-page_token"

gorm/README.md

@@ -65,6 +65,22 @@ var people []Person
 db.Find(&people)
 ...
 ```
+### Applying query.Searching
+
+```golang
+...
+db, assoc, err = gorm.ApplySearchingEx(ctx, db, searching, &PersonORM{}, fieldsForFTS, &Person{})
+if err != nil {
+    ...
+}
+db, err = gorm.JoinAssociations(ctx, db, assoc, &PersonORM{})
+if err != nil {
+    ...
+}
+var people []Person
+db.Find(&people)
+...
+```
 
 ### Applying everything
 
@@ -78,6 +94,18 @@ var people []Person
 db.Find(&people)
 ...
 ```
+### Applying everything with Searching
+
+```golang
+...
+db, err = gorm.ApplyCollectionOperatorsWithSearchingEx(ctx, db, &PersonORM{}, &Person{}, filtering, sorting, pagination, fields, searching, fieldsForFTS)
+if err != nil {
+    ...
+}
+var people []Person
+db.Find(&people)
+...
+```
 
 
 ## Transaction Management

gorm/collection_operators.go

@@ -23,11 +23,16 @@ type PaginationConverter interface {
 	PaginationToGorm(ctx context.Context, p *query.Pagination) (offset, limit int32)
 }
 
+type SearchingConverter interface {
+	SearchingToGorm(ctx context.Context, s *query.Searching, fieldsForFTS []string, obj interface{}) (string, error)
+}
+
 type CollectionOperatorsConverter interface {
 	FilteringConditionConverter
 	SortingCriteriaConverter
 	FieldSelectionConverter
 	PaginationConverter
+	SearchingConverter
 }
 
 func ApplyCollectionOperatorsEx(ctx context.Context, db *gorm.DB, obj interface{}, c CollectionOperatorsConverter, f *query.Filtering, s *query.Sorting, p *query.Pagination, fs *query.FieldSelection) (*gorm.DB, error) {
@@ -62,6 +67,46 @@ func ApplyCollectionOperatorsEx(ctx context.Context, db *gorm.DB, obj interface{
 	return db, nil
 }
 
+func ApplyCollectionOperatorsWithSearchingEx(ctx context.Context, db *gorm.DB, obj interface{}, c CollectionOperatorsConverter, f *query.Filtering, s *query.Sorting, p *query.Pagination, fs *query.FieldSelection, sc *query.Searching, fieldsForFTS []string) (*gorm.DB, error) {
+	db, err := ApplyCollectionOperatorsEx(ctx, db, obj, c, f, s, p, fs)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err = ApplySearchingEx(ctx, db, sc, obj, fieldsForFTS, c)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+// ApplySearchingEx applies searching operator s to gorm instance db.
+func ApplySearchingEx(ctx context.Context, db *gorm.DB, s *query.Searching, obj interface{}, fieldsForFTS []string, c SearchingConverter) (*gorm.DB, error) {
+	str, err := c.SearchingToGorm(ctx, s, fieldsForFTS, obj)
+	if err != nil {
+		return nil, err
+	}
+	if s != nil && s.Query != "" {
+		s.Query = strings.TrimSpace(s.Query)
+		s.Query = strings.ReplaceAll(s.Query, ":", " ")
+		splChar := []string{"(", ")", "|", "+", "<", "'", "&", "!", "%", ";"}
+		for _, spl := range splChar {
+			if strings.Contains(s.Query, spl) {
+				s.Query = ""
+				return db.Where(str, s.Query), nil
+			}
+		}
+		s.Query = strings.Join(strings.Fields(s.Query), " ")
+		if s.Query != "" {
+			s.Query = strings.ReplaceAll(s.Query, " ", " & ")
+			s.Query = s.Query + ":*"
+		}
+		return db.Where(str, s.Query), nil
+	}
+	return db, nil
+}
+
 // ApplyFiltering applies filtering operator f to gorm instance db.
 func ApplyFilteringEx(ctx context.Context, db *gorm.DB, f *query.Filtering, obj interface{}, c FilteringConditionConverter) (*gorm.DB, map[string]struct{}, error) {
 	str, args, assocToJoin, err := FilteringToGormEx(ctx, f, obj, c)

gorm/converter.go

@@ -30,12 +30,16 @@ type DefaultSortingCriteriaConverter struct{}
 // DefaultPaginationConverter performs default convertion for Paging collection operator
 type DefaultPaginationConverter struct{}
 
+// DefaultSearchingConverter performs default convertion for Searching operator
+type DefaultSearchingConverter struct{}
+
 // DefaultPbToOrmConverter performs default convertion for all collection operators
 type DefaultPbToOrmConverter struct {
 	DefaultFilteringConditionConverter
 	DefaultSortingCriteriaConverter
 	DefaultFieldSelectionConverter
 	DefaultPaginationConverter
+	DefaultSearchingConverter
 }
 
 // NewDefaultPbToOrmConverter creates default converter for all collection operators
@@ -45,6 +49,7 @@ func NewDefaultPbToOrmConverter(pb proto.Message) CollectionOperatorsConverter {
 		DefaultSortingCriteriaConverter{},
 		DefaultFieldSelectionConverter{},
 		DefaultPaginationConverter{},
+		DefaultSearchingConverter{},
 	}
 }
 
@@ -360,3 +365,9 @@ func (converter *DefaultPaginationConverter) PaginationToGorm(ctx context.Contex
 	}
 	return 0, 0
 }
+
+func (converter *DefaultSearchingConverter) SearchingToGorm(ctx context.Context, s *query.Searching, fieldsForFTS []string, obj interface{}) (string, error) {
+	mask := GetFullTextSearchDBMask(obj, fieldsForFTS, " ")
+	fullTextSearchQuery := FormFullTextSearchQuery(mask)
+	return fullTextSearchQuery, nil
+}

gorm/filtering.go

@@ -57,7 +57,7 @@ func FilterStringToGorm(ctx context.Context, filter string, obj interface{}, pb
 	return FilteringToGormEx(ctx, f, obj, c)
 }
 
-//Deprecated: Use FilteringToGormEx instead
+// Deprecated: Use FilteringToGormEx instead
 // FilteringToGorm returns GORM Plain SQL representation of the filtering expression.
 func FilteringToGorm(ctx context.Context, m *query.Filtering, obj interface{}, pb proto.Message) (string, []interface{}, map[string]struct{}, error) {
 	c := &DefaultFilteringConditionConverter{&DefaultFilteringConditionProcessor{pb}}

gorm/searching.go

@@ -0,0 +1,58 @@
+package gorm
+
+import (
+	"reflect"
+	"time"
+)
+
+// GetFullTextSearchDBMask ...
+func GetFullTextSearchDBMask(object interface{}, fields []string, separator string) string {
+	mask := ""
+	objectVal := indirectValue(reflect.ValueOf(object))
+	if objectVal.Kind() != reflect.Struct {
+		return mask
+	}
+	fieldsSize := len(fields)
+	for i, fieldName := range fields {
+		fieldVal := objectVal.FieldByName(camelCase(fieldName))
+		if !fieldVal.IsValid() {
+			continue
+		}
+		underlyingVal := indirectValue(fieldVal)
+		if !underlyingVal.IsValid() {
+			switch fieldVal.Interface().(type) {
+			case *time.Time:
+				underlyingVal = fieldVal
+			default:
+				continue
+			}
+		}
+		switch underlyingVal.Interface().(type) {
+		case int32:
+			mask += fieldName
+		case string:
+			mask += fieldName
+			mask += " || '" + separator + "' || "
+			mask += "replace(" + fieldName + ", '@', ' ')"
+			mask += " || '" + separator + "' || "
+			mask += "replace(" + fieldName + ", '.', ' ')"
+		case *time.Time:
+			mask += "coalesce(to_char(" + fieldName + ", 'MM/DD/YY HH:MI pm'), '')"
+		case bool:
+			mask += fieldName
+		default:
+			continue
+		}
+		if i != fieldsSize-1 {
+			mask += " || '" + separator + "' || "
+		}
+	}
+
+	return mask
+}
+
+// FormFullTextSearchQuery ...
+func FormFullTextSearchQuery(mask string) string {
+	fullTextSearchQuery := "to_tsvector('simple', " + mask + ") @@ to_tsquery('simple', ?)"
+	return fullTextSearchQuery
+}

gorm/utilities.go

@@ -41,7 +41,7 @@ func HandleFieldPath(ctx context.Context, fieldPath []string, obj interface{}) (
 	return dbPath, "", nil
 }
 
-//HandleJSONFiledPath translate field path to JSONB path for postgres jsonb
+// HandleJSONFiledPath translate field path to JSONB path for postgres jsonb
 func HandleJSONFieldPath(ctx context.Context, fieldPath []string, obj interface{}, values ...string) (string, string, error) {
 	operator := "#>>"
 	if isRawJSON(values...) {
@@ -82,7 +82,7 @@ func isRawJSON(values ...string) bool {
 	return true
 }
 
-//TODO: add supprt for embeded objects
+// TODO: add supprt for embeded objects
 func IsJSONCondition(ctx context.Context, fieldPath []string, obj interface{}) bool {
 	fieldName := util.Camel(fieldPath[0])
 	objType := indirectType(reflect.TypeOf(obj))
@@ -189,6 +189,17 @@ func indirectType(t reflect.Type) reflect.Type {
 	}
 }
 
+func indirectValue(val reflect.Value) reflect.Value {
+	for {
+		switch val.Kind() {
+		case reflect.Ptr, reflect.Slice, reflect.Array:
+			val = val.Elem()
+		default:
+			return val
+		}
+	}
+}
+
 func isModel(t reflect.Type) bool {
 	kind := t.Kind()
 	_, isValuer := reflect.Zero(t).Interface().(driver.Valuer)
@@ -214,3 +225,13 @@ type EmptyFieldPathError struct {
 func (e *EmptyFieldPathError) Error() string {
 	return fmt.Sprintf("Empty field path is not allowed")
 }
+
+func camelCase(v string) string {
+	sp := strings.Split(v, "_")
+	r := make([]string, len(sp))
+	for i, v := range sp {
+		r[i] = strings.ToUpper(v[:1]) + v[1:]
+	}
+
+	return strings.Join(r, "")
+}

query/README.md

@@ -7,6 +7,7 @@ These types are:
 - `infoblox.api.Pagination`
 - `infoblox.api.PageInfo`(used in response)
 - `infoblox.api.FieldSelection`
+- `infoblox.api.Searching`
 
 ## Enabling *collection operators* in your application
 
@@ -22,6 +23,7 @@ message MyListRequest {
     infoblox.api.Sorting sorting = 2;
     infoblox.api.Pagination pagination = 3;
     infoblox.api.FieldSelection fields = 4;
+    infoblox.api.Searching searching = 5;
 }
 ```
 
@@ -162,3 +164,38 @@ server.WithGateway(
   )
 )
 ```
+## Searching
+
+The syntax of REST representation of `infoblox.api.Searching` is the following.
+
+| Request Parameter | Description                              |
+| ----------------- |------------------------------------------|
+| _fts              | A string expression which performs a full-text-search on the DB |
+
+Full-text-search is an optimized mechanism to retrieve data from the DB efficiently when the user is only aware of some portion of the data they are searching.
+
+#### Example:
+Consider that you have the following objects:
+```
+{
+ name: "my first object",
+ city: "Santa Clara",
+ country: "USA"
+},
+{
+ name: "my second object",
+ city: "Tacoma",
+ country: "USA"
+}
+```
+If "name" is provided as a field for FTS, the Searching query will look like:
+```
+...?_fts=my first object
+```
+In this case, only the first object would be returned.
+
+If "country" is provided as a field for FTS, the Searching query will look like:
+```
+...?_fts=USA
+```
+In this case, both the objects would be returned.