Sort on fields, rather than _source (#8)

In ApproveEvents, sort events on the returned `fields`,
which may include runtime fields or constant_keywords
defined in the mapping but which aren't in _source.
Also, include the raw, unparsed fields in search hits.

pkg/approvaltest/events.go

@@ -20,6 +20,7 @@ package approvaltest
 import (
 	"path/filepath"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/tidwall/gjson"
@@ -91,15 +92,23 @@ func (hits apmEventSearchHits) Swap(i, j int) {
 
 func (hits apmEventSearchHits) Less(i, j int) bool {
 	for _, field := range apmEventSortFields {
-		ri := gjson.GetBytes(hits[i].RawSource, field)
-		rj := gjson.GetBytes(hits[j].RawSource, field)
+		path := strings.ReplaceAll(field, ".", "\\.")
+		ri := gjson.GetBytes(hits[i].RawFields, path)
+		rj := gjson.GetBytes(hits[j].RawFields, path)
+		if ri.Exists() && rj.Exists() {
+			// 'fields' always returns an array
+			// of values, but all of the fields
+			// we sort on are single value fields.
+			ri = ri.Array()[0]
+			rj = rj.Array()[0]
+		}
 		if ri.Less(rj, true) {
 			return true
 		}
 		if rj.Less(ri, true) {
 			return false
 		}
 	}
-	// All _source fields are equivalent, so compare doc _ids.
+	// All sort fields are equivalent, so compare doc _ids.
 	return hits[i].ID < hits[j].ID
 }

pkg/espoll/search.go

@@ -158,15 +158,16 @@ type SearchHit struct {
 	Fields    map[string][]any
 	Source    map[string]any
 	RawSource json.RawMessage
+	RawFields json.RawMessage
 }
 
 func (h *SearchHit) UnmarshalJSON(data []byte) error {
 	var searchHit struct {
-		Index  string           `json:"_index"`
-		ID     string           `json:"_id"`
-		Score  float64          `json:"_score"`
-		Source json.RawMessage  `json:"_source"`
-		Fields map[string][]any `json:"fields"`
+		Index  string          `json:"_index"`
+		ID     string          `json:"_id"`
+		Score  float64         `json:"_score"`
+		Source json.RawMessage `json:"_source"`
+		Fields json.RawMessage `json:"fields"`
 	}
 	if err := json.Unmarshal(data, &searchHit); err != nil {
 		return err
@@ -175,9 +176,16 @@ func (h *SearchHit) UnmarshalJSON(data []byte) error {
 	h.ID = searchHit.ID
 	h.Score = searchHit.Score
 	h.RawSource = searchHit.Source
-	h.Fields = searchHit.Fields
+	h.RawFields = searchHit.Fields
 	h.Source = make(map[string]any)
-	return json.Unmarshal(h.RawSource, &h.Source)
+	h.Fields = make(map[string][]interface{})
+	if err := json.Unmarshal(h.RawSource, &h.Source); err != nil {
+		return fmt.Errorf("error unmarshaling _source: %w", err)
+	}
+	if err := json.Unmarshal(h.RawFields, &h.Fields); err != nil {
+		return fmt.Errorf("error unmarshaling fields: %w", err)
+	}
+	return nil
 }
 
 func (h *SearchHit) UnmarshalSource(out any) error {