Add ApproveFields for approving flattened fields (#30)

ApproveFields approves flattened fields, including
runtime fields, rather than document _source.

We also add cmd/flatten-approvals for converting
existing _source-based approvals to flattened
fields. They may not be perfectly aligned (e.g.
we can't materialise runtime fields from source
in this way), but this gives us a way to perform
a mechanical translation before switching to
ApproveFields, minimising the non-mechanical diff.

cmd/flatten-approvals/main.go

@@ -0,0 +1,113 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+var inplace = flag.Bool("i", false, "modify file in place")
+
+func main() {
+	flag.Parse()
+	if err := flatten(flag.Args()); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func flatten(args []string) error {
+	var filepaths []string
+	for _, arg := range args {
+		matches, err := filepath.Glob(arg)
+		if err != nil {
+			return err
+		}
+		filepaths = append(filepaths, matches...)
+	}
+	for _, filepath := range filepaths {
+		if err := transform(filepath); err != nil {
+			return fmt.Errorf("error transforming %q: %w", filepath, err)
+		}
+	}
+	return nil
+}
+
+// transform []{"events": {"object": {"field": ...}}} to []{"field", "field", ...}
+func transform(filepath string) error {
+	var input struct {
+		Events []map[string]any `json:"events"`
+	}
+	if err := decodeJSONFile(filepath, &input); err != nil {
+		return fmt.Errorf("could not read existing approved events file: %w", err)
+	}
+	out := make([]map[string][]any, 0, len(input.Events))
+	for _, event := range input.Events {
+		fields := make(map[string][]any)
+		flattenFields("", event, fields)
+		out = append(out, fields)
+	}
+
+	var w io.Writer = os.Stdout
+	if *inplace {
+		f, err := os.Create(filepath)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		w = f
+	}
+	enc := json.NewEncoder(w)
+	enc.SetIndent("", "    ")
+	return enc.Encode(out)
+}
+
+func flattenFields(k string, v any, out map[string][]any) {
+	switch v := v.(type) {
+	case map[string]any:
+		for k2, v := range v {
+			if k != "" {
+				k2 = k + "." + k2
+			}
+			flattenFields(k2, v, out)
+		}
+	case []any:
+		for _, v := range v {
+			flattenFields(k, v, out)
+		}
+	default:
+		out[k] = append(out[k], v)
+	}
+}
+
+func decodeJSONFile(path string, out interface{}) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := json.NewDecoder(f).Decode(&out); err != nil {
+		return fmt.Errorf("cannot unmarshal file %q: %w", path, err)
+	}
+	return nil
+}

pkg/approvaltest/approvals.go

@@ -53,20 +53,6 @@ const (
 func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) {
 	t.Helper()
 
-	// Fields generated by the server (e.g. observer.*)
-	// agent which may change between tests.
-	//
-	// Ignore their values in comparisons, but compare
-	// existence: either the field exists in both, or neither.
-	dynamic = append([]string{
-		"ecs.version",
-		"event.ingested",
-		"observer.ephemeral_id",
-		"observer.hostname",
-		"observer.id",
-		"observer.version",
-	}, dynamic...)
-
 	// Sort events for repeatable diffs.
 	sort.Slice(hits, func(i, j int) bool {
 		return compareDocumentFields(hits[i].RawFields, hits[j].RawFields) < 0
@@ -79,6 +65,32 @@ func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic .
 	approveEventDocs(t, filepath.Join("approvals", name), sources, dynamic...)
 }
 
+// ApproveFields compares the fields of the search hits with the
+// contents of the file in "approvals/<name>.approved.json".
+//
+// Dynamic fields (@timestamp, observer.id, etc.) are replaced
+// with a static string for comparison. Integration tests elsewhere
+// use canned data to test fields that we do not cover here.
+//
+// TODO(axw) eventually remove ApproveEvents when we have updated
+// all calls to use ApproveFields. ApproveFields should be used
+// since it includes runtime fields, whereas ApproveEvents only
+// looks at _source.
+func ApproveFields(t testing.TB, name string, hits []espoll.SearchHit, dynamic ...string) {
+	t.Helper()
+
+	// Sort events for repeatable diffs.
+	sort.Slice(hits, func(i, j int) bool {
+		return compareDocumentFields(hits[i].RawFields, hits[j].RawFields) < 0
+	})
+
+	fields := make([][]byte, len(hits))
+	for i, hit := range hits {
+		fields[i] = hit.RawFields
+	}
+	approveFields(t, filepath.Join("approvals", name), fields, dynamic...)
+}
+
 // approveEventDocs compares the given event documents with
 // the contents of the file in "<name>.approved.json".
 //
@@ -89,6 +101,20 @@ func ApproveEvents(t testing.TB, name string, hits []espoll.SearchHit, dynamic .
 func approveEventDocs(t testing.TB, name string, eventDocs [][]byte, dynamic ...string) {
 	t.Helper()
 
+	// Fields generated by the server (e.g. observer.*)
+	// agent which may change between tests.
+	//
+	// Ignore their values in comparisons, but compare
+	// existence: either the field exists in both, or neither.
+	dynamic = append([]string{
+		"ecs.version",
+		"event.ingested",
+		"observer.ephemeral_id",
+		"observer.hostname",
+		"observer.id",
+		"observer.version",
+	}, dynamic...)
+
 	// Rewrite all dynamic fields to have a known value,
 	// so dynamic fields don't affect diffs.
 	events := make([]interface{}, len(eventDocs))
@@ -117,6 +143,42 @@ func approveEventDocs(t testing.TB, name string, eventDocs [][]byte, dynamic ...
 	approve(t, name, received)
 }
 
+func approveFields(t testing.TB, name string, docs [][]byte, dynamic ...string) {
+	t.Helper()
+
+	// Fields generated by the server (e.g. observer.*)
+	// agent which may change between tests.
+	//
+	// Ignore their values in comparisons, but compare
+	// existence: either the field exists in both, or neither.
+	dynamic = append([]string{
+		"ecs.version",
+		"event.ingested",
+		"observer.ephemeral_id",
+		"observer.hostname",
+		"observer.id",
+		"observer.version",
+	}, dynamic...)
+
+	// Rewrite all dynamic fields to have a known value,
+	// so dynamic fields don't affect diffs.
+	decodedDocs := make([]any, len(docs))
+	for i, doc := range docs {
+		var fields map[string]any
+		if err := json.Unmarshal(doc, &fields); err != nil {
+			t.Fatal(err)
+		}
+		for _, field := range dynamic {
+			if _, ok := fields[field]; ok {
+				fields[field] = []any{"dynamic"}
+			}
+		}
+		decodedDocs[i] = fields
+	}
+
+	approve(t, name, decodedDocs)
+}
+
 // approve compares the given value with the contents of the file
 // "<name>.approved.json".
 //