From 8151648063761150781f1da59070038d469f7cf3 Mon Sep 17 00:00:00 2001 From: Josh Carp Date: Wed, 14 Jan 2026 10:23:53 -0500 Subject: [PATCH] Add golden tests. It's easy to make changes to the code generation logic that generate sdk code that appears to be correct and passes unit tests, but doesn't interact correctly with nexus. This patch introduces the concept of golden tests for nexus endpoints: we fetch real responses from a few endpoints of interest, check in the resulting json, and assert that we can unmarshal it into our generated types and marshal it back to equivalent json correctly. We also include a script and make target for refreshing golden files. --- Makefile | 5 + oxide/golden_test.go | 115 +++++++++++ oxide/testdata/main.go | 179 ++++++++++++++++++ .../recordings/disk_list_response.json | 1 + .../loopback_addresses_response.json | 1 + .../recordings/timeseries_query_response.json | 1 + 6 files changed, 302 insertions(+) create mode 100644 oxide/golden_test.go create mode 100644 oxide/testdata/main.go create mode 100644 oxide/testdata/recordings/disk_list_response.json create mode 100644 oxide/testdata/recordings/loopback_addresses_response.json create mode 100644 oxide/testdata/recordings/timeseries_query_response.json diff --git a/Makefile b/Makefile index 5a01cf4..acd4246 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,11 @@ test: ## Runs the go tests. @ echo "+ Running Go tests..." @ $(GO) test -v -tags "$(BUILDTAGS)" ./... +.PHONY: golden-fixtures +golden-fixtures: ## Refreshes golden test fixtures. Requires OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT. + @ echo "+ Refreshing golden test fixtures..." + @ $(GO) run ./oxide/testdata/main.go + .PHONY: vet vet: ## Verifies `go vet` passes. @ echo "+ Verifying go vet passes..." diff --git a/oxide/golden_test.go b/oxide/golden_test.go new file mode 100644 index 0000000..6845697 --- /dev/null +++ b/oxide/golden_test.go @@ -0,0 +1,115 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package oxide + +import ( + "encoding/json" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" +) + +// TestGoldenRoundTrip tests that real API responses can be unmarshaled and +// marshaled back to equivalent JSON. This catches mismatches between our +// generated types and the actual API format. +// +// To refresh the fixtures, run: +// +// go run ./oxide/testdata/main.go +func TestGoldenRoundTrip(t *testing.T) { + tests := []struct { + name string + fixture string + test func(t *testing.T, fixture string) + }{ + { + name: "timeseries_query_response", + fixture: "testdata/recordings/timeseries_query_response.json", + test: testRoundTrip[OxqlQueryResult], + }, + { + name: "disk_list_response", + fixture: "testdata/recordings/disk_list_response.json", + test: testRoundTrip[DiskResultsPage], + }, + { + name: "loopback_addresses_response", + fixture: "testdata/recordings/loopback_addresses_response.json", + test: testRoundTrip[LoopbackAddressResultsPage], + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.test(t, tt.fixture) + }) + } +} + +func testRoundTrip[T any](t *testing.T, fixturePath string) { + data, err := os.ReadFile(fixturePath) + require.NoError(t, err, "failed to read fixture") + + var typed T + err = json.Unmarshal(data, &typed) + require.NoError(t, err, "failed to unmarshal fixture") + + remarshaled, err := json.Marshal(typed) + require.NoError(t, err, "failed to marshal") + + var expected, actual any + require.NoError(t, json.Unmarshal(data, &expected)) + require.NoError(t, json.Unmarshal(remarshaled, &actual)) + + expected = stripNulls(expected) + actual = stripNulls(actual) + + if diff := cmp.Diff(expected, actual, timestampComparer()); diff != "" { + t.Errorf("round-trip mismatch (-fixture +remarshaled):\n%s", diff) + } +} + +// timestampComparer returns a cmp.Option that compares timestamp strings +// by their actual time value, handling precision differences. Rust and go format timestamps +// slightly differently, so we need to normalize to avoid spurious differences in marshalled values. +func timestampComparer() cmp.Option { + return cmp.Comparer(func(a, b string) bool { + ta, errA := time.Parse(time.RFC3339Nano, a) + tb, errB := time.Parse(time.RFC3339Nano, b) + if errA == nil && errB == nil { + return ta.Equal(tb) + } + return a == b + }) +} + +// stripNulls recursively removes null values from JSON-unmarshaled data. We use this workaround +// because the SDK and API don't always handle null fields consistently. +// +// TODO: Investigate options to harmonize null handling across services so that we don't have to +// pre-process the responses here. +func stripNulls(v any) any { + switch val := v.(type) { + case map[string]any: + result := make(map[string]any) + for k, v := range val { + if v != nil { + result[k] = stripNulls(v) + } + } + return result + case []any: + result := make([]any, len(val)) + for i, v := range val { + result[i] = stripNulls(v) + } + return result + default: + return v + } +} diff --git a/oxide/testdata/main.go b/oxide/testdata/main.go new file mode 100644 index 0000000..57b23e0 --- /dev/null +++ b/oxide/testdata/main.go @@ -0,0 +1,179 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//go:build ignore + +// This script records real API responses for use in golden file tests. +// Run with: go run ./oxide/testdata/main.go [-api-version VERSION] +// +// Requires OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT environment variables. +// Optionally pass -api-version to set the API-Version header on requests. +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" +) + +var ( + apiVersion = flag.String("api-version", "", "API version to send in requests (optional)") + + client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } +) + +func main() { + flag.Parse() + + host := os.Getenv("OXIDE_HOST") + token := os.Getenv("OXIDE_TOKEN") + project := os.Getenv("OXIDE_PROJECT") + + if host == "" || token == "" || project == "" { + log.Fatalf("OXIDE_HOST, OXIDE_TOKEN, and OXIDE_PROJECT environment variables must be set") + } + + if *apiVersion != "" { + fmt.Printf("Using API-Version: %s\n", *apiVersion) + } + + testdataDir := "./oxide/testdata/recordings" + + recordTimeseriesQuery(host, token, testdataDir) + recordDiskList(host, token, project, testdataDir) + recordLoopbackAddresses(host, token, testdataDir) +} + +func recordTimeseriesQuery(host, token, testdataDir string) { + fmt.Println("Recording timeseries query response...") + + body := `{"query": "get hardware_component:voltage | filter slot == 0 && sensor == \"V1P0_MGMT\" | filter timestamp > @now() - 5m | last 5"}` + data, err := doRequest("POST", host+"/v1/system/timeseries/query", token, body) + if err != nil { + log.Printf("Warning: timeseries query failed: %v", err) + return + } + + normalized, err := normalizeJSON(data) + if err != nil { + log.Printf("Warning: failed to normalize JSON: %v", err) + return + } + if err := saveFixture(testdataDir, "timeseries_query_response.json", normalized); err != nil { + log.Printf("Warning: %v", err) + return + } +} + +func recordDiskList(host, token, project, testdataDir string) { + fmt.Println("Recording disk list response...") + + url := fmt.Sprintf("%s/v1/disks?project=%s&limit=5", host, project) + data, err := doRequest("GET", url, token, "") + if err != nil { + log.Printf("Warning: disk list failed: %v", err) + return + } + + normalized, err := normalizeJSON(data) + if err != nil { + log.Printf("Warning: failed to normalize JSON: %v", err) + return + } + if err := saveFixture(testdataDir, "disk_list_response.json", normalized); err != nil { + log.Printf("Warning: %v", err) + return + } +} + +func recordLoopbackAddresses(host, token, testdataDir string) { + fmt.Println("Recording loopback addresses response...") + + url := fmt.Sprintf("%s/v1/system/networking/loopback-address?limit=5", host) + data, err := doRequest("GET", url, token, "") + if err != nil { + log.Printf("Warning: loopback addresses failed: %v", err) + return + } + + normalized, err := normalizeJSON(data) + if err != nil { + log.Printf("Warning: failed to normalize JSON: %v", err) + return + } + if err := saveFixture(testdataDir, "loopback_addresses_response.json", normalized); err != nil { + log.Printf("Warning: %v", err) + return + } +} + +// doRequest makes a request to the configured nexus instance. We use the standard library here +// and not our own sdk because we're generating test files to verify the generated code. +func doRequest(method, url, token, body string) ([]byte, error) { + var reqBody io.Reader + if body != "" { + reqBody = bytes.NewBufferString(body) + } + + req, err := http.NewRequest(method, url, reqBody) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+token) + if *apiVersion != "" { + req.Header.Set("API-Version", *apiVersion) + } + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, respBody) + } + + return io.ReadAll(resp.Body) +} + +func saveFixture(testdataDir, filename string, data []byte) error { + path := filepath.Join(testdataDir, filename) + if err := os.WriteFile(path, data, 0644); err != nil { + return fmt.Errorf("failed to write %s: %w", path, err) + } + return nil +} + +// normalizeJSON strips undocumented fields from API responses. +func normalizeJSON(data []byte) ([]byte, error) { + var v any + if err := json.Unmarshal(data, &v); err != nil { + return nil, err + } + + // Nexus returns an undocumented `query_summaries` field that's not in the OpenAPI spec. Ignore it for now. + // + // TODO: fully drop `query_summaries` from nexus unless requested. + if m, ok := v.(map[string]any); ok { + delete(m, "query_summaries") + } + + return json.Marshal(v) +} diff --git a/oxide/testdata/recordings/disk_list_response.json b/oxide/testdata/recordings/disk_list_response.json new file mode 100644 index 0000000..09e09b1 --- /dev/null +++ b/oxide/testdata/recordings/disk_list_response.json @@ -0,0 +1 @@ +{"items":[{"block_size":512,"description":"boot","device_path":"/mnt/boot","disk_type":"distributed","id":"9caa44a1-2683-4f52-a5ca-8a5ee96c7362","image_id":"14e36227-0984-484e-8c94-baab1a6be648","name":"boot","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":21474836480,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-09-25T15:11:27.776013Z","time_modified":"2025-09-25T15:11:27.776013Z"},{"block_size":512,"description":"Created as a boot disk for builder-omni","device_path":"/mnt/builder-omni-omnios-bloody-20250124-d6fb1a","disk_type":"distributed","id":"810c075b-27d0-41b1-b259-7551fed1a004","image_id":"7e7c352c-f4aa-4d8b-a34d-7e7c6eeb615a","name":"builder-omni-omnios-bloody-20250124-d6fb1a","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1098437885952,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-11-27T02:51:41.697116Z","time_modified":"2025-11-27T02:51:41.697116Z"},{"block_size":512,"description":"Created as a boot disk for builder-omni","device_path":"/mnt/builder-omni-omnios-r151056-cloud-228ca2","disk_type":"distributed","id":"2c310fbf-a0bf-4c31-8a3d-a4e38feb53c7","image_id":"6e989035-2319-4980-88a3-31fe7112d87f","name":"builder-omni-omnios-r151056-cloud-228ca2","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1073741824000,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-12-05T20:57:41.635747Z","time_modified":"2025-12-05T20:57:41.635747Z"},{"block_size":512,"description":"Created as a boot disk for ch-builder-omni","device_path":"/mnt/ch-builder-omni-omnios-cloud-1693471113-aadda2","disk_type":"distributed","id":"d1c47f4b-a998-42aa-a273-3b7b87e3e780","image_id":"04ffb229-6c78-4bc4-baa3-b4f07f16bea3","name":"ch-builder-omni-omnios-cloud-1693471113-aadda2","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":1073741824000,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-11-27T02:21:32.352141Z","time_modified":"2025-11-27T02:21:32.352141Z"},{"block_size":4096,"description":"data","device_path":"/mnt/data","disk_type":"distributed","id":"84a528d2-b2d5-4420-ae39-3b4643d46a92","image_id":null,"name":"data","project_id":"4fb1705d-6f8e-4711-9c38-85fe0fbdc8c1","size":214748364800,"snapshot_id":null,"state":{"state":"detached"},"time_created":"2025-09-25T15:11:27.728556Z","time_modified":"2025-09-25T15:11:27.728556Z"}],"next_page":"eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJuYW1lX2FzY2VuZGluZyIsInByb2plY3QiOiJjYXJwIiwibGFzdF9zZWVuIjoiZGF0YSJ9fQ=="} \ No newline at end of file diff --git a/oxide/testdata/recordings/loopback_addresses_response.json b/oxide/testdata/recordings/loopback_addresses_response.json new file mode 100644 index 0000000..23cf21a --- /dev/null +++ b/oxide/testdata/recordings/loopback_addresses_response.json @@ -0,0 +1 @@ +{"items":[{"address":"fd00:99::1/64","address_lot_block_id":"e30373de-88b5-427c-acb7-65f896695e40","id":"887c5912-4bc6-4b6e-a2d6-64d4aa64b5e0","rack_id":"de608e01-b8e4-4d93-b972-a7dbed36dd22","switch_location":"switch0"},{"address":"fd00:99::1/64","address_lot_block_id":"e30373de-88b5-427c-acb7-65f896695e40","id":"b7101671-162a-4a5a-b30e-1bd7696984c5","rack_id":"de608e01-b8e4-4d93-b972-a7dbed36dd22","switch_location":"switch1"}],"next_page":"eyJ2IjoidjEiLCJwYWdlX3N0YXJ0Ijp7InNvcnRfYnkiOiJpZF9hc2NlbmRpbmciLCJsYXN0X3NlZW4iOiJiNzEwMTY3MS0xNjJhLTRhNWEtYjMwZS0xYmQ3Njk2OTg0YzUifX0="} \ No newline at end of file diff --git a/oxide/testdata/recordings/timeseries_query_response.json b/oxide/testdata/recordings/timeseries_query_response.json new file mode 100644 index 0000000..6f39820 --- /dev/null +++ b/oxide/testdata/recordings/timeseries_query_response.json @@ -0,0 +1 @@ +{"tables":[{"name":"hardware_component:voltage","timeseries":[{"fields":{"chassis_kind":{"type":"string","value":"switch"},"chassis_model":{"type":"string","value":"913-0000006"},"chassis_revision":{"type":"u32","value":4},"chassis_serial":{"type":"string","value":"BRM44220012"},"component_id":{"type":"string","value":"U21"},"component_kind":{"type":"string","value":"tps546b24a"},"description":{"type":"string","value":"V1P0_MGMT rail"},"gateway_id":{"type":"uuid","value":"c0cea24f-ab91-4026-8593-870d64c34673"},"hubris_archive_id":{"type":"string","value":"29806c00ad5fc171"},"rack_id":{"type":"uuid","value":"de608e01-b8e4-4d93-b972-a7dbed36dd22"},"sensor":{"type":"string","value":"V1P0_MGMT"},"slot":{"type":"u32","value":0}},"points":{"start_times":null,"timestamps":["2026-01-12T21:50:24.195794351Z","2026-01-12T21:50:25.192639350Z","2026-01-12T21:50:26.193419069Z","2026-01-12T21:50:27.193322592Z","2026-01-12T21:50:28.290379572Z"],"values":[{"metric_type":"gauge","values":{"type":"double","values":[1.001953125,1.001953125,1.001953125,1.001953125,1.001953125]}}]}},{"fields":{"chassis_kind":{"type":"string","value":"switch"},"chassis_model":{"type":"string","value":"913-0000006"},"chassis_revision":{"type":"u32","value":4},"chassis_serial":{"type":"string","value":"BRM44220012"},"component_id":{"type":"string","value":"U21"},"component_kind":{"type":"string","value":"tps546b24a"},"description":{"type":"string","value":"V1P0_MGMT rail"},"gateway_id":{"type":"uuid","value":"eb645e8f-4228-43fa-9a55-97feabf8ab66"},"hubris_archive_id":{"type":"string","value":"29806c00ad5fc171"},"rack_id":{"type":"uuid","value":"de608e01-b8e4-4d93-b972-a7dbed36dd22"},"sensor":{"type":"string","value":"V1P0_MGMT"},"slot":{"type":"u32","value":0}},"points":{"start_times":null,"timestamps":["2026-01-12T21:50:24.492202137Z","2026-01-12T21:50:25.562686874Z","2026-01-12T21:50:26.493035980Z","2026-01-12T21:50:27.492070298Z","2026-01-12T21:50:28.709615454Z"],"values":[{"metric_type":"gauge","values":{"type":"double","values":[1.001953125,1.001953125,1.001953125,1.001953125,1.001953125]}}]}}]}]} \ No newline at end of file