Merge pull request #67 from grafana/66-adds-the-capability-to-test-generated-files

feat: added testability of generated files

Author: szkiba

README.md

@@ -472,6 +472,7 @@ name   | reqired | default | description
 in     |   yes   |         | input file name
 out    |    no   |  stdout | output file name
 api    |    no   |         | output directory name
+test   |    no   |         | api path(s) to test after generation
 quiet  |    no   | `false` | no output, only validation
 loose  |    no   | `false` | skip JSON schema validation
 lint   |    no   | `false` | enable built-in linter
@@ -483,6 +484,8 @@ In GitHub action mode, the change can be indicated by comparing the output to a
 
 The `api` parameter can be used to specify a directory into which the outputs are written. The `registry.json` file is placed in the root directory. The `extension.json` file and the `badge.svg` file are placed in a directory with the same name as the go module path of the extension (if the `lint` parameter is `true`).
 
+The `test` parameter can be used to test registry and catalog files generated with the `api` parameter. The test is successful if the file is not empty, contains `k6` and at least one extension, and if all extensions meet the minimum requirements (e.g. it has versions).
+
 **Outputs**
 
 name    | description
@@ -521,6 +524,8 @@ The output of the generation will be written to the standard output by default.
 
 The `--api` flag can be used to specify a directory to which the outputs will be written. The `registry.json` file is placed in the root directory. The `extension.json` file and the `badge.svg` file (if the `--lint` flag is used) are placed in a directory with the same name as the extension's go module path.
 
+The `--test` flag can be used to test registry and catalog files generated with the `--api` flag. The test is successful if the file is not empty, contains `k6` and at least one extension, and if all extensions meet the minimum requirements (e.g. it has versions).
+
 
 ```
 k6registry [flags] [source-file]
@@ -529,15 +534,17 @@ k6registry [flags] [source-file]
 ### Flags
 
 ```
-  -o, --out string   write output to file instead of stdout
-      --api string   write outputs to directory instead of stdout
-  -q, --quiet        no output, only validation
-      --loose        skip JSON schema validation
-      --lint         enable built-in linter
-  -c, --compact      compact instead of pretty-printed output
-      --catalog      generate catalog instead of registry
-  -V, --version      print version
-  -h, --help         help for k6registry
+  -o, --out string     write output to file instead of stdout
+      --api string     write outputs to directory instead of stdout
+      --test strings   test api path(s) (example: /registry.json,/catalog.json)
+  -q, --quiet          no output, only validation
+      --loose          skip JSON schema validation
+      --lint           enable built-in linter
+  -c, --compact        compact instead of pretty-printed output
+      --catalog        generate catalog instead of registry
+  -v, --verbose        verbose logging
+  -V, --version        print version
+  -h, --help           help for k6registry
 ```
 
 <!-- #endregion cli -->

action.yml

@@ -47,6 +47,10 @@ inputs:
     description: reference output URL for change detection
     required: false
 
+  test:
+    description: api path(s) to test after generation
+    required: false
+
 outputs:
   changed:
     description: "true if the output has changed compared to ref"

cmd/api.go

@@ -2,8 +2,11 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/grafana/k6registry"
 	"github.com/narqo/go-badge"
@@ -338,3 +341,65 @@ func badgecolor(grade k6registry.Grade) badge.Color {
 		return "blue"
 	}
 }
+
+func isCatalog(filename string) bool {
+	basename := strings.TrimSuffix(filename, filepath.Ext(filename))
+
+	return strings.HasSuffix(basename, "catalog")
+}
+
+func testAPI(paths []string, dir string) error {
+	for _, name := range paths {
+		name = filepath.FromSlash(strings.TrimPrefix(name, "/"))
+		name = filepath.Join(dir, name)
+
+		if err := testFile(name); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+//nolint:forbidigo
+func testFile(filename string) error {
+	data, err := os.ReadFile(filepath.Clean(filename))
+	if err != nil {
+		return err
+	}
+
+	var catalog k6registry.Catalog
+
+	if isCatalog(filename) {
+		if err := json.Unmarshal(data, &catalog); err != nil {
+			return err
+		}
+	} else {
+		var registry k6registry.Registry
+
+		if err := json.Unmarshal(data, &registry); err != nil {
+			return err
+		}
+
+		catalog = k6registry.RegistryToCatalog(registry)
+	}
+
+	_, found := catalog[k6ImportPath]
+	if !found {
+		return fmt.Errorf("%w: %s: missing %s module", errTestFailed, filename, k6Module)
+	}
+
+	if len(catalog) == 1 {
+		return fmt.Errorf("%w: %s: missing extensions", errTestFailed, filename)
+	}
+
+	for _, ext := range catalog {
+		if ok, msgs := lintExtension(ext); !ok && len(msgs) != 0 {
+			return fmt.Errorf("%w: %s: %s", errTestFailed, filename, msgs[0])
+		}
+	}
+
+	return nil
+}
+
+var errTestFailed = errors.New("test failed")

cmd/cmd.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/json"
+	"io"
 	"io/fs"
 	"log/slog"
 	"os"
@@ -25,6 +26,7 @@ type options struct {
 	loose   bool
 	lint    bool
 	api     string
+	test    []string
 }
 
 // New creates new cobra command for exec command.
@@ -68,6 +70,7 @@ func New(levelVar *slog.LevelVar) (*cobra.Command, error) {
 
 	flags.StringVarP(&opts.out, "out", "o", "", "write output to file instead of stdout")
 	flags.StringVar(&opts.api, "api", "", "write outputs to directory instead of stdout")
+	flags.StringSliceVar(&opts.test, "test", []string{}, "test api path(s) (example: /registry.json,/catalog.json)")
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "no output, only validation")
 	flags.BoolVar(&opts.loose, "loose", false, "skip JSON schema validation")
 	flags.BoolVar(&opts.lint, "lint", false, "enable built-in linter")
@@ -139,13 +142,21 @@ func run(ctx context.Context, args []string, opts *options) (result error) {
 	}
 
 	if len(opts.api) != 0 {
-		return writeAPI(registry, opts.api)
+		if err = writeAPI(registry, opts.api); err != nil {
+			return err
+		}
+
+		return testAPI(opts.test, opts.api)
 	}
 
 	if opts.quiet {
 		return nil
 	}
 
+	return writeOutput(registry, output, opts)
+}
+
+func writeOutput(registry k6registry.Registry, output io.Writer, opts *options) error {
 	encoder := json.NewEncoder(output)
 
 	if !opts.compact {
@@ -158,12 +169,7 @@ func run(ctx context.Context, args []string, opts *options) (result error) {
 		source = k6registry.RegistryToCatalog(registry)
 	}
 
-	err = encoder.Encode(source)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return encoder.Encode(source)
 }
 
 const (

cmd/help.md

@@ -9,3 +9,5 @@ The source is read from file specified as command line argument. If it is missin
 The output of the generation will be written to the standard output by default. The output can be saved to a file using the `-o/--out` flag.
 
 The `--api` flag can be used to specify a directory to which the outputs will be written. The `registry.json` file is placed in the root directory. The `extension.json` file and the `badge.svg` file (if the `--lint` flag is used) are placed in a directory with the same name as the extension's go module path.
+
+The `--test` flag can be used to test registry and catalog files generated with the `--api` flag. The test is successful if the file is not empty, contains `k6` and at least one extension, and if all extensions meet the minimum requirements (e.g. it has versions).

cmd/k6registry/main.go

@@ -5,7 +5,9 @@ import (
 	"log"
 	"log/slog"
 	"os"
+	"strings"
 
+	"github.com/google/shlex"
 	"github.com/grafana/k6registry/cmd"
 	sloglogrus "github.com/samber/slog-logrus/v2"
 	"github.com/sirupsen/logrus"
@@ -102,6 +104,15 @@ func getArgs() []string {
 		args = append(args, "--out", out)
 	}
 
+	if paths := getenv("INPUT_TEST", ""); len(paths) != 0 {
+		parts, err := shlex.Split(paths)
+		if err != nil {
+			paths = strings.Join(parts, ",")
+		}
+
+		args = append(args, "--test", paths)
+	}
+
 	args = append(args, getenv("INPUT_IN", ""))
 
 	return args

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/cli/go-gh/v2 v2.9.0
 	github.com/go-git/go-git/v5 v5.12.0
 	github.com/google/go-github/v62 v62.0.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/grafana/clireadme v0.1.0
 	github.com/grafana/k6lint v0.1.0
 	github.com/narqo/go-badge v0.0.0-20230821190521-c9a75c019a59

go.sum

@@ -75,6 +75,8 @@ github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwM
 github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/grafana/clireadme v0.1.0 h1:KYEYSnYdSzmHf3bufaK6fQZ5j4dzvM/T+G6Ba+qNnAM=
 github.com/grafana/clireadme v0.1.0/go.mod h1:Wy4KIG2ZBGMYAYyF9l7qAy+yoJVasqk/txsRgoRI3gc=
 github.com/grafana/k6lint v0.1.0 h1:egUuy8Dmc1Wi5eXBnbGC2QkZIt71KTrOnMr/B5bgkzk=

releases/v0.1.24.md

@@ -0,0 +1,9 @@
+k6registry `v0.1.24` is here 🎉!
+
+This is an internal maintenance release.
+
+**Added testability of generated files**
+
+The generated registry and catalog will play an important role in the k6 binary provisioning subsystem, so it is important that there is no loss of service due to generated files. It is advisable to test the generated files to see if they meet the minimum requirements.
+
+The `--test` flag can be used to test registry and catalog files generated with the `--api` flag. The test is successful if the file is not empty, contains k6 and at least one extension, and if all extensions meet the minimum requirements (e.g. it has versions).