Merge pull request #78 from grafana/77-support-for-import-of-extensions-from-other-registry

feat: Support for import extension properties from other registry

Author: szkiba

CONTRIBUTING.md

@@ -68,8 +68,9 @@ mdcode update README.md
 ## custom - Update custom example
 
 ```bash
-go run ./cmd/k6registry --lint -o docs/custom.json docs/custom.yaml
-go run ./cmd/k6registry --lint --catalog -o docs/custom-catalog.json docs/custom.yaml
+export ORIGIN=https://registry.k6.io/registry.json
+go run ./cmd/k6registry --lint -o docs/custom.json --origin $ORIGIN docs/custom.yaml
+go run ./cmd/k6registry --lint --catalog -o docs/custom-catalog.json  --origin $ORIGIN docs/custom.yaml
 ```
 
 ## readme - Update README.md

README.md

@@ -479,6 +479,7 @@ lint   |    no   | `false` | enable built-in linter
 compact|    no   | `false` | compact instead of pretty-printed output
 catalog|    no   | `false` | generate catalog instead of registry
 ref    |    no   |         | reference output URL for change detection
+origin |    no   |         | external registry URL for default values
 
 In GitHub action mode, the change can be indicated by comparing the output to a reference output. The reference output URL can be passed in the `ref` action parameter. The `changed` output variable will be `true` or `false` depending on whether the output has changed or not compared to the reference output.
 
@@ -534,17 +535,18 @@ k6registry [flags] [source-file]
 ### Flags
 
 ```
-  -o, --out string     write output to file instead of stdout
-      --api string     write outputs to directory instead of stdout
-      --test strings   test api path(s) (example: /registry.json,/catalog.json)
-  -q, --quiet          no output, only validation
-      --loose          skip JSON schema validation
-      --lint           enable built-in linter
-  -c, --compact        compact instead of pretty-printed output
-      --catalog        generate catalog instead of registry
-  -v, --verbose        verbose logging
-  -V, --version        print version
-  -h, --help           help for k6registry
+  -o, --out string      write output to file instead of stdout
+      --api string      write outputs to directory instead of stdout
+      --origin string   external registry URL for default values
+      --test strings    test api path(s) (example: /registry.json,/catalog.json)
+  -q, --quiet           no output, only validation
+      --loose           skip JSON schema validation
+      --lint            enable built-in linter
+  -c, --compact         compact instead of pretty-printed output
+      --catalog         generate catalog instead of registry
+  -v, --verbose         verbose logging
+  -V, --version         print version
+  -h, --help            help for k6registry
 ```
 
 <!-- #endregion cli -->

action.yml

@@ -43,6 +43,10 @@ inputs:
     description: generate catalog instead of registry
     required: false
 
+  origin:
+    description: external registry URL for default values
+    required: false
+
   ref:
     description: reference output URL for change detection
     required: false

cmd/cmd.go

@@ -27,6 +27,7 @@ type options struct {
 	lint    bool
 	api     string
 	test    []string
+	origin  string
 }
 
 // New creates new cobra command for exec command.
@@ -70,6 +71,7 @@ func New(levelVar *slog.LevelVar) (*cobra.Command, error) {
 
 	flags.StringVarP(&opts.out, "out", "o", "", "write output to file instead of stdout")
 	flags.StringVar(&opts.api, "api", "", "write outputs to directory instead of stdout")
+	flags.StringVar(&opts.origin, "origin", "", "external registry URL for default values")
 	flags.StringSliceVar(&opts.test, "test", []string{}, "test api path(s) (example: /registry.json,/catalog.json)")
 	flags.BoolVarP(&opts.quiet, "quiet", "q", false, "no output, only validation")
 	flags.BoolVar(&opts.loose, "loose", false, "skip JSON schema validation")
@@ -136,7 +138,7 @@ func run(ctx context.Context, args []string, opts *options) (result error) {
 		output = file
 	}
 
-	registry, err := load(ctx, input, opts.loose, opts.lint)
+	registry, err := load(ctx, input, opts.loose, opts.lint, opts.origin)
 	if err != nil {
 		return err
 	}

cmd/k6registry/main.go

@@ -113,6 +113,10 @@ func getArgs() []string {
 		args = append(args, "--test", paths)
 	}
 
+	if origin := getenv("INPUT_ORIGIN", ""); len(origin) != 0 {
+		args = append(args, "--origin", origin)
+	}
+
 	args = append(args, getenv("INPUT_IN", ""))
 
 	return args

cmd/load.go

@@ -31,7 +31,7 @@ func k6AsExtension() k6registry.Extension {
 	}
 }
 
-func load(ctx context.Context, in io.Reader, loose bool, lint bool) (k6registry.Registry, error) {
+func loadSource(in io.Reader, loose bool) (k6registry.Registry, error) {
 	var (
 		raw []byte
 		err error
@@ -58,10 +58,40 @@ func load(ctx context.Context, in io.Reader, loose bool, lint bool) (k6registry.
 		return nil, err
 	}
 
-	registry = append(registry, k6AsExtension())
+	k6 := false
+
+	for idx := range registry {
+		if registry[idx].Module == k6Module {
+			k6 = true
+			break
+		}
+	}
+
+	if !k6 {
+		registry = append(registry, k6AsExtension())
+	}
+
+	return registry, nil
+}
+
+func load(ctx context.Context, in io.Reader, loose bool, lint bool, origin string) (k6registry.Registry, error) {
+	registry, err := loadSource(in, loose)
+	if err != nil {
+		return nil, err
+	}
+
+	orig, err := loadOrigin(ctx, origin)
+	if err != nil {
+		return nil, err
+	}
 
 	for idx, ext := range registry {
 		slog.Debug("Process extension", "module", ext.Module)
+
+		if fromOrigin(&registry[idx], orig, origin) {
+			continue
+		}
+
 		if len(ext.Tier) == 0 {
 			registry[idx].Tier = k6registry.TierCommunity
 		}
@@ -87,7 +117,7 @@ func load(ctx context.Context, in io.Reader, loose bool, lint bool) (k6registry.
 			registry[idx].Versions = filterVersions(tags)
 		}
 
-		if lint && ext.Module != k6Module {
+		if lint && ext.Module != k6Module && ext.Compliance == nil && ext.Repo != nil {
 			compliance, err := checkCompliance(ctx, ext.Module, repo.CloneURL, repo.Timestamp)
 			if err != nil {
 				return nil, err

cmd/origin.go

@@ -0,0 +1,101 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/grafana/k6registry"
+)
+
+func loadOrigin(ctx context.Context, from string) (map[string]k6registry.Extension, error) {
+	dict := make(map[string]k6registry.Extension, 0)
+
+	if len(from) == 0 {
+		return dict, nil
+	}
+
+	client := &http.Client{Timeout: httpTimeout}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, from, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close() //nolint:errcheck
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var reg k6registry.Registry
+
+	err = json.Unmarshal(data, &reg)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, ext := range reg {
+		dict[ext.Module] = ext
+	}
+
+	return dict, nil
+}
+
+func fromOrigin(ext *k6registry.Extension, origin map[string]k6registry.Extension, loc string) bool {
+	oext, found := origin[ext.Module]
+	if !found {
+		return false
+	}
+
+	slog.Debug("Import extension", "module", ext.Module, "origin", loc)
+
+	if ext.Compliance == nil {
+		ext.Compliance = oext.Compliance
+	}
+
+	if ext.Repo == nil {
+		ext.Repo = oext.Repo
+	}
+
+	if len(ext.Versions) == 0 {
+		ext.Versions = oext.Versions
+	}
+
+	if len(ext.Description) == 0 {
+		ext.Description = oext.Description
+	}
+
+	if len(ext.Tier) == 0 {
+		ext.Tier = oext.Tier
+	}
+
+	if len(ext.Categories) == 0 {
+		ext.Categories = oext.Categories
+	}
+
+	if len(ext.Products) == 0 {
+		ext.Products = oext.Products
+	}
+
+	if len(ext.Imports) == 0 {
+		ext.Imports = oext.Imports
+	}
+
+	if len(ext.Outputs) == 0 {
+		ext.Outputs = oext.Outputs
+	}
+
+	return true
+}
+
+const httpTimeout = 10 * time.Second