Implement global fabric config (#85)

cmd/render.go

@@ -34,7 +34,6 @@ func render(dest io.Writer, docName string) {
 		)
 		return
 	}
-
 	doc, found := result.Blocks.Documents[docName]
 	if !found {
 		diags.Add(
@@ -47,29 +46,20 @@ func render(dest io.Writer, docName string) {
 		)
 		return
 	}
-
-	// TODO: read pluginsDir from config #5
 	var pluginsDir string
-	if cliArgs.pluginsDir != "" {
-		pluginsDir = cliArgs.pluginsDir
+	if result.Blocks.GlobalConfig != nil && result.Blocks.GlobalConfig.PluginRegistry != nil {
+		pluginsDir = result.Blocks.GlobalConfig.PluginRegistry.MirrorDir
+	}
+	var pluginVersions runner.VersionMap
+	if result.Blocks.GlobalConfig != nil {
+		pluginVersions = result.Blocks.GlobalConfig.PluginVersions
 	}
 	runner, stdDiag := runner.Load(
 		runner.WithBuiltIn(
 			builtin.Plugin(version),
 		),
 		runner.WithPluginDir(pluginsDir),
-		// TODO: get versions from the fabric configuration file.
-		// atm, it's hardcoded to use all plugins with the same version as the CLI.
-		runner.WithPluginVersions(runner.VersionMap{
-			"blackstork/elasticsearch": version,
-			"blackstork/github":        version,
-			"blackstork/graphql":       version,
-			"blackstork/openai":        version,
-			"blackstork/opencti":       version,
-			"blackstork/postgresql":    version,
-			"blackstork/sqlite":        version,
-			"blackstork/terraform":     version,
-		}),
+		runner.WithPluginVersions(runner.VersionMap(pluginVersions)),
 	)
 	if diags.Extend(diagnostics.Diag(stdDiag)) {
 		return

cmd/root.go

@@ -86,11 +86,6 @@ var rootCmd = &cobra.Command{
 		}
 		cliArgs.sourceDir = rawArgs.sourceDir
 
-		// TODO: make optional after #5 is implemented
-		err = validateDir("plugins dir", rawArgs.pluginsDir)
-		if err != nil {
-			return
-		}
 		cliArgs.pluginsDir = rawArgs.pluginsDir
 
 		cliArgs.colorize = rawArgs.colorize && term.IsTerminal(int(os.Stderr.Fd()))
@@ -188,8 +183,4 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(
 		&rawArgs.pluginsDir, "plugins-dir", "", "override for plugins dir from fabric configuration (required)",
 	)
-	err := rootCmd.MarkPersistentFlagRequired("plugins-dir")
-	if err != nil {
-		panic(err)
-	}
 }

examples/templates/example-openai.fabric → examples/templates/openai/example.fabric

@@ -1,10 +1,20 @@
+fabric {
+    cache_dir = "./.fabric"
+    plugin_registry {
+        mirror_dir = "dist/plugins"
+    }
+    plugin_versions = {
+        "blackstork/openai" = "0.0.0-dev"
+    }
+}
+
 config data csv {}
 
-document "example-openai" {
+document "example" {
     title = "Testing plugins"
 
     data csv "csv_file" {
-        path = "./data.csv"
+        path = "./examples/templates/openai/data.csv"
     }
     content text {
         text = "Values from the CSV file"
@@ -38,10 +48,10 @@ document "example-openai" {
     }
     content openai_text {
         config {
-            api_key = "<API_KEY>"
+            api_key = "<API-KEY>"
         }
         query = ".data.csv.csv_file.result"
         model = "gpt-3.5-turbo"
-        prompt = "List names of these people"
+        prompt = "Decribe each user in a sentence"
     }
 }

go.mod

@@ -5,6 +5,7 @@ go 1.21.0
 toolchain go1.21.1
 
 require (
+	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/elastic/go-elasticsearch/v8 v8.11.1
 	github.com/golang-cz/devslog v0.0.8
 	github.com/google/go-github/v58 v58.0.0

go.sum

@@ -5,6 +5,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=

justfile

@@ -4,7 +4,7 @@ build:
     goreleaser build --config ./.goreleaser-dev.yaml --single-target --snapshot --clean
 
 test-run:
-    ./dist/fabric render "document.hello" --source-dir ./examples/templates/basic_hello/ -v --plugins-dir ./dist/plugins/
+    ./dist/fabric render "document.hello" --source-dir ./examples/templates/basic_hello/ -v
 
 format:
     go mod tidy

parser/definedBlocks.go

@@ -12,10 +12,11 @@ import (
 // Collection of defined blocks
 
 type DefinedBlocks struct {
-	Config    map[definitions.Key]*definitions.Config
-	Documents map[string]*definitions.Document
-	Sections  map[string]*definitions.Section
-	Plugins   map[definitions.Key]*definitions.Plugin
+	GlobalConfig *definitions.GlobalConfig
+	Config       map[definitions.Key]*definitions.Config
+	Documents    map[string]*definitions.Document
+	Sections     map[string]*definitions.Section
+	Plugins      map[definitions.Key]*definitions.Plugin
 }
 
 func (db *DefinedBlocks) GetSection(expr hcl.Expression) (section *definitions.Section, diags diagnostics.Diag) {
@@ -78,6 +79,12 @@ func (db *DefinedBlocks) DefaultConfigFor(plugin *definitions.Plugin) (config *d
 }
 
 func (db *DefinedBlocks) Merge(other *DefinedBlocks) (diags diagnostics.Diag) {
+	if other.GlobalConfig != nil {
+		if db.GlobalConfig != nil {
+			diags.Add("Global config declared multiple times", "")
+		}
+		db.GlobalConfig = other.GlobalConfig
+	}
 	for k, v := range other.Config {
 		diags.Append(AddIfMissing(db.Config, k, v))
 	}

parser/definitions/definitions.go

@@ -5,12 +5,13 @@ import (
 )
 
 const (
-	BlockKindDocument = "document"
-	BlockKindConfig   = "config"
-	BlockKindContent  = "content"
-	BlockKindData     = "data"
-	BlockKindMeta     = "meta"
-	BlockKindSection  = "section"
+	BlockKindDocument     = "document"
+	BlockKindConfig       = "config"
+	BlockKindContent      = "content"
+	BlockKindData         = "data"
+	BlockKindMeta         = "meta"
+	BlockKindSection      = "section"
+	BlockKindGlobalConfig = "fabric"
 
 	PluginTypeRef = "ref"
 	AttrRefBase   = "base"

parser/definitions/global_config.go

@@ -0,0 +1,105 @@
+package definitions
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/convert"
+
+	"github.com/blackstork-io/fabric/pkg/diagnostics"
+)
+
+var globalConfigSpec = &hcldec.ObjectSpec{
+	"cache_dir": &hcldec.AttrSpec{
+		Name:     "cache_dir",
+		Type:     cty.String,
+		Required: false,
+	},
+	"plugin_registry": &hcldec.BlockSpec{
+		TypeName: "plugin_registry",
+		Nested: hcldec.ObjectSpec{
+			"mirror_dir": &hcldec.AttrSpec{
+				Name:     "mirror_dir",
+				Type:     cty.String,
+				Required: false,
+			},
+		},
+	},
+	"plugin_versions": &hcldec.AttrSpec{
+		Name:     "plugin_versions",
+		Type:     cty.Map(cty.String),
+		Required: false,
+	},
+}
+
+type GlobalConfig struct {
+	block          *hclsyntax.Block
+	CacheDir       string
+	PluginRegistry *PluginRegistry
+	PluginVersions map[string]string
+}
+
+type PluginRegistry struct {
+	MirrorDir string
+}
+
+func DefineGlobalConfig(block *hclsyntax.Block) (cfg *GlobalConfig, diags diagnostics.Diag) {
+	if len(block.Labels) > 0 {
+		return nil, diagnostics.Diag{{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid global config",
+			Detail:   "Global config should not have labels",
+		}}
+	}
+	value, hclDiags := hcldec.Decode(block.Body, globalConfigSpec, nil)
+	if diags.ExtendHcl(hclDiags) {
+		return
+	}
+	typ := hcldec.ImpliedType(globalConfigSpec)
+	errs := value.Type().TestConformance(typ)
+	if len(errs) > 0 {
+		var err error
+		value, err = convert.Convert(value, typ)
+		if err != nil {
+			diags.AppendErr(err, "Error while serializing global config")
+			return
+		}
+	}
+	cfg = &GlobalConfig{
+		block:          block,
+		CacheDir:       "./.fabric",
+		PluginVersions: make(map[string]string),
+	}
+	cacheDir := value.GetAttr("cache_dir")
+	if !cacheDir.IsNull() || cacheDir.AsString() != "" {
+		cfg.CacheDir = cacheDir.AsString()
+	}
+	pluginRegistry := value.GetAttr("plugin_registry")
+	if !pluginRegistry.IsNull() {
+		mirrorDir := pluginRegistry.GetAttr("mirror_dir")
+		if !mirrorDir.IsNull() || mirrorDir.AsString() != "" {
+			cfg.PluginRegistry = &PluginRegistry{
+				MirrorDir: mirrorDir.AsString(),
+			}
+		}
+	}
+	pluginVersions := value.GetAttr("plugin_versions")
+	if !pluginVersions.IsNull() {
+		versionMap := pluginVersions.AsValueMap()
+		for k, v := range versionMap {
+			if v.Type() != cty.String {
+				diags.Append(&hcl.Diagnostic{
+					Severity: hcl.DiagError,
+					Summary:  "Invalid plugin version",
+					Detail:   fmt.Sprintf("Version of plugin '%s' should be a string", k),
+				})
+				continue
+			}
+			cfg.PluginVersions[k] = v.AsString()
+		}
+	}
+	return cfg, nil
+}

parser/parser.go

@@ -211,6 +211,15 @@ func parseBlockDefinitions(body *hclsyntax.Body) (res *DefinedBlocks, diags diag
 				panic("unable to get the key of the top-level block")
 			}
 			diags.Append(AddIfMissing(res.Config, *key, cfg))
+		case definitions.BlockKindGlobalConfig:
+			globalCfg, dgs := definitions.DefineGlobalConfig(block)
+			if diags.Extend(dgs) {
+				continue
+			}
+			if res.GlobalConfig != nil {
+				diags.Add("Global config declared multiple times", "")
+			}
+			res.GlobalConfig = globalCfg
 		default:
 			diags.Append(definitions.NewNestingDiag(
 				"Top level of fabric document",
@@ -222,6 +231,7 @@ func parseBlockDefinitions(body *hclsyntax.Body) (res *DefinedBlocks, diags diag
 					definitions.BlockKindDocument,
 					definitions.BlockKindSection,
 					definitions.BlockKindConfig,
+					definitions.BlockKindGlobalConfig,
 				}))
 		}
 	}

plugin/runner/loader.go

@@ -3,7 +3,6 @@ package runner
 import (
 	"fmt"
 	"os"
-	"path"
 
 	"github.com/hashicorp/hcl/v2"
 
@@ -27,17 +26,17 @@ type loadedContentProvider struct {
 }
 
 type loader struct {
-	pluginDir  string
+	resolver   *resolver
 	versionMap VersionMap
 	builtin    []*plugin.Schema
 	pluginMap  map[string]loadedPlugin
 	dataMap    map[string]loadedDataSource
 	contentMap map[string]loadedContentProvider
 }
 
-func makeLoader(pluginDir string, builtin []*plugin.Schema, pluginMap VersionMap) *loader {
+func makeLoader(mirrorDir string, builtin []*plugin.Schema, pluginMap VersionMap) *loader {
 	return &loader{
-		pluginDir:  pluginDir,
+		resolver:   makeResolver(mirrorDir),
 		versionMap: pluginMap,
 		builtin:    builtin,
 		pluginMap:  make(map[string]loadedPlugin),
@@ -46,10 +45,6 @@ func makeLoader(pluginDir string, builtin []*plugin.Schema, pluginMap VersionMap
 	}
 }
 
-func (l *loader) pluginLoc(name, version string) string {
-	return path.Join(l.pluginDir, fmt.Sprintf("%s@%s", name, version))
-}
-
 func nopCloser() error {
 	return nil
 }
@@ -147,7 +142,10 @@ func (l *loader) registerPlugin(p *plugin.Schema, closefn func() error) hcl.Diag
 }
 
 func (l *loader) loadBinary(name, version string) hcl.Diagnostics {
-	loc := l.pluginLoc(name, version)
+	loc, diags := l.resolver.resolve(name, version)
+	if diags.HasErrors() {
+		return diags
+	}
 	if info, err := os.Stat(loc); os.IsNotExist(err) {
 		return hcl.Diagnostics{{
 			Severity: hcl.DiagError,