diff --git a/cmd/render.go b/cmd/render.go index 38aa00fb..05fba697 100644 --- a/cmd/render.go +++ b/cmd/render.go @@ -34,7 +34,6 @@ func render(dest io.Writer, docName string) { ) return } - doc, found := result.Blocks.Documents[docName] if !found { diags.Add( @@ -47,29 +46,20 @@ func render(dest io.Writer, docName string) { ) return } - - // TODO: read pluginsDir from config #5 var pluginsDir string - if cliArgs.pluginsDir != "" { - pluginsDir = cliArgs.pluginsDir + if result.Blocks.GlobalConfig != nil && result.Blocks.GlobalConfig.PluginRegistry != nil { + pluginsDir = result.Blocks.GlobalConfig.PluginRegistry.MirrorDir + } + var pluginVersions runner.VersionMap + if result.Blocks.GlobalConfig != nil { + pluginVersions = result.Blocks.GlobalConfig.PluginVersions } runner, stdDiag := runner.Load( runner.WithBuiltIn( builtin.Plugin(version), ), runner.WithPluginDir(pluginsDir), - // TODO: get versions from the fabric configuration file. - // atm, it's hardcoded to use all plugins with the same version as the CLI. - runner.WithPluginVersions(runner.VersionMap{ - "blackstork/elasticsearch": version, - "blackstork/github": version, - "blackstork/graphql": version, - "blackstork/openai": version, - "blackstork/opencti": version, - "blackstork/postgresql": version, - "blackstork/sqlite": version, - "blackstork/terraform": version, - }), + runner.WithPluginVersions(runner.VersionMap(pluginVersions)), ) if diags.Extend(diagnostics.Diag(stdDiag)) { return diff --git a/cmd/root.go b/cmd/root.go index 1d70f3c6..f593cd61 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -86,11 +86,6 @@ var rootCmd = &cobra.Command{ } cliArgs.sourceDir = rawArgs.sourceDir - // TODO: make optional after #5 is implemented - err = validateDir("plugins dir", rawArgs.pluginsDir) - if err != nil { - return - } cliArgs.pluginsDir = rawArgs.pluginsDir cliArgs.colorize = rawArgs.colorize && term.IsTerminal(int(os.Stderr.Fd())) @@ -188,8 +183,4 @@ func init() { rootCmd.PersistentFlags().StringVar( &rawArgs.pluginsDir, "plugins-dir", "", "override for plugins dir from fabric configuration (required)", ) - err := rootCmd.MarkPersistentFlagRequired("plugins-dir") - if err != nil { - panic(err) - } } diff --git a/examples/templates/data.csv b/examples/templates/openai/data.csv similarity index 100% rename from examples/templates/data.csv rename to examples/templates/openai/data.csv diff --git a/examples/templates/example-openai.fabric b/examples/templates/openai/example.fabric similarity index 74% rename from examples/templates/example-openai.fabric rename to examples/templates/openai/example.fabric index 2e41b6eb..203e6ea9 100644 --- a/examples/templates/example-openai.fabric +++ b/examples/templates/openai/example.fabric @@ -1,10 +1,20 @@ +fabric { + cache_dir = "./.fabric" + plugin_registry { + mirror_dir = "dist/plugins" + } + plugin_versions = { + "blackstork/openai" = "0.0.0-dev" + } +} + config data csv {} -document "example-openai" { +document "example" { title = "Testing plugins" data csv "csv_file" { - path = "./data.csv" + path = "./examples/templates/openai/data.csv" } content text { text = "Values from the CSV file" @@ -38,10 +48,10 @@ document "example-openai" { } content openai_text { config { - api_key = "" + api_key = "" } query = ".data.csv.csv_file.result" model = "gpt-3.5-turbo" - prompt = "List names of these people" + prompt = "Decribe each user in a sentence" } } \ No newline at end of file diff --git a/go.mod b/go.mod index 0615d0ab..74862c58 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.0 toolchain go1.21.1 require ( + github.com/Masterminds/semver/v3 v3.2.1 github.com/elastic/go-elasticsearch/v8 v8.11.1 github.com/golang-cz/devslog v0.0.8 github.com/google/go-github/v58 v58.0.0 diff --git a/go.sum b/go.sum index 0ad2e760..7ac873e9 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= diff --git a/justfile b/justfile index 7cb4149b..75aecb14 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,7 @@ build: goreleaser build --config ./.goreleaser-dev.yaml --single-target --snapshot --clean test-run: - ./dist/fabric render "document.hello" --source-dir ./examples/templates/basic_hello/ -v --plugins-dir ./dist/plugins/ + ./dist/fabric render "document.hello" --source-dir ./examples/templates/basic_hello/ -v format: go mod tidy diff --git a/parser/definedBlocks.go b/parser/definedBlocks.go index 75bd86e9..b598d733 100644 --- a/parser/definedBlocks.go +++ b/parser/definedBlocks.go @@ -12,10 +12,11 @@ import ( // Collection of defined blocks type DefinedBlocks struct { - Config map[definitions.Key]*definitions.Config - Documents map[string]*definitions.Document - Sections map[string]*definitions.Section - Plugins map[definitions.Key]*definitions.Plugin + GlobalConfig *definitions.GlobalConfig + Config map[definitions.Key]*definitions.Config + Documents map[string]*definitions.Document + Sections map[string]*definitions.Section + Plugins map[definitions.Key]*definitions.Plugin } func (db *DefinedBlocks) GetSection(expr hcl.Expression) (section *definitions.Section, diags diagnostics.Diag) { @@ -78,6 +79,12 @@ func (db *DefinedBlocks) DefaultConfigFor(plugin *definitions.Plugin) (config *d } func (db *DefinedBlocks) Merge(other *DefinedBlocks) (diags diagnostics.Diag) { + if other.GlobalConfig != nil { + if db.GlobalConfig != nil { + diags.Add("Global config declared multiple times", "") + } + db.GlobalConfig = other.GlobalConfig + } for k, v := range other.Config { diags.Append(AddIfMissing(db.Config, k, v)) } diff --git a/parser/definitions/definitions.go b/parser/definitions/definitions.go index 2f552bba..18ab11e8 100644 --- a/parser/definitions/definitions.go +++ b/parser/definitions/definitions.go @@ -5,12 +5,13 @@ import ( ) const ( - BlockKindDocument = "document" - BlockKindConfig = "config" - BlockKindContent = "content" - BlockKindData = "data" - BlockKindMeta = "meta" - BlockKindSection = "section" + BlockKindDocument = "document" + BlockKindConfig = "config" + BlockKindContent = "content" + BlockKindData = "data" + BlockKindMeta = "meta" + BlockKindSection = "section" + BlockKindGlobalConfig = "fabric" PluginTypeRef = "ref" AttrRefBase = "base" diff --git a/parser/definitions/global_config.go b/parser/definitions/global_config.go new file mode 100644 index 00000000..8e95ddc2 --- /dev/null +++ b/parser/definitions/global_config.go @@ -0,0 +1,105 @@ +package definitions + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" + + "github.com/blackstork-io/fabric/pkg/diagnostics" +) + +var globalConfigSpec = &hcldec.ObjectSpec{ + "cache_dir": &hcldec.AttrSpec{ + Name: "cache_dir", + Type: cty.String, + Required: false, + }, + "plugin_registry": &hcldec.BlockSpec{ + TypeName: "plugin_registry", + Nested: hcldec.ObjectSpec{ + "mirror_dir": &hcldec.AttrSpec{ + Name: "mirror_dir", + Type: cty.String, + Required: false, + }, + }, + }, + "plugin_versions": &hcldec.AttrSpec{ + Name: "plugin_versions", + Type: cty.Map(cty.String), + Required: false, + }, +} + +type GlobalConfig struct { + block *hclsyntax.Block + CacheDir string + PluginRegistry *PluginRegistry + PluginVersions map[string]string +} + +type PluginRegistry struct { + MirrorDir string +} + +func DefineGlobalConfig(block *hclsyntax.Block) (cfg *GlobalConfig, diags diagnostics.Diag) { + if len(block.Labels) > 0 { + return nil, diagnostics.Diag{{ + Severity: hcl.DiagError, + Summary: "Invalid global config", + Detail: "Global config should not have labels", + }} + } + value, hclDiags := hcldec.Decode(block.Body, globalConfigSpec, nil) + if diags.ExtendHcl(hclDiags) { + return + } + typ := hcldec.ImpliedType(globalConfigSpec) + errs := value.Type().TestConformance(typ) + if len(errs) > 0 { + var err error + value, err = convert.Convert(value, typ) + if err != nil { + diags.AppendErr(err, "Error while serializing global config") + return + } + } + cfg = &GlobalConfig{ + block: block, + CacheDir: "./.fabric", + PluginVersions: make(map[string]string), + } + cacheDir := value.GetAttr("cache_dir") + if !cacheDir.IsNull() || cacheDir.AsString() != "" { + cfg.CacheDir = cacheDir.AsString() + } + pluginRegistry := value.GetAttr("plugin_registry") + if !pluginRegistry.IsNull() { + mirrorDir := pluginRegistry.GetAttr("mirror_dir") + if !mirrorDir.IsNull() || mirrorDir.AsString() != "" { + cfg.PluginRegistry = &PluginRegistry{ + MirrorDir: mirrorDir.AsString(), + } + } + } + pluginVersions := value.GetAttr("plugin_versions") + if !pluginVersions.IsNull() { + versionMap := pluginVersions.AsValueMap() + for k, v := range versionMap { + if v.Type() != cty.String { + diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid plugin version", + Detail: fmt.Sprintf("Version of plugin '%s' should be a string", k), + }) + continue + } + cfg.PluginVersions[k] = v.AsString() + } + } + return cfg, nil +} diff --git a/parser/parser.go b/parser/parser.go index c0f1948d..334a8819 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -211,6 +211,15 @@ func parseBlockDefinitions(body *hclsyntax.Body) (res *DefinedBlocks, diags diag panic("unable to get the key of the top-level block") } diags.Append(AddIfMissing(res.Config, *key, cfg)) + case definitions.BlockKindGlobalConfig: + globalCfg, dgs := definitions.DefineGlobalConfig(block) + if diags.Extend(dgs) { + continue + } + if res.GlobalConfig != nil { + diags.Add("Global config declared multiple times", "") + } + res.GlobalConfig = globalCfg default: diags.Append(definitions.NewNestingDiag( "Top level of fabric document", @@ -222,6 +231,7 @@ func parseBlockDefinitions(body *hclsyntax.Body) (res *DefinedBlocks, diags diag definitions.BlockKindDocument, definitions.BlockKindSection, definitions.BlockKindConfig, + definitions.BlockKindGlobalConfig, })) } } diff --git a/plugin/runner/loader.go b/plugin/runner/loader.go index b6417b70..0a5429a4 100644 --- a/plugin/runner/loader.go +++ b/plugin/runner/loader.go @@ -3,7 +3,6 @@ package runner import ( "fmt" "os" - "path" "github.com/hashicorp/hcl/v2" @@ -27,7 +26,7 @@ type loadedContentProvider struct { } type loader struct { - pluginDir string + resolver *resolver versionMap VersionMap builtin []*plugin.Schema pluginMap map[string]loadedPlugin @@ -35,9 +34,9 @@ type loader struct { contentMap map[string]loadedContentProvider } -func makeLoader(pluginDir string, builtin []*plugin.Schema, pluginMap VersionMap) *loader { +func makeLoader(mirrorDir string, builtin []*plugin.Schema, pluginMap VersionMap) *loader { return &loader{ - pluginDir: pluginDir, + resolver: makeResolver(mirrorDir), versionMap: pluginMap, builtin: builtin, pluginMap: make(map[string]loadedPlugin), @@ -46,10 +45,6 @@ func makeLoader(pluginDir string, builtin []*plugin.Schema, pluginMap VersionMap } } -func (l *loader) pluginLoc(name, version string) string { - return path.Join(l.pluginDir, fmt.Sprintf("%s@%s", name, version)) -} - func nopCloser() error { return nil } @@ -147,7 +142,10 @@ func (l *loader) registerPlugin(p *plugin.Schema, closefn func() error) hcl.Diag } func (l *loader) loadBinary(name, version string) hcl.Diagnostics { - loc := l.pluginLoc(name, version) + loc, diags := l.resolver.resolve(name, version) + if diags.HasErrors() { + return diags + } if info, err := os.Stat(loc); os.IsNotExist(err) { return hcl.Diagnostics{{ Severity: hcl.DiagError, diff --git a/plugin/runner/resolver.go b/plugin/runner/resolver.go new file mode 100644 index 00000000..ccf9294d --- /dev/null +++ b/plugin/runner/resolver.go @@ -0,0 +1,96 @@ +package runner + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/hashicorp/hcl/v2" +) + +type resolver struct { + mirrorDir string +} + +func makeResolver(mirrorDir string) *resolver { + return &resolver{ + mirrorDir: mirrorDir, + } +} + +func (r *resolver) resolve(name, version string) (loc string, diags hcl.Diagnostics) { + nameSpace, pluginName, err := r.parseName(name) + if err != nil { + return "", hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Invalid plugin name", + Detail: fmt.Sprintf("Invalid plugin name '%s': %s", name, err), + }} + } + constraint, err := semver.NewConstraint(version) + if err != nil { + return "", hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to resolve plugin", + Detail: fmt.Sprintf("Invalid version constraint: %s", err), + }} + } + entry, err := os.ReadDir(filepath.Join(r.mirrorDir, nameSpace)) + if err != nil { + return "", hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to resolve plugin", + Detail: fmt.Sprintf("Failed to read directory for namespace '%s': %s", nameSpace, err), + }} + } + matched := make(map[string]*semver.Version) + for _, e := range entry { + if e.IsDir() { + continue + } + parts := strings.SplitN(e.Name(), "@", 2) + if len(parts) != 2 || parts[0] != pluginName { + continue + } + v, err := semver.NewVersion(parts[1]) + if err != nil { + continue + } + if !constraint.Check(v) { + continue + } + matched[parts[1]] = v + } + if len(matched) == 0 { + return "", hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Failed to resolve plugin binary", + Detail: fmt.Sprintf("No plugin matches version constraint for %s@%s", name, version), + }} + } + // find latest version that matches version constraint + var latestVerStr string + var latestVer *semver.Version + for str, ver := range matched { + if latestVer == nil { + latestVerStr = str + latestVer = ver + continue + } + if ver.Compare(latestVer) > 0 { + latestVerStr = str + latestVer = ver + } + } + return filepath.Join(r.mirrorDir, nameSpace, fmt.Sprintf("%s@%s", pluginName, latestVerStr)), nil +} + +func (r *resolver) parseName(name string) (string, string, error) { + parts := strings.SplitN(name, "/", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("plugin name '%s' is not in the form '/'", name) + } + return parts[0], parts[1], nil +} diff --git a/plugin/runner/runner.go b/plugin/runner/runner.go index bd27f267..18b35b79 100644 --- a/plugin/runner/runner.go +++ b/plugin/runner/runner.go @@ -2,7 +2,10 @@ package runner import ( "fmt" + "strings" + "unicode" + "github.com/Masterminds/semver/v3" "github.com/hashicorp/hcl/v2" "github.com/blackstork-io/fabric/plugin" @@ -17,11 +20,67 @@ type Runner struct { contentMap map[string]loadedContentProvider } +func validatePluginName(name string) hcl.Diagnostics { + parts := strings.Split(name, "/") + if len(parts) != 2 { + return hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Invalid plugin name", + Detail: fmt.Sprintf("plugin name '%s' is not in the form '/'", name), + }} + } + for _, r := range parts[0] { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' { + return hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Invalid plugin name", + Detail: fmt.Sprintf("plugin name '%s' contains invalid character: '%c'", name, r), + }} + } + } + for _, r := range parts[1] { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' { + return hcl.Diagnostics{{ + Severity: hcl.DiagError, + Summary: "Invalid plugin name", + Detail: fmt.Sprintf("plugin name '%s' contains invalid character: '%c'", name, r), + }} + } + } + return nil +} + +func validatePluginVersionMap(versionMap VersionMap) (diags hcl.Diagnostics) { + for name, version := range versionMap { + diags = validatePluginName(name).Extend(diags) + if version == "" { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing plugin version", + Detail: fmt.Sprintf("Missing plugin version for '%s'", name), + }) + continue + } + _, err := semver.NewConstraint(version) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid plugin version", + Detail: fmt.Sprintf("Invalid version constraint for '%s': %s", name, err), + }) + } + } + return diags +} + func Load(o ...Option) (*Runner, hcl.Diagnostics) { opts := defaultOptions for _, opt := range o { opt(&opts) } + if diags := validatePluginVersionMap(opts.versionMap); diags.HasErrors() { + return nil, diags + } loader := makeLoader(opts.pluginDir, opts.builtin, opts.versionMap) if diags := loader.loadAll(); diags.HasErrors() { return nil, diags