Data block updates (#88)

.golangci.yaml

@@ -11,7 +11,15 @@ run:
 
   # default is true. Enables skipping of directories:
   #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs-use-default: true
+  skip-dirs-use-default: false
+
+  skip-dirs:
+    - (^|/)vendor($|/)
+    - (^|/)third_party($|/)
+    - (^|/)testdata($|/)
+    - (^|/)examples($|/)
+    - (^|/)Godeps($|/)
+    # - (^|/)builtin($|/) # disable standard builtin regexp, it was hiding builtin plugins
 
 
 # output configuration options
@@ -35,7 +43,6 @@ output:
   # sorts results by: filepath, line and column
   sort-results: false
 
-
 linters-settings:
   dupl:
     threshold: 100
@@ -52,7 +59,7 @@ linters-settings:
     min-occurrences: 2
   staticcheck:
     go: "1.19"
-    checks: [ "all", "-SA1019"]
+    checks: ["all", "-SA1019"]
   gocritic:
     enabled-tags:
       - diagnostic
@@ -79,7 +86,6 @@ linters-settings:
   nolintlint:
     require-explanation: false # don't require an explanation for nolint directives
     require-specific: false # don't require nolint directives to be specific about which linter is being skipped
-
 
 linters:
   disable-all: true

cmd/render.go

@@ -43,7 +43,7 @@ func Render(pluginsDir string, sourceDir fs.FS, docName string) (results []strin
 	}
 
 	if pluginsDir == "" && blocks.GlobalConfig != nil && blocks.GlobalConfig.PluginRegistry != nil {
-		// use pluginsDir from config, unless overriden by cli arg
+		// use pluginsDir from config, unless overridden by cli arg
 		pluginsDir = blocks.GlobalConfig.PluginRegistry.MirrorDir
 	}
 
@@ -57,7 +57,7 @@ func Render(pluginsDir string, sourceDir fs.FS, docName string) (results []strin
 			builtin.Plugin(version),
 		),
 		runner.WithPluginDir(pluginsDir),
-		runner.WithPluginVersions(runner.VersionMap(pluginVersions)),
+		runner.WithPluginVersions(pluginVersions),
 	)
 	if diags.ExtendHcl(stdDiag) {
 		return
@@ -127,7 +127,10 @@ var renderCmd = &cobra.Command{
 		}
 		diagnostics.PrintDiags(os.Stderr, diags, fileMap, cliArgs.colorize)
 		if diags.HasErrors() {
-			return fmt.Errorf("failed to render the document")
+			// Errors have been already displayed
+			rootCmd.SilenceErrors = true
+			rootCmd.SilenceUsage = true
+			return diags
 		}
 		return nil
 	},

cmd/root.go

@@ -20,7 +20,7 @@ import (
 
 const devVersion = "v0.0.0-dev"
 
-// Overriden by goreleaser
+// Overridden by goreleaser.
 var version = devVersion
 
 type logLevels struct {
@@ -179,7 +179,6 @@ func init() {
 	)
 	rootCmd.PersistentFlags().BoolVar(&rawArgs.colorize, "color", true, "enables colorizing the logs and diagnostics (if supported by the terminal and log format)")
 	rootCmd.PersistentFlags().BoolVarP(&rawArgs.verbose, "verbose", "v", false, "a shortcut to --log-level debug")
-	// TODO: after #5 is implemented - make optional
 	rootCmd.PersistentFlags().StringVar(
 		&rawArgs.pluginsDir, "plugins-dir", "", "override for plugins dir from fabric configuration (required)",
 	)

examples/plugins/basic/data_random_numbers.go

@@ -79,7 +79,7 @@ func fetchRandomNumbers(ctx context.Context, params *plugin.RetrieveDataParams)
 
 	data := make(plugin.ListData, lengthInt)
 	for i := int64(0); i < lengthInt; i++ {
-		n := rand.Int63() % (maxInt - minInt + 1)
+		n := rand.Int63() % (maxInt - minInt + 1) //nolint:G404 // weak rng is ok here
 		data[i] = plugin.NumberData(n + minInt)
 	}
 	return data, nil

internal/builtin/content_table.go

@@ -67,7 +67,7 @@ func parseTableContentArgs(params *plugin.ProvideContentParams) (headers, values
 		var (
 			header cty.Value
 			value  cty.Value
-			ok     = false
+			ok     bool
 		)
 		if header, ok = obj["header"]; !ok || header.IsNull() {
 			return nil, nil, fmt.Errorf("missing header in table cell")

internal/builtin/data_csv.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"io"
 	"os"
+	"unicode/utf8"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hcldec"
@@ -36,6 +37,28 @@ func makeCSVDataSource() *plugin.DataSource {
 	}
 }
 
+func getDelim(config cty.Value) (r rune, diags hcl.Diagnostics) {
+	r = defaultCSVDelimiter
+	if config.IsNull() {
+		return
+	}
+	delim := config.GetAttr("delimiter")
+	if delim.IsNull() {
+		return
+	}
+	delimStr := delim.AsString()
+	delimRune, runeLen := utf8.DecodeRuneInString(delimStr)
+	if runeLen == 0 || len(delimStr) != runeLen {
+		diags = hcl.Diagnostics{{
+			Severity: hcl.DiagError,
+			Summary:  "delimiter must be a single character",
+		}}
+		return
+	}
+	r = delimRune
+	return
+}
+
 func fetchCSVData(ctx context.Context, params *plugin.RetrieveDataParams) (plugin.Data, hcl.Diagnostics) {
 	path := params.Args.GetAttr("path")
 	if path.IsNull() || path.AsString() == "" {
@@ -44,21 +67,11 @@ func fetchCSVData(ctx context.Context, params *plugin.RetrieveDataParams) (plugi
 			Summary:  "path is required",
 		}}
 	}
-	delim := cty.StringVal(string(defaultCSVDelimiter))
-	if !params.Config.IsNull() {
-		delim = params.Config.GetAttr("delimiter")
-		if delim.IsNull() {
-			delim = cty.StringVal(string(defaultCSVDelimiter))
-		}
-	}
-	if len(delim.AsString()) != 1 {
-		return nil, hcl.Diagnostics{{
-			Severity: hcl.DiagError,
-			Summary:  "delimiter must be a single character",
-		}}
+	delim, diags := getDelim(params.Config)
+	if diags != nil {
+		return nil, diags
 	}
-	delimRune := []rune(delim.AsString())[0]
-	data, err := readCSVFile(ctx, path.AsString(), delimRune)
+	data, err := readCSVFile(ctx, path.AsString(), delim)
 	if err != nil {
 		return nil, hcl.Diagnostics{{
 			Severity: hcl.DiagError,

internal/openai/client/client.go

@@ -60,6 +60,7 @@ func (c *client) GenerateChatCompletion(ctx context.Context, params *ChatComplet
 	if err != nil {
 		return nil, err
 	}
+	defer res.Body.Close()
 	if res.StatusCode != http.StatusOK {
 		var errRes ErrorResponse
 		if err := json.NewDecoder(res.Body).Decode(&errRes); err != nil {

parser/caller.go

@@ -17,9 +17,6 @@ import (
 	"github.com/blackstork-io/fabric/plugin/runner"
 )
 
-// Stub implementation of plugin caller
-// TODO: attach to plugin discovery mechanism
-
 type pluginData struct {
 	ConfigSpec     hcldec.Spec
 	InvocationSpec hcldec.Spec
@@ -149,11 +146,8 @@ func (c *Caller) callPlugin(kind, name string, config evaluation.Configuration,
 			Config: configVal,
 			Args:   pluginArgs,
 		})
-		result.Result = map[string]any{}
 		if data != nil {
-			result.Result = map[string]any{
-				"result": data.Any(),
-			}
+			result.Result = data.Any()
 		}
 		result.Diags = diags
 	case "content":

parser/evaluation/blockInvocation.go

@@ -30,8 +30,6 @@ func (b *BlockInvocation) DefRange() hcl.Range {
 }
 
 func hclBodyToVal(body *hclsyntax.Body) (val cty.Value, diags diagnostics.Diag) {
-	// TODO: this makes a full dump of all of the attributes, not abiding by hidden
-	// Think about ways to fix, or whether fix is needed
 	obj := make(map[string]cty.Value, len(body.Attributes)+len(body.Blocks))
 	for name, attr := range body.Attributes {
 		attrVal, diag := attr.Expr.Value(nil)

parser/mockPluginCaller.go

@@ -9,6 +9,7 @@ import (
 	"github.com/blackstork-io/fabric/parser/definitions"
 	"github.com/blackstork-io/fabric/parser/evaluation"
 	"github.com/blackstork-io/fabric/pkg/diagnostics"
+	"github.com/blackstork-io/fabric/pkg/utils"
 )
 
 type MockCaller struct{}
@@ -18,32 +19,27 @@ func (c *MockCaller) dumpContext(context map[string]any) string {
 }
 
 func (c *MockCaller) dumpConfig(config evaluation.Configuration) string {
+	if utils.IsNil(config) {
+		return "NoConfig"
+	}
 	switch c := config.(type) {
 	case *definitions.ConfigPtr:
-		if c == nil {
-			return "NoConfig"
-		}
 		attrs, _ := c.Cfg.Body.JustAttributes()
 		return litter.Sdump("ConfigPtr", maps.Keys(attrs))
 	case *definitions.Config:
-		if c == nil {
-			return "NoConfig"
-		}
 		attrs, _ := c.Block.Body.JustAttributes()
 		return litter.Sdump("Config", maps.Keys(attrs))
-	case nil:
-		return "NoConfig"
 	default:
 		return "UnknownConfig " + litter.Sdump(c)
 	}
 }
 
 func (c *MockCaller) dumpInvocation(invoke evaluation.Invocation) string {
+	if utils.IsNil(invoke) {
+		return "NoConfig"
+	}
 	switch inv := invoke.(type) {
 	case *evaluation.BlockInvocation:
-		if inv == nil {
-			return "NoConfig"
-		}
 		attrStringed := map[string]string{}
 		attrs, _ := inv.Body.JustAttributes()
 		for k, v := range attrs {
@@ -53,13 +49,8 @@ func (c *MockCaller) dumpInvocation(invoke evaluation.Invocation) string {
 
 		return litter.Sdump("BlockInvocation", attrStringed)
 	case *definitions.TitleInvocation:
-		if inv == nil {
-			return "NoInvocation"
-		}
 		val, _ := inv.Expression.Value(nil)
 		return litter.Sdump("TitleInvocation", val.GoString())
-	case nil:
-		return "NoInvocation"
 	default:
 		return "UnknownInvocation " + litter.Sdump(inv)
 	}

parser/parsePluginBlock.go

@@ -50,7 +50,6 @@ func (db *DefinedBlocks) ParsePlugin(plugin *definitions.Plugin) (res *definitio
 }
 
 func (db *DefinedBlocks) parsePlugin(plugin *definitions.Plugin) (parsed *definitions.ParsedPlugin, diags diagnostics.Diag) {
-	// var res evaluation.Plugin
 	res := definitions.ParsedPlugin{
 		PluginName: plugin.Name(),
 		BlockName:  plugin.BlockName(),
@@ -133,8 +132,14 @@ func (db *DefinedBlocks) parsePlugin(plugin *definitions.Plugin) (parsed *defini
 				diags.Append(&hcl.Diagnostic{
 					Severity: hcl.DiagWarning,
 					Summary:  "Potential data conflict",
-					Detail:   fmt.Sprintf("This 'data ref' will inherit its name from its base (\"%s\"). Creating another anonymous 'data ref' with the same 'base' would override the current block's execution results. We recommend naming all 'data ref' blocks uniquely", res.BlockName),
-					Subject:  plugin.DefRange().Ptr(),
+					Detail: fmt.Sprintf(
+						"This 'data ref' will inherit its name from its base (%q). "+
+							"Creating another anonymous 'data ref' with the same 'base' would "+
+							"override the current block's execution results. "+
+							"We recommend namingall 'data ref' blocks uniquely",
+						res.BlockName,
+					),
+					Subject: plugin.DefRange().Ptr(),
 				})
 			}
 		}

parser/parseSectionBlock.go

@@ -126,48 +126,49 @@ func (db *DefinedBlocks) parseSection(section *definitions.Section) (parsed *def
 				continue
 			}
 			circularRefDetector.Add(section, block.DefRange().Ptr())
-			//nolint:gocritic
-			defer circularRefDetector.Remove(section, &diags)
 			parsedSubSection, diag := db.ParseSection(subSection)
+			circularRefDetector.Remove(section, &diag)
 			if diags.Extend(diag) {
 				continue
 			}
 			res.Content = append(res.Content, parsedSubSection)
 		}
 	}
 
-	if refBase != nil {
-		baseSection, diag := Resolve[*definitions.Section](db, refBase)
-		if diags.Extend(diag) {
-			return
-		}
-		circularRefDetector.Add(section, refBase.Range().Ptr())
-		defer circularRefDetector.Remove(section, &diags)
-		if circularRefDetector.Check(baseSection) {
-			diags.Append(&hcl.Diagnostic{
-				Severity: hcl.DiagError,
-				Summary:  "Circular reference detected",
-				Detail:   "Looped back to this block through reference chain:",
-				Subject:  section.Block.DefRange().Ptr(),
-				Extra:    circularRefDetector.ExtraMarker,
-			})
-			return
-		}
-		baseEval, diag := db.ParseSection(baseSection)
-		if diags.Extend(diag) {
-			return
-		}
+	if refBase == nil {
+		parsed = &res
+		return
+	}
+	// Parse ref
+	baseSection, diag := Resolve[*definitions.Section](db, refBase)
+	if diags.Extend(diag) {
+		return
+	}
+	circularRefDetector.Add(section, refBase.Range().Ptr())
+	defer circularRefDetector.Remove(section, &diags)
+	if circularRefDetector.Check(baseSection) {
+		diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Circular reference detected",
+			Detail:   "Looped back to this block through reference chain:",
+			Subject:  section.Block.DefRange().Ptr(),
+			Extra:    circularRefDetector.ExtraMarker,
+		})
+		return
+	}
+	baseEval, diag := db.ParseSection(baseSection)
+	if diags.Extend(diag) {
+		return
+	}
 
-		// update from base:
-		if res.Title == nil {
-			res.Title = baseEval.Title
-		}
-		if res.Meta == nil {
-			res.Meta = baseEval.Meta
-		}
-		// TODO: make an suggestion-issue: allow adding blocks in refs, order-preserving base attr
-		res.Content = append(res.Content, baseEval.Content...)
+	// update from base:
+	if res.Title == nil {
+		res.Title = baseEval.Title
+	}
+	if res.Meta == nil {
+		res.Meta = baseEval.Meta
 	}
+	res.Content = append(res.Content, baseEval.Content...)
 
 	parsed = &res
 	return