internal/elasticsearch: bug fixes (#104)

Author: dobarx

docs/plugins/elasticsearch.md

@@ -32,7 +32,7 @@ The data source supports the following configuration parameters:
 config data elasticsearch {
     api_key = <list of string>  # optional
     api_key_str = <string>  # optional
-    base_url = <string>  # required
+    base_url = <string>  # optional
     basic_auth_password = <string>  # optional
     basic_auth_username = <string>  # optional
     bearer_auth = <string>  # optional
@@ -52,5 +52,6 @@ data elasticsearch {
     index = <string>  # required
     query = <map of dynamic>  # optional
     query_string = <string>  # optional
+    size = <number>  # optional
 }
 ```

go.mod

@@ -19,7 +19,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-sqlite3 v1.14.20
 	github.com/pelletier/go-toml/v2 v2.1.1
-	github.com/sanity-io/litter v1.5.5
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -37,7 +37,6 @@ github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -151,16 +150,13 @@ github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOS
 github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo=
-github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U=
 github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
 github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
@@ -181,7 +177,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

internal/builtin/content_frontmatter.go

@@ -27,7 +27,7 @@ var frontMatterAllowedFormats = []string{"yaml", "toml", "json"}
 func makeFrontMatterContentProvider() *plugin.ContentProvider {
 	return &plugin.ContentProvider{
 		ContentFunc: genFrontMatterContent,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"format": &hcldec.AttrSpec{
 				Name:     "format",
 				Type:     cty.String,

internal/builtin/content_image.go

@@ -15,7 +15,7 @@ import (
 func makeImageContentProvider() *plugin.ContentProvider {
 	return &plugin.ContentProvider{
 		ContentFunc: genImageContent,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"src": &hcldec.AttrSpec{
 				Name:     "src",
 				Type:     cty.String,

internal/builtin/content_list.go

@@ -26,7 +26,7 @@ var listAllowedFormats = []string{"unordered", "ordered", "tasklist"}
 func makeListContentProvider() *plugin.ContentProvider {
 	return &plugin.ContentProvider{
 		ContentFunc: genListContent,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"item_template": &hcldec.AttrSpec{
 				Name:     "item_template",
 				Type:     cty.String,

internal/builtin/content_table.go

@@ -19,7 +19,7 @@ type tableCellTmpl = *template.Template
 func makeTableContentProvider() *plugin.ContentProvider {
 	return &plugin.ContentProvider{
 		ContentFunc: genTableContent,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"columns": &hcldec.AttrSpec{
 				Name: "columns",
 				Type: cty.List(cty.Object(map[string]cty.Type{

internal/builtin/content_text.go

@@ -28,7 +28,7 @@ const (
 func makeTextContentProvider() *plugin.ContentProvider {
 	return &plugin.ContentProvider{
 		ContentFunc: genTextContent,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"text": &hcldec.AttrSpec{
 				Name:     "text",
 				Type:     cty.String,

internal/builtin/data_csv.go

@@ -20,14 +20,14 @@ const defaultCSVDelimiter = ','
 func makeCSVDataSource() *plugin.DataSource {
 	return &plugin.DataSource{
 		DataFunc: fetchCSVData,
-		Config: &hcldec.ObjectSpec{
+		Config: hcldec.ObjectSpec{
 			"delimiter": &hcldec.AttrSpec{
 				Name:     "delimiter",
 				Type:     cty.String,
 				Required: false,
 			},
 		},
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"path": &hcldec.AttrSpec{
 				Name:     "path",
 				Type:     cty.String,

internal/builtin/data_json.go

@@ -17,7 +17,7 @@ import (
 func makeJSONDataSource() *plugin.DataSource {
 	return &plugin.DataSource{
 		DataFunc: fetchJSONData,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"glob": &hcldec.AttrSpec{
 				Name:     "glob",
 				Type:     cty.String,

internal/builtin/data_txt.go

@@ -15,7 +15,7 @@ import (
 func makeTXTDataSource() *plugin.DataSource {
 	return &plugin.DataSource{
 		DataFunc: fetchTXTData,
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"path": &hcldec.AttrSpec{
 				Name:     "path",
 				Type:     cty.String,

internal/elasticsearch/data_elasticsearch.go

@@ -17,7 +17,7 @@ func makeElasticSearchDataSource() *plugin.DataSource {
 			"base_url": &hcldec.AttrSpec{
 				Name:     "base_url",
 				Type:     cty.String,
-				Required: true,
+				Required: false,
 			},
 			"cloud_id": &hcldec.AttrSpec{
 				Name:     "cloud_id",
@@ -55,14 +55,14 @@ func makeElasticSearchDataSource() *plugin.DataSource {
 				Required: false,
 			},
 		},
-		Args: &hcldec.ObjectSpec{
+		Args: hcldec.ObjectSpec{
 			"index": &hcldec.AttrSpec{
 				Name:     "index",
 				Type:     cty.String,
 				Required: true,
 			},
 			"id": &hcldec.AttrSpec{
-				Name:     "index",
+				Name:     "id",
 				Type:     cty.String,
 				Required: false,
 			},
@@ -81,6 +81,11 @@ func makeElasticSearchDataSource() *plugin.DataSource {
 				Type:     cty.List(cty.String),
 				Required: false,
 			},
+			"size": &hcldec.AttrSpec{
+				Name:     "size",
+				Type:     cty.Number,
+				Required: false,
+			},
 		},
 	}
 }

internal/elasticsearch/data_elasticsearch_test.go

@@ -18,8 +18,8 @@ import (
 	"github.com/blackstork-io/fabric/plugin"
 )
 
-// ElasticSearchDataTestSuite is a test suite to test integration with real elasticsearch
-type ElasticSearchDataTestSuite struct {
+// IntegrationTestSuite is a test suite to test integration with real elasticsearch
+type IntegrationTestSuite struct {
 	suite.Suite
 	container *elasticsearch.ElasticsearchContainer
 	client    *es.Client
@@ -28,14 +28,14 @@ type ElasticSearchDataTestSuite struct {
 	ctx       context.Context
 }
 
-func TestElasticSearchDataSuite(t *testing.T) {
+func TestIntegrationSuite(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration tests")
 	}
-	suite.Run(t, &ElasticSearchDataTestSuite{})
+	suite.Run(t, &IntegrationTestSuite{})
 }
 
-func (s *ElasticSearchDataTestSuite) SetupSuite() {
+func (s *IntegrationTestSuite) SetupSuite() {
 	s.ctx = context.Background()
 	opts := []testcontainers.ContainerCustomizer{
 		testcontainers.WithImage("docker.io/elasticsearch:8.9.0"),
@@ -68,7 +68,7 @@ func (s *ElasticSearchDataTestSuite) SetupSuite() {
 	s.plugin = Plugin("")
 }
 
-func (s *ElasticSearchDataTestSuite) TearDownSuite() {
+func (s *IntegrationTestSuite) TearDownSuite() {
 	s.Require().NoError(s.container.Terminate(s.ctx), "failed to stop elasticsearch container")
 }
 
@@ -80,7 +80,7 @@ type testDataObject struct {
 	Name   string `json:"name"`
 }
 
-func (s *ElasticSearchDataTestSuite) SetupTest() {
+func (s *IntegrationTestSuite) SetupTest() {
 	file, err := os.ReadFile("testdata/data.json")
 	s.Require().NoError(err, "failed to read data.json")
 	dataList := []testDataObject{}
@@ -104,19 +104,20 @@ func (s *ElasticSearchDataTestSuite) SetupTest() {
 	s.Require().False(res.IsError(), "failed to refresh indices")
 }
 
-func (s *ElasticSearchDataTestSuite) TearDownTest() {
+func (s *IntegrationTestSuite) TearDownTest() {
 	res, err := s.client.Indices.Delete([]string{"test_index"})
 	s.Require().NoError(err, "failed to delete indices")
 	s.Require().False(res.IsError(), "failed to delete indices: %s", res.String())
 }
 
-func (s *ElasticSearchDataTestSuite) TestSearchDefaults() {
+func (s *IntegrationTestSuite) TestSearchDefaults() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.NullVal(cty.String),
 		"index":        cty.StringVal("test_index"),
 		"query":        cty.NullVal(cty.DynamicPseudoType),
 		"query_string": cty.NullVal(cty.String),
 		"fields":       cty.NullVal(cty.String),
+		"size":         cty.NullVal(cty.Number),
 	})
 	data, diags := s.plugin.RetrieveData(s.ctx, "elasticsearch", &plugin.RetrieveDataParams{
 		Config: s.cfg,
@@ -173,13 +174,14 @@ func (s *ElasticSearchDataTestSuite) TestSearchDefaults() {
 	  }`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestSearchFields() {
+func (s *IntegrationTestSuite) TestSearchFields() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.NullVal(cty.String),
 		"index":        cty.StringVal("test_index"),
 		"query":        cty.NullVal(cty.DynamicPseudoType),
 		"query_string": cty.NullVal(cty.String),
 		"fields":       cty.ListVal([]cty.Value{cty.StringVal("name"), cty.StringVal("age")}),
+		"size":         cty.NullVal(cty.Number),
 	})
 	data, diags := s.plugin.RetrieveData(s.ctx, "elasticsearch", &plugin.RetrieveDataParams{
 		Config: s.cfg,
@@ -227,13 +229,14 @@ func (s *ElasticSearchDataTestSuite) TestSearchFields() {
 	  }`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestSearchQueryString() {
+func (s *IntegrationTestSuite) TestSearchQueryString() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.NullVal(cty.String),
 		"index":        cty.StringVal("test_index"),
 		"query":        cty.NullVal(cty.DynamicPseudoType),
 		"query_string": cty.StringVal("type:foo"),
 		"fields":       cty.NullVal(cty.String),
+		"size":         cty.NullVal(cty.Number),
 	})
 	data, diags := s.plugin.RetrieveData(s.ctx, "elasticsearch", &plugin.RetrieveDataParams{
 		Config: s.cfg,
@@ -278,7 +281,7 @@ func (s *ElasticSearchDataTestSuite) TestSearchQueryString() {
 	  }`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestSearchQuery() {
+func (s *IntegrationTestSuite) TestSearchQuery() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":    cty.NullVal(cty.String),
 		"index": cty.StringVal("test_index"),
@@ -287,6 +290,7 @@ func (s *ElasticSearchDataTestSuite) TestSearchQuery() {
 		}),
 		"query_string": cty.NullVal(cty.String),
 		"fields":       cty.NullVal(cty.String),
+		"size":         cty.NullVal(cty.Number),
 	})
 	data, diags := s.plugin.RetrieveData(s.ctx, "elasticsearch", &plugin.RetrieveDataParams{
 		Config: s.cfg,
@@ -343,7 +347,47 @@ func (s *ElasticSearchDataTestSuite) TestSearchQuery() {
 	}`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestGetByID() {
+func (s *IntegrationTestSuite) TestSearchSize() {
+	args := cty.ObjectVal(map[string]cty.Value{
+		"id":           cty.NullVal(cty.String),
+		"index":        cty.StringVal("test_index"),
+		"query":        cty.NullVal(cty.DynamicPseudoType),
+		"query_string": cty.NullVal(cty.String),
+		"fields":       cty.NullVal(cty.String),
+		"size":         cty.NumberIntVal(1),
+	})
+	data, diags := s.plugin.RetrieveData(s.ctx, "elasticsearch", &plugin.RetrieveDataParams{
+		Config: s.cfg,
+		Args:   args,
+	})
+	s.Require().Nil(diags)
+	m := data.(plugin.MapData)
+	raw, err := json.MarshalIndent(m["hits"], "", "  ")
+	s.Require().NoError(err, "failed to marshal data: %v", err)
+	s.JSONEq(`{
+		"hits": [
+		  {
+			"_id": "54f7a815-eac5-4f7c-a339-5fefd0f54967",
+			"_index": "test_index",
+			"_score": 1,
+			"_source": {
+			  "active": false,
+			  "age": 39,
+			  "id": "54f7a815-eac5-4f7c-a339-5fefd0f54967",
+			  "name": "Davidson",
+			  "type": "foo"
+			}
+		  }
+		],
+		"max_score": 1,
+		"total": {
+		  "relation": "eq",
+		  "value": 3
+		}
+	  }`, string(raw))
+}
+
+func (s *IntegrationTestSuite) TestGetByID() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.StringVal("0c68e63d-daaa-4a62-92e6-e855bd144fb6"),
 		"index":        cty.StringVal("test_index"),
@@ -376,7 +420,7 @@ func (s *ElasticSearchDataTestSuite) TestGetByID() {
 	}`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestGetByIDFields() {
+func (s *IntegrationTestSuite) TestGetByIDFields() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.StringVal("0c68e63d-daaa-4a62-92e6-e855bd144fb6"),
 		"index":        cty.StringVal("test_index"),
@@ -406,7 +450,7 @@ func (s *ElasticSearchDataTestSuite) TestGetByIDFields() {
 	}`, string(raw))
 }
 
-func (s *ElasticSearchDataTestSuite) TestGetByIDNotFound() {
+func (s *IntegrationTestSuite) TestGetByIDNotFound() {
 	args := cty.ObjectVal(map[string]cty.Value{
 		"id":           cty.StringVal("00000000-0000-0000-0000-000000000000"),
 		"index":        cty.StringVal("test_index"),

internal/elasticsearch/plugin.go

@@ -34,12 +34,18 @@ func makeClient(pcfg cty.Value) (*es.Client, error) {
 		Addresses: []string{defaultBaseURL},
 		Username:  defaultUsername,
 	}
+	if pcfg.IsNull() {
+		return nil, fmt.Errorf("configuration is required")
+	}
 	if baseURL := pcfg.GetAttr("base_url"); !baseURL.IsNull() {
 		cfg.Addresses = []string{baseURL.AsString()}
 	}
 	if cloudID := pcfg.GetAttr("cloud_id"); !cloudID.IsNull() {
 		cfg.CloudID = cloudID.AsString()
 	}
+	if len(cfg.Addresses) == 0 && cfg.CloudID == "" {
+		return nil, fmt.Errorf("either one of base_url or cloud_id is required")
+	}
 	if apiKeyStr := pcfg.GetAttr("api_key_str"); !apiKeyStr.IsNull() {
 		cfg.APIKey = apiKeyStr.AsString()
 	}
@@ -123,6 +129,13 @@ func search(fn esapi.Search, args cty.Value) (plugin.Data, error) {
 		}
 		opts = append(opts, fn.WithBody(bytes.NewReader(queryRaw)))
 	}
+	if size := args.GetAttr("size"); !size.IsNull() {
+		n, _ := size.AsBigFloat().Int64()
+		if n <= 0 {
+			return nil, fmt.Errorf("size must be greater than 0")
+		}
+		opts = append(opts, fn.WithSize(int(n)))
+	}
 	if fields := args.GetAttr("fields"); !fields.IsNull() {
 		fieldSlice := fields.AsValueSlice()
 		fieldStrings := make([]string, len(fieldSlice))