feat: Notion Publisher

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2
+	github.com/brittonhayes/notionmd v0.6.1
+	github.com/dstotijn/go-notion v0.11.0
 	github.com/elastic/go-elasticsearch/v8 v8.14.0
 	github.com/evanphx/go-hclog-slog v0.0.0-20240717231540-be48fc4c4df5
 	github.com/gobwas/glob v0.2.3
@@ -90,6 +92,7 @@ require (
 	github.com/go-swiss/fonts v0.0.0-20221219152310-0b267088f53d // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect

go.sum

@@ -42,6 +42,8 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq
 github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
 github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
 github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
+github.com/brittonhayes/notionmd v0.6.1 h1:yCeL1PLb5Zoksnq8BrNB08BEtmbIpHgzG5LprlF3lVI=
+github.com/brittonhayes/notionmd v0.6.1/go.mod h1:r6V8qENKn8hyDEDZsUJFAnY9+fdyEqPA57vwIvEeMpA=
 github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA=
 github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8=
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
@@ -74,6 +76,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dstotijn/go-notion v0.11.0 h1:v+ZUiyKd+UBk1SRkUSa86QOU5DP8ziSI4E7NFIS4rRU=
+github.com/dstotijn/go-notion v0.11.0/go.mod h1:FWfmGRnE8Drm6CnNQQO7slXcu1lrKmRY2KfFgeq6Z2g=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
 github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
 github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow=
@@ -106,6 +110,8 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9 h1:TRYrIWJziqvMVn1owO8bmkDJTlMQFYnf74yhD8LXfgU=
+github.com/gomarkdown/markdown v0.0.0-20240723152757-afa4a469d4f9/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

internal/builtin/content_helpers.go

@@ -22,6 +22,11 @@ func countDeclarations(data *plugin.ContentSection, name string) int {
 	return count
 }
 
+func ParseScope(datactx plugindata.Map) (document, section *plugin.ContentSection) {
+	return parseScope(datactx)
+}
+
+// TODO:(britton) change signature of ParseScope to be exported
 func parseScope(datactx plugindata.Map) (document, section *plugin.ContentSection) {
 	documentMap, ok := datactx["document"]
 	if !ok {

internal/notion/data_markdown.go

@@ -0,0 +1,148 @@
+package notion
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty"
+
+	"github.com/blackstork-io/fabric/pkg/diagnostics"
+	"github.com/blackstork-io/fabric/plugin"
+	"github.com/blackstork-io/fabric/plugin/dataspec"
+	"github.com/blackstork-io/fabric/plugin/plugindata"
+)
+
+// TODO:(britton) markdown data source should come from shared local file source
+func makeMarkdownDataSource() *plugin.DataSource {
+	return &plugin.DataSource{
+		DataFunc: fetchMarkdownData,
+		Args: &dataspec.RootSpec{
+			Attrs: []*dataspec.AttrSpec{
+				{
+					Name:       "glob",
+					Type:       cty.String,
+					ExampleVal: cty.StringVal("path/to/file*.md"),
+					Doc:        `A glob pattern to select MD files to read`,
+				},
+				{
+					Name:       "path",
+					Type:       cty.String,
+					ExampleVal: cty.StringVal("path/to/file.md"),
+					Doc:        `A file path to a MD file to read`,
+				},
+			},
+		},
+		Doc: `
+		Loads markdown files with the names that match a provided ` + "`glob`" + ` pattern or a single file from a provided path.
+
+		Either ` + "`glob`" + ` or ` + "`path`" + ` argument must be set.
+
+		When ` + "`path`" + ` argument is specified, the data source returns only the content of a file.
+		When ` + "`glob`" + ` argument is specified, the data source returns a list of dicts that contain the content of a file and file's metadata. For example:
+		` + "```json" + `
+		[
+		  {
+			"file_path": "path/file-a.md",
+			"file_name": "file-a.md",
+			"content": "foobar"
+		  },
+		  {
+			"file_path": "path/file-b.md",
+			"file_name": "file-b.md",
+			"content": "x\\ny\\nz"
+		  }
+		]
+		` + "```",
+	}
+}
+
+func readMarkdownFile(path string) (plugindata.Data, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, diagnostics.Diag{{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to open a file",
+			Detail:   err.Error(),
+		}}
+	}
+	defer f.Close()
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return nil, diagnostics.Diag{{
+			Severity: hcl.DiagError,
+			Summary:  "Failed to read a file",
+			Detail:   err.Error(),
+		}}
+	}
+	return plugindata.String(string(data)), nil
+}
+
+func readMarkdownFiles(_ context.Context, pattern string) (plugindata.Data, error) {
+	paths, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(plugindata.List, 0, len(paths))
+	for _, path := range paths {
+		fileData, err := readMarkdownFile(path)
+		if err != nil {
+			return result, err
+		}
+		result = append(result, plugindata.Map{
+			"file_path": plugindata.String(path),
+			"file_name": plugindata.String(filepath.Base(path)),
+			"content":   fileData,
+		})
+	}
+	return result, nil
+}
+
+func fetchMarkdownData(ctx context.Context, params *plugin.RetrieveDataParams) (plugindata.Data, diagnostics.Diag) {
+	glob := params.Args.GetAttrVal("glob")
+	path := params.Args.GetAttrVal("path")
+
+	if !path.IsNull() && path.AsString() != "" {
+		slog.Debug("Reading a file from the path", "path", path.AsString())
+		data, err := readMarkdownFile(path.AsString())
+		if err != nil {
+			slog.Error(
+				"Error while reading a file",
+				slog.String("path", path.AsString()),
+				slog.Any("error", err),
+			)
+			return nil, diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to read a file",
+				Detail:   err.Error(),
+			}}
+		}
+		return data, nil
+	} else if !glob.IsNull() && glob.AsString() != "" {
+		slog.Debug("Reading the files that match the glob pattern", "glob", glob.AsString())
+		data, err := readMarkdownFiles(ctx, glob.AsString())
+		if err != nil {
+			slog.Error(
+				"Error while reading the files",
+				slog.String("glob", glob.AsString()),
+				slog.Any("error", err),
+			)
+			return nil, diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to read the files",
+				Detail:   err.Error(),
+			}}
+		}
+		return data, nil
+	}
+	slog.Error("Either \"glob\" value or \"path\" value must be provided")
+	return nil, diagnostics.Diag{{
+		Severity: hcl.DiagError,
+		Summary:  "Failed to parse provided arguments",
+		Detail:   "Either \"glob\" value or \"path\" value must be provided",
+	}}
+}

internal/notion/plugin.go

@@ -0,0 +1,21 @@
+package notion
+
+import (
+	"log/slog"
+
+	"github.com/blackstork-io/fabric/plugin"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func Plugin(version string, logger *slog.Logger, tracer trace.Tracer) *plugin.Schema {
+	return &plugin.Schema{
+		Name:    "blackstork/notion",
+		Version: version,
+		DataSources: plugin.DataSources{
+			"md": makeMarkdownDataSource(),
+		},
+		Publishers: plugin.Publishers{
+			"notion_page": makeNotionPagePublisher(logger, tracer),
+		},
+	}
+}

internal/notion/publish_notion_page.go

@@ -0,0 +1,156 @@
+package notion
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"log/slog"
+
+	"github.com/blackstork-io/fabric/internal/builtin"
+	"github.com/blackstork-io/fabric/pkg/diagnostics"
+	"github.com/blackstork-io/fabric/plugin"
+	"github.com/blackstork-io/fabric/plugin/dataspec"
+	"github.com/blackstork-io/fabric/plugin/dataspec/constraint"
+	"github.com/blackstork-io/fabric/plugin/plugindata"
+	"github.com/blackstork-io/fabric/print/mdprint"
+	"github.com/brittonhayes/notionmd"
+	"github.com/dstotijn/go-notion"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/zclconf/go-cty/cty"
+	"go.opentelemetry.io/otel/trace"
+	nooptrace "go.opentelemetry.io/otel/trace/noop"
+)
+
+func makeNotionPagePublisher(logger *slog.Logger, tracer trace.Tracer) *plugin.Publisher {
+	if logger == nil {
+		logger = slog.New(slog.NewTextHandler(io.Discard, nil))
+	}
+	if tracer == nil {
+		tracer = nooptrace.Tracer{}
+	}
+
+	return &plugin.Publisher{
+		Doc:  "Publishes content to a Notion page",
+		Tags: []string{},
+		Args: &dataspec.RootSpec{
+			Attrs: []*dataspec.AttrSpec{
+				{
+					Name:        "title",
+					Doc:         "Title of the Notion page",
+					Type:        cty.String,
+					ExampleVal:  cty.StringVal("My Notion Page"),
+					Constraints: constraint.Required,
+				},
+				{
+					Name:        "parent_page_id",
+					Doc:         "Notion parent page ID",
+					Type:        cty.String,
+					ExampleVal:  cty.StringVal("1234567890"),
+					Constraints: constraint.Required,
+				},
+				{
+					Name:        "api_key",
+					Doc:         "Notion API key",
+					Type:        cty.String,
+					ExampleVal:  cty.StringVal("secret_1234567890"),
+					Constraints: constraint.Required,
+					Secret:      true,
+				},
+			},
+		},
+		AllowedFormats: []plugin.OutputFormat{plugin.OutputFormatMD},
+		PublishFunc:    publishNotionPage(logger, tracer),
+	}
+}
+
+func publishNotionPage(logger *slog.Logger, _ trace.Tracer) plugin.PublishFunc {
+	return func(ctx context.Context, params *plugin.PublishParams) diagnostics.Diag {
+		document, _ := builtin.ParseScope(params.DataContext)
+		if document == nil {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse document",
+				Detail:   "document is required",
+			}}
+		}
+
+		datactx := params.DataContext
+		datactx["format"] = plugindata.String(params.Format.String())
+
+		titleAttr := params.Args.GetAttrVal("title")
+		if titleAttr.IsNull() || titleAttr.AsString() == "" {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse arguments",
+				Detail:   "title is required",
+			}}
+		}
+
+		parentPageIDAttr := params.Args.GetAttrVal("parent_page_id")
+		if parentPageIDAttr.IsNull() || parentPageIDAttr.AsString() == "" {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse arguments",
+				Detail:   "parent_page_id is required",
+			}}
+		}
+
+		apiKeyAttr := params.Args.GetAttrVal("api_key")
+		if apiKeyAttr.IsNull() || apiKeyAttr.AsString() == "" {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to parse arguments",
+				Detail:   "api_key is required",
+			}}
+		}
+
+		writer := bytes.NewBuffer([]byte{})
+
+		printer := mdprint.New()
+		err := printer.Print(ctx, writer, document)
+		if err != nil {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to print content",
+				Detail:   err.Error(),
+			}}
+		}
+
+		blocks, err := notionmd.Convert(writer.String())
+		if err != nil {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to convert content to Notion blocks",
+				Detail:   err.Error(),
+			}}
+		}
+
+		// Publish to Notion
+		logger.InfoContext(ctx, "Publishing to Notion", "title", titleAttr.AsString())
+		client := notion.NewClient(apiKeyAttr.AsString())
+		page, err := client.CreatePage(ctx, notion.CreatePageParams{
+			ParentType: notion.ParentTypePage,
+			ParentID:   parentPageIDAttr.AsString(),
+			Title: []notion.RichText{
+				{
+					Type: notion.RichTextTypeText,
+					Text: &notion.Text{
+						Content: titleAttr.AsString(),
+					},
+				},
+			},
+			Children: blocks,
+		})
+		if err != nil {
+			return diagnostics.Diag{{
+				Severity: hcl.DiagError,
+				Summary:  "Failed to create a Notion page",
+				Detail:   err.Error(),
+			}}
+		}
+
+		logger.InfoContext(ctx, "Published to Notion", "page_id", page.ID)
+
+		return nil
+	}
+}

internal/plugin_validity_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/blackstork-io/fabric/internal/hackerone"
 	"github.com/blackstork-io/fabric/internal/microsoft"
 	"github.com/blackstork-io/fabric/internal/nistnvd"
+	"github.com/blackstork-io/fabric/internal/notion"
 	"github.com/blackstork-io/fabric/internal/openai"
 	"github.com/blackstork-io/fabric/internal/opencti"
 	"github.com/blackstork-io/fabric/internal/postgresql"
@@ -46,6 +47,7 @@ func TestAllPluginSchemaValidity(t *testing.T) {
 		nistnvd.Plugin(ver, nil),
 		snyk.Plugin(ver, nil),
 		microsoft.Plugin(ver, nil, nil),
+		notion.Plugin(ver, nil, nil),
 	}
 	for _, p := range plugins {
 		p := p