update comment

README.md

@@ -13,7 +13,6 @@ go install github.com/mimuret/dtap/v2@latest
 ## Configuration
 
 - [JSON schema](./schemas/schema.json)
-- TBD
 
 ## design
 

doc/README.md

@@ -0,0 +1,44 @@
+# Configuration
+
+
+## Configuration file
+
+- [see godoc](https://pkg.go.dev/github.com/mimuret/dtap/v2@v2.0.0-alpha.18/pkg/config#Config)
+
+
+## filter_plugin
+
+- [iphash](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/iphash#IPHash)
+- [label](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/label#Label)
+- [mask](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/mask#Mask)
+- [matcher](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/matcher#MatcherConfig)
+- [metrics](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/metrics#Metrics)
+
+### for debug
+
+- [nop](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/nop#Nop)
+- [static](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/filter/static#Static)
+
+## input plugin
+
+- [file](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/input/file#File)
+- [nats](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/input/nats#Nats)
+- [tcp](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/input/tcp#TCPSocket)
+- [unix](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/input/unix#UnixSocket)
+
+## output plugins
+
+- [file](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/file#Output)
+- [nats](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/nats#Nats)
+- [stdout](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/stdout#Stdout)
+- [tcp](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/tcp#TCP)
+- [unix](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/unix#Unix)
+
+### debug and experimental plugins
+
+- [dns](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/dns#DNS)
+- [fluent](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/fluent#Fluent)
+- [kafka](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/kafka#Kafka)
+- [loki](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/loki#Loki)
+- [loki](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/loki#Loki)
+- [nop](https://pkg.go.dev/github.com/mimuret/dtap/v2/pkg/plugin/output/nop#NOP)

pkg/config/config.go

@@ -35,7 +35,9 @@ const DefaultOutputBufferSize = 10000
 const DefaultInputFilterWorkerNum = 1
 
 type BufferConfig struct {
+	// Buffer name
 	Name string
+	// Buffer size
 	Size uint
 }
 
@@ -47,22 +49,58 @@ func (c *BufferConfig) GetSize() uint {
 	return c.Size
 }
 
+// An output group consists of a buffer (queue) for the output group,
+// Filter settings specific to the output group, and output plug-in settings.
+// When a message passes through the global filter and is written to the
+// output group's buffer, the global workers runs a filter for the output group.
+// Only messages that pass through it are written to the output group's Buffer.
+// Each output plugin runs in its own goroutine, receiving and processing
+// messages from the output buffer (queue).
+// Since output plug-ins are executed in parallel, a message is not processed
+// by all output plug-ins, but by one of them. If you want all plug-ins to process
+// a message at the same time, for example, standard output and file output, please separate the output groups.
+// The current use case for setting up multiple output plugins is to increase
+// throughput by setting up multiple plugins with the same configuration, for example, for forwarding to remote.
 type OutputGroupConfig struct {
-	Name         string
+	// Output group name
+	Name string
+	// Output group shared queue config
 	BufferConfig *BufferConfig
-	Filters      plugin.FilterPlugins
-	Outputs      plugin.OutputPlugins
+	// Filter settings for groups.
+	// After filtering, it is added to the output buffer.
+	Filters plugin.FilterPlugins
+	// Output plugin settings.
+	// A message is processed only by one of the plugins.
+	Outputs plugin.OutputPlugins
 }
 
+// Configuration file structure.
+// The configuration file must be in yaml or json format.
+// The file extension must be 'yaml' or 'yml' or 'json'.
 type Config struct {
-	// number of workers that moves msg from input buffer to output buffer with msg filtering.
+	// Number of global workers that process messages.
+	// Global workers input messages from global buffers,
+	// filter messages with global filters, and copy messages
+	// with filter words to output group buffers.
+	// default value is 1
 	InputFilterWorkerNum uint
-	LogLevel             string
-	MetricsListen        string
-	InputBufferConfig    *BufferConfig
-	Inputs               plugin.InputPlugins
-	Filters              plugin.FilterPlugins
-	OutputGroups         []OutputGroupConfig
+	// Output log level.
+	// Select one of 'debug', 'info', 'warn', 'error', or 'fatal'.
+	// default value is 'info'
+	LogLevel string
+	// Listen IP and port to output metrics
+	MetricsListen string
+	// Input buffer settings
+	InputBufferConfig *BufferConfig
+	// Input plugin settings. Must not be empty.
+	Inputs plugin.InputPlugins
+	// The global filters are the filter that is processed for all messages.
+	// It can be empty.
+	Filters plugin.FilterPlugins
+	// Output group settings. Must not be empty.
+	// If multiple output groups are specified, messages that pass through
+	// the Global Filter are copied to all output groups.
+	OutputGroups []OutputGroupConfig
 }
 
 func LoadConfig(fs afero.Fs, cfgFile string) (*Config, error) {

pkg/plugin/filter/iphash/setup.go

@@ -33,10 +33,15 @@ func Setup(raw json.RawMessage) (types.FilterPlugin, error) {
 
 var _ types.FilterPlugin = &IPHash{}
 
+// The IPHash plugin creates QueryAddressHash and ResponseAddressHash labels from query and response addresses.
+
 type IPHash struct {
 	plugin.PluginCommon
-	Salt                   string
-	QueryAddressEnabled    bool
+	// salt for creating hash
+	Salt string
+	// If QueryAddressEnabled is true, create a QueryAddressHash label. Default is true.
+	QueryAddressEnabled bool
+	// If ResponseAddressEnabled is true, create a ResponseAddressHash label. Default is true.
 	ResponseAddressEnabled bool
 }
 

pkg/plugin/filter/label/setup.go

@@ -47,8 +47,15 @@ const (
 )
 
 type AddLabel struct {
-	Name   string
-	Type   AddLabelType
+	// Label key name for adding or updating. It must not be empty.
+	Name string
+	// Type can be one of `DNSTAP`, `DNS`, or `STATIC`.
+	Type AddLabelType
+	// If `Type` is `DNSTAP`, the function name of DNSTAP of the getter function is set to value, and the value of DNSTAP message can be written in the label.
+	// You can see getter names https://pkg.go.dev/github.com/mimuret/dnsutils/getter#DnstapGetterName.
+	// If `Type` is `DNS`, the function name of the DNS message of the getter function is set to value, and the value of the DNS message can be written into the label.
+	// You can see getter names https://pkg.go.dev/github.com/mimuret/dnsutils/getter#DnsMsgGetterName.
+	// If the Type is `STATIC`, the string of the value is written directly into the label.
 	Value  string
 	getter types.DnstapMessageGetFunc
 }
@@ -86,7 +93,9 @@ func (l *AddLabel) Setup() error {
 	return nil
 }
 
+// Delete Label
 type DelLabel struct {
+	// Label key name for deleting. It must not be empty.
 	Name string
 }
 
@@ -97,9 +106,12 @@ func (l *DelLabel) Validate() error {
 	return nil
 }
 
+// The label plugin edits labels.
 type Label struct {
 	plugin.PluginCommon
+	// Adding and updating labels
 	Add []*AddLabel
+	// Deleting labels
 	Del []*DelLabel
 }
 

pkg/plugin/filter/mask/setup.go

@@ -40,11 +40,16 @@ func Setup(raw json.RawMessage) (types.FilterPlugin, error) {
 
 var _ types.FilterPlugin = &Mask{}
 
+// Mask plug-ins mask IP addresses.
 type Mask struct {
 	plugin.PluginCommon
-	MaskLen4               uint8
-	MaskLen6               uint8
-	QueryAddressEnabled    bool
+	// PrefixLength value to mask ipv4 address. default is 22.
+	MaskLen4 uint8
+	// PrefixLength value to mask ipv6 address. default is 40.
+	MaskLen6 uint8
+	// If QueryAddressEnabled is true, mask QueryAddress. defaut is true
+	QueryAddressEnabled bool
+	// If ResponseAddressEnabled is true, mask ResponseAddress. defaut is true
 	ResponseAddressEnabled bool
 	mask4                  net.IPMask
 	mask6                  net.IPMask

pkg/plugin/filter/matcher/export_test.go

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022 Manabu Sonoda
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package matcher
+
+import (
+	umatcher "github.com/mimuret/dnsutils/matcher"
+)
+
+func UpdateSet(f *Matcher, set *umatcher.MatcherSet) *Matcher {
+	f.set = set
+	return f
+}
+func GetSet(f *Matcher) *umatcher.MatcherSet {
+	return f.set
+}

pkg/plugin/filter/matcher/setup.go

@@ -30,10 +30,7 @@ func init() {
 
 func setup(data json.RawMessage) (types.FilterPlugin, error) {
 	g := &Matcher{}
-	t := struct {
-		plugin.PluginCommon
-		Rule umatcher.Config `json:"Rule"`
-	}{}
+	t := MatcherConfig{}
 	if err := json.Unmarshal(data, &t); err != nil {
 		return nil, err
 	}
@@ -42,23 +39,30 @@ func setup(data json.RawMessage) (types.FilterPlugin, error) {
 		return nil, err
 	}
 	g.PluginCommon = t.PluginCommon
-	g.Set = set
+	g.set = set
 
 	return g, nil
 }
 
 var _ types.FilterPlugin = &Matcher{}
 
-type Matcher struct {
+// The Matcher plugin filters DNSTAP messages using the Matcher function.
+type MatcherConfig struct {
 	plugin.PluginCommon
+	// Rule is match settings
+	// see https://pkg.go.dev/github.com/mimuret/dnsutils/matcher#Config
+	Rule umatcher.Config `json:"Rule"`
+}
 
-	Set *umatcher.MatcherSet `json:"-"`
+type Matcher struct {
+	plugin.PluginCommon
+	set *umatcher.MatcherSet `json:"-"`
 }
 
 func (f *Matcher) Filter(dt *types.DnstapMessage) *types.DnstapMessage {
 	dnstap := dt.GetDnstap()
 	dnsmsg := dt.GetMessage()
-	if f.Set.Match(dnstap, dnsmsg) {
+	if f.set.Match(dnstap, dnsmsg) {
 		return dt
 	}
 	return nil

pkg/plugin/filter/matcher/setup_test.go

@@ -59,19 +59,19 @@ var _ = Describe("Static", func() {
 				staticDnsFalse, _ := matcher.NewMatchDNSMsgStatic(false)
 				staticDnsTrue, _ := matcher.NewMatchDNSMsgStatic(true)
 				Expect(err).To(Succeed())
-				Expect(fp).To(Equal(&pmatcher.Matcher{
+				Expect(fp).To(Equal(pmatcher.UpdateSet(&pmatcher.Matcher{
 					PluginCommon: plugin.PluginCommon{Name: "matcher"},
-					Set: &matcher.MatcherSet{
-						Op:             matcher.SetOpOR,
-						DnstapMatchers: []matcher.DnstapMatcher{staticDnstapFalse},
-						DnsMsgMatchers: []matcher.DnsMsgMatcher{staticDnsFalse},
-						SubSets: []*matcher.MatcherSet{
-							{
-								Op:             matcher.SetOpOR,
-								DnsMsgMatchers: []matcher.DnsMsgMatcher{staticDnsFalse, staticDnsTrue},
-							},
+				}, &matcher.MatcherSet{
+					Op:             matcher.SetOpOR,
+					DnstapMatchers: []matcher.DnstapMatcher{staticDnstapFalse},
+					DnsMsgMatchers: []matcher.DnsMsgMatcher{staticDnsFalse},
+					SubSets: []*matcher.MatcherSet{
+						{
+							Op:             matcher.SetOpOR,
+							DnsMsgMatchers: []matcher.DnsMsgMatcher{staticDnsFalse, staticDnsTrue},
 						},
-					}}))
+					},
+				})))
 			})
 		})
 	})
@@ -85,12 +85,11 @@ var _ = Describe("Static", func() {
 		BeforeEach(func() {
 			staticDnstapTrue, _ = matcher.NewMatchDnstapStatic(true)
 			staticDnsTrue, _ = matcher.NewMatchDNSMsgStatic(true)
-			fp = &pmatcher.Matcher{
-				Set: &matcher.MatcherSet{
-					Op:             matcher.SetOpAND,
-					DnstapMatchers: []matcher.DnstapMatcher{staticDnstapTrue},
-				},
-			}
+			fp = pmatcher.UpdateSet(&pmatcher.Matcher{}, &matcher.MatcherSet{
+				Op:             matcher.SetOpAND,
+				DnstapMatchers: []matcher.DnstapMatcher{staticDnstapTrue},
+			},
+			)
 		})
 		When("valid dns message", func() {
 			BeforeEach(func() {
@@ -114,7 +113,9 @@ var _ = Describe("Static", func() {
 			When("match rule", func() {
 				BeforeEach(func() {
 					m, _ := matcher.NewMatchDNSMsgQueryName("www.example.jp.")
-					fp.Set.DnsMsgMatchers = append(fp.Set.DnsMsgMatchers, staticDnsTrue, m)
+					fpSet := pmatcher.GetSet(fp)
+					fpSet.DnsMsgMatchers = append(fpSet.DnsMsgMatchers, staticDnsTrue, m)
+					pmatcher.UpdateSet(fp, fpSet)
 					dm2 = fp.Filter(dm1)
 				})
 				It("through", func() {
@@ -124,7 +125,9 @@ var _ = Describe("Static", func() {
 			When("not match rule", func() {
 				BeforeEach(func() {
 					m, _ := matcher.NewMatchDNSMsgQueryName("www.example.com")
-					fp.Set.DnsMsgMatchers = append(fp.Set.DnsMsgMatchers, staticDnsTrue, m)
+					fpSet := pmatcher.GetSet(fp)
+					fpSet.DnsMsgMatchers = append(fpSet.DnsMsgMatchers, staticDnsTrue, m)
+					pmatcher.UpdateSet(fp, fpSet)
 					dm2 = fp.Filter(dm1)
 				})
 				It("filterd", func() {