From 33bd6bd78786564f56f458618df611e700eeeea3 Mon Sep 17 00:00:00 2001 From: koplas Date: Wed, 12 Jun 2024 10:08:06 +0200 Subject: [PATCH 1/8] Extend unittest coverage in util --- util/csv_test.go | 40 +++++++++ util/file_test.go | 141 ++++++++++++++++++++++++++++++- util/hash_test.go | 109 ++++++++++++++++++++++++ util/json_test.go | 209 ++++++++++++++++++++++++++++++++++++++++++++++ util/set_test.go | 65 ++++++++++++++ util/url_test.go | 36 ++++++++ 6 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 util/csv_test.go create mode 100644 util/hash_test.go create mode 100644 util/json_test.go create mode 100644 util/set_test.go create mode 100644 util/url_test.go diff --git a/util/csv_test.go b/util/csv_test.go new file mode 100644 index 00000000..a744b75b --- /dev/null +++ b/util/csv_test.go @@ -0,0 +1,40 @@ +// This file is Free Software under the Apache-2.0 License +// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. +// +// SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package util + +import ( + "bytes" + "testing" +) + +func TestCSV(t *testing.T) { + buf := new(bytes.Buffer) + csvWriter := NewFullyQuotedCSWWriter(buf) + for _, x := range [][]string{{"a", "b", "c"}, {"d", "e", "f"}} { + err := csvWriter.Write(x) + if err != nil { + t.Error(err) + } + } + + csvWriter.Flush() + err := csvWriter.Error() + if err != nil { + t.Error(err) + } + for _, want := range []string{`"a","b","c"`, `"d","e","f"`} { + got, err := buf.ReadString('\n') + if err != nil { + t.Error(err) + } + if got[:len(got)-1] != want { + t.Errorf("FullyQuotedCSWWriter: Expected %q but got %q.", want, got) + } + } +} diff --git a/util/file_test.go b/util/file_test.go index 3f648b8b..320f3d45 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -10,6 +10,8 @@ package util import ( "bytes" + "os" + "path/filepath" "testing" ) @@ -55,8 +57,54 @@ func TestConformingFileName(t *testing.T) { } } -func TestNWriter(t *testing.T) { +func TestIDMatchesFilename(t *testing.T) { + pathEval := NewPathEval() + + doc := make(map[string]interface{}) + doc["document"] = map[string]interface{}{ + "tracking": map[string]interface{}{ + "id": "valid.json", + }, + } + + err := IDMatchesFilename(pathEval, doc, "valid.json") + if err != nil { + t.Errorf("IDMatchesFilename: Expected nil, got %q", err) + } + + err = IDMatchesFilename(pathEval, doc, "different_file_name.json") + if err == nil { + t.Error("IDMatchesFilename: Expected error, got nil") + } + + doc["document"] = map[string]interface{}{ + "tracking": map[string]interface{}{}, + } + err = IDMatchesFilename(pathEval, doc, "valid.json") + if err == nil { + t.Error("IDMatchesFilename: Expected error, got nil") + } +} + +func TestPathExists(t *testing.T) { + got, err := PathExists("/this/path/does/not/exist") + if err != nil { + t.Error(err) + } + if got != false { + t.Error("PathExists: Expected false, got true") + } + dir := t.TempDir() + got, err = PathExists(dir) + if err != nil { + t.Error(err) + } + if got != true { + t.Error("PathExists: Expected true, got false") + } +} +func TestNWriter(t *testing.T) { msg := []byte("Gruß!\n") first, second := msg[:len(msg)/2], msg[len(msg)/2:] @@ -78,3 +126,94 @@ func TestNWriter(t *testing.T) { t.Errorf("Expected %q, but got %q", msg, out) } } + +func TestWriteToFile(t *testing.T) { + filename := filepath.Join(t.TempDir(), "test_file") + wt := bytes.NewBufferString("test_data") + err := WriteToFile(filename, wt) + if err != nil { + t.Error(err) + } + fileData, err := os.ReadFile(filename) + if err != nil { + t.Error(err) + } + if !bytes.Equal(fileData, []byte("test_data")) { + t.Errorf("DeepCopy: Expected test_data, got %v", fileData) + } +} + +func TestMakeUniqFile(t *testing.T) { + dir := t.TempDir() + _, file, err := MakeUniqFile(dir) + if err != nil { + t.Error(err) + } + _, err = file.Write([]byte("test_data")) + if err != nil { + t.Error(err) + } + err = file.Close() + if err != nil { + t.Error(err) + } +} + +func Test_mkUniq(t *testing.T) { + dir := t.TempDir() + name, err := mkUniq(dir+"/", func(name string) error { + return nil + }) + if err != nil { + t.Error(err) + } + firstTime := true + name1, err := mkUniq(dir+"/", func(_ string) error { + if firstTime { + firstTime = false + return os.ErrExist + } + return nil + }) + if err != nil { + t.Error(err) + } + if name == name1 { + t.Errorf("mkUniq: Expected unique names, got %v and %v", name, name1) + } +} + +func TestDeepCopy(t *testing.T) { + dir := t.TempDir() + os.MkdirAll(filepath.Join(dir, "src/folder0"), 0755) + os.MkdirAll(filepath.Join(dir, "dst"), 0755) + os.MkdirAll(filepath.Join(dir, "dst1"), 0755) + err := os.WriteFile(filepath.Join(dir, "src/folder0/test_file"), []byte("test_data"), 0755) + if err != nil { + t.Error(err) + } + + err = DeepCopy(filepath.Join(dir, "dst"), filepath.Join(dir, "src")) + if err != nil { + t.Error(err) + } + + fileData, err := os.ReadFile(filepath.Join(dir, "dst/folder0/test_file")) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(fileData, []byte("test_data")) { + t.Errorf("DeepCopy: Expected test_data, got %v", fileData) + } + + err = DeepCopy("/path/does/not/exist", filepath.Join(dir, "src")) + if err == nil { + t.Error("DeepCopy: Expected error, got nil") + } + + err = DeepCopy(filepath.Join(dir, "dst1"), "/path/does/not/exist") + if err == nil { + t.Error("DeepCopy: Expected error, got nil") + } +} diff --git a/util/hash_test.go b/util/hash_test.go new file mode 100644 index 00000000..ed0f0b22 --- /dev/null +++ b/util/hash_test.go @@ -0,0 +1,109 @@ +// This file is Free Software under the Apache-2.0 License +// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. +// +// SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package util + +import ( + "hash" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestHashFromReader(t *testing.T) { + r := strings.NewReader("deadbeef") + want := []byte{0xde, 0xad, 0xbe, 0xef} + if got, err := HashFromReader(r); !reflect.DeepEqual(want, got) { + if err != nil { + t.Error(err) + } + t.Errorf("HashFromReader: Expected %v, got %v", want, got) + } +} + +func TestHashFromFile(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "test_file") + testFile, err := os.Create(filePath) + if err != nil { + t.Error(err) + } + + testFile.WriteString("deadbeef") + want := []byte{0xde, 0xad, 0xbe, 0xef} + + testFile.Close() + + if got, err := HashFromFile(filePath); !reflect.DeepEqual(want, got) { + if err != nil { + t.Error(err) + } + t.Errorf("HashFromFile: Expected %v, got %v", want, got) + } +} + +type deadbeefHash struct { + hash.Hash +} + +func (deadbeefHash) Write(p []byte) (int, error) { return len(p), nil } +func (deadbeefHash) Sum(_ []byte) []byte { return []byte{0xde, 0xad, 0xbe, 0xef} } + +func TestWriteHashToFile(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "test_file") + + hashArg := deadbeefHash{} + nameArg := "name" + want := "deadbeef " + nameArg + "\n" + + err := WriteHashToFile(filePath, nameArg, hashArg, []byte{}) + if err != nil { + t.Error(err) + } + testFile, err := os.Open(filePath) + if err != nil { + t.Error(err) + } + defer testFile.Close() + fileContent, err := os.ReadFile(filePath) + if err != nil { + t.Error(err) + } + if got := string(fileContent); got != want { + t.Errorf("WriteHashToFile: Expected %v, got %v", want, got) + } +} + +func TestWriteHashSumToFile(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "test_file") + + sum := []byte{0xde, 0xad, 0xbe, 0xef} + nameArg := "name" + want := "deadbeef " + nameArg + "\n" + + err := WriteHashSumToFile(filePath, nameArg, sum) + if err != nil { + t.Error(err) + } + testFile, err := os.Open(filePath) + if err != nil { + t.Error(err) + } + defer testFile.Close() + fileContent, err := os.ReadFile(filePath) + if err != nil { + t.Error(err) + } + if got := string(fileContent); got != want { + t.Errorf("WriteHashSumToFile: Expected %v, got %v", want, got) + } +} diff --git a/util/json_test.go b/util/json_test.go new file mode 100644 index 00000000..452fabe9 --- /dev/null +++ b/util/json_test.go @@ -0,0 +1,209 @@ +// This file is Free Software under the Apache-2.0 License +// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. +// +// SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package util + +import ( + "context" + "reflect" + "testing" + "time" +) + +func TestPathEval_Compile(t *testing.T) { + pathEval := NewPathEval() + eval, err := pathEval.Compile("foo") + if err != nil { + t.Error(err) + } + + // Check caching + eval1, err := pathEval.Compile("foo") + if err != nil { + t.Error(err) + } + if reflect.ValueOf(eval).Pointer() != reflect.ValueOf(eval1).Pointer() { + t.Error("PathEval_Compile: Expected cached eval") + } + + got, err := eval.EvalInt(context.Background(), map[string]interface{}{"foo": 5}) + if err != nil { + t.Error(err) + } + if got != 5 { + t.Errorf("PathEval_Compile: Expected 5, got %v", got) + } +} + +func TestPathEval_Eval(t *testing.T) { + pathEval := NewPathEval() + _, err := pathEval.Eval("foo", nil) + if err == nil { + t.Error("PathEval_Eval: Expected error, got nil") + } + got, err := pathEval.Eval("foo", map[string]interface{}{"foo": 5}) + if err != nil { + t.Error(err) + } + if got != 5 { + t.Errorf("PathEval_Compile: Expected 5, got %v", got) + } +} + +func TestReMarshalMatcher(t *testing.T) { + var intDst int + var uintSrc uint = 2 + remarshalFunc := ReMarshalMatcher(&intDst) + err := remarshalFunc(uintSrc) + if err != nil { + t.Error(err) + } + if intDst != 2 { + t.Errorf("ReMarshalMatcher: Expected %v, got %v", uintSrc, intDst) + } +} + +func TestBoolMatcher(t *testing.T) { + var boolDst bool + boolFunc := BoolMatcher(&boolDst) + err := boolFunc(true) + if err != nil { + t.Error(err) + } + + if boolDst != true { + t.Error("BoolMatcher: Expected true got false") + } + + err = boolFunc(1) + if err == nil { + t.Error("BoolMatcher: Expected error, got nil") + } +} + +func TestStringMatcher(t *testing.T) { + var stringDst string + stringFunc := StringMatcher(&stringDst) + err := stringFunc("test") + if err != nil { + t.Error(err) + } + + if stringDst != "test" { + t.Errorf("StringMatcher: Expected test, got %v", stringDst) + } + + err = stringFunc(1) + if err == nil { + t.Error("StringMatcher: Expected error, got nil") + } +} + +func TestStringTreeMatcher(t *testing.T) { + var stringTreeDst []string + stringTreeFunc := StringTreeMatcher(&stringTreeDst) + err := stringTreeFunc([]any{"a", "a", "b"}) + if err != nil { + t.Error(err) + } + + wantAnySlice := []any{"a", "b"} + if reflect.DeepEqual(stringTreeDst, wantAnySlice) { + t.Errorf("StringTreeMatcher: Expected %v, got %v", wantAnySlice, stringTreeDst) + } + + err = stringTreeFunc([]string{"a", "a", "b"}) + if err == nil { + t.Error("StringTreeMatcher: Expected error, got nil") + } + + err = stringTreeFunc(1) + if err == nil { + t.Error("StringTreeMatcher: Expected error, got nil") + } +} + +func TestTimeMatcher(t *testing.T) { + var timeDst time.Time + timeFunc := TimeMatcher(&timeDst, time.RFC3339) + err := timeFunc("2024-03-18T12:57:48.236Z") + if err != nil { + t.Error(err) + } + wantTime := time.Date(2024, time.March, 18, 12, 57, 48, 236_000_000, time.UTC) + if timeDst != wantTime { + t.Errorf("TimeMatcher: Expected %v, got %v", wantTime, timeDst) + } + + err = timeFunc("") + if err == nil { + t.Error("TimeMatcher: Expected error, got nil") + } + + err = timeFunc(1) + if err == nil { + t.Error("TimeMatcher: Expected error, got nil") + } +} + +func TestPathEval_Extract(t *testing.T) { + pathEval := NewPathEval() + var result string + matcher := StringMatcher(&result) + err := pathEval.Extract("foo", matcher, true, map[string]interface{}{"foo": "bar"}) + if err != nil { + t.Error(err) + } + if result != "bar" { + t.Errorf("PathEval_Extract: Expected bar, got %v", result) + } +} + +func TestPathEval_Match(t *testing.T) { + var got string + doc := map[string]interface{}{"foo": "bar"} + + pe := NewPathEval() + pem := PathEvalMatcher{Expr: "foo", Action: StringMatcher(&got)} + + err := pe.Match([]PathEvalMatcher{pem}, doc) + if err != nil { + t.Error(err) + } + if got != "bar" { + t.Errorf("PathEval_Match: Expected bar, got %v", got) + } +} + +func TestPathEval_Strings(t *testing.T) { + pe := NewPathEval() + doc := map[string]interface{}{"foo": "bar"} + want := []string{"bar"} + + got, err := pe.Strings([]string{"foo"}, true, doc) + if err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("PathEval_Strings: Expected %v, got %v", want, got) + } +} + +func TestAsStrings(t *testing.T) { + arg := []interface{}{"foo", "bar"} + want := []string{"foo", "bar"} + + got, valid := AsStrings(arg) + if !valid { + t.Error("AsStrings: Expected true, got false") + } + if !reflect.DeepEqual(got, want) { + t.Errorf("AsStrings: Expected %v, got %v", want, got) + } +} diff --git a/util/set_test.go b/util/set_test.go new file mode 100644 index 00000000..a28878e6 --- /dev/null +++ b/util/set_test.go @@ -0,0 +1,65 @@ +// This file is Free Software under the Apache-2.0 License +// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. +// +// SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package util + +import ( + "reflect" + "sort" + "testing" +) + +func TestSet(t *testing.T) { + s := Set[int]{} + if s.Contains(0) { + t.Error("Set.Contains: Expected false got true") + } + s.Add(0) + if !s.Contains(0) { + t.Error("Set.Contains: Expected true got false") + } + + s0 := Set[int]{} + s1 := Set[int]{} + + s0.Add(0) + s0.Add(1) + + s1.Add(0) + s1.Add(1) + s1.Add(2) + + diff0 := s0.Difference(s1) + diff1 := s1.Difference(s0) + + if reflect.DeepEqual(diff0, diff1) { + t.Errorf("Set.Difference: %q and %q are different", diff0, diff1) + } + + if s0.ContainsAll(s1) { + t.Error("Set.ContainsAll: Expected false got true") + } + + if !s1.ContainsAll(s0) { + t.Error("Set.ContainsAll: Expected true got false") + } + + s2 := Set[int]{} + s2.Add(0) + s2.Add(1) + s2.Add(2) + s2.Add(3) + + wantKeys := []int{0, 1, 2, 3} + gotKeys := s2.Keys() + sort.Ints(gotKeys) + + if !reflect.DeepEqual(wantKeys, gotKeys) { + t.Errorf("Set.Keys: Expected %q got %q", wantKeys, gotKeys) + } +} diff --git a/util/url_test.go b/util/url_test.go new file mode 100644 index 00000000..dec73dce --- /dev/null +++ b/util/url_test.go @@ -0,0 +1,36 @@ +// This file is Free Software under the Apache-2.0 License +// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. +// +// SPDX-License-Identifier: Apache-2.0 +// +// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) +// Software-Engineering: 2022 Intevation GmbH + +package util + +import ( + "net/url" + "testing" +) + +func TestBaseUrl(t *testing.T) { + for _, x := range [][2]string{ + {`http://example.com`, `http://example.com/`}, + {`scheme://example.com`, `scheme://example.com/`}, + {`https://example.com`, `https://example.com/`}, + {`https://example.com:8080/`, `https://example.com:8080/`}, + {`https://user@example.com:8080/`, `https://user@example.com:8080/`}, + {`https://user@example.com:8080/resource`, `https://user@example.com:8080/`}, + {`https://user@example.com:8080/resource/`, `https://user@example.com:8080/resource/`}, + {`https://user@example.com:8080/resource/#fragment`, `https://user@example.com:8080/resource/`}, + {`https://user@example.com:8080/resource/?query=test#fragment`, `https://user@example.com:8080/resource/`}, + } { + url, _ := url.Parse(x[0]) + if got, err := BaseURL(url); got != x[1] { + if err != nil { + t.Error(err) + } + t.Errorf("%q: Expected %q but got %q.", x[0], x[1], got) + } + } +} From e2ad3d3f8302a81be9fe4d20153aac2f0dc041bd Mon Sep 17 00:00:00 2001 From: "Bernhard E. Reiter" Date: Fri, 21 Jun 2024 14:02:51 +0200 Subject: [PATCH 2/8] docs: fix licensing info for generated files (#542) * docs: fix licensing info for generated files * change generate_cvss_enums.go to note that the input file is relevant for the license. * change license and copyright of cvss20enums.go and cvss3enums.go to BSD-3-Clause and FIRST. * add reuse.software 3.0 compatible files for the schema cvss files. * Stamp right license into generated files. --------- Co-authored-by: Sascha L. Teichmann --- LICENSES/BSD-3-Clause.txt | 11 +++++++++++ csaf/cvss20enums.go | 9 ++------- csaf/cvss3enums.go | 9 ++------- csaf/generate_cvss_enums.go | 28 +++++++++++++++++++++------- csaf/schema/cvss-v2.0.json.license | 2 ++ csaf/schema/cvss-v3.0.json.license | 2 ++ csaf/schema/cvss-v3.1.json.license | 2 ++ 7 files changed, 42 insertions(+), 21 deletions(-) create mode 100644 LICENSES/BSD-3-Clause.txt create mode 100644 csaf/schema/cvss-v2.0.json.license create mode 100644 csaf/schema/cvss-v3.0.json.license create mode 100644 csaf/schema/cvss-v3.1.json.license diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt new file mode 100644 index 00000000..ea890afb --- /dev/null +++ b/LICENSES/BSD-3-Clause.txt @@ -0,0 +1,11 @@ +Copyright (c) . + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/csaf/cvss20enums.go b/csaf/cvss20enums.go index 7056f3e7..97d2e105 100644 --- a/csaf/cvss20enums.go +++ b/csaf/cvss20enums.go @@ -1,10 +1,5 @@ -// This file is Free Software under the Apache-2.0 License -// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. -// -// SPDX-License-Identifier: Apache-2.0 -// -// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) -// Software-Engineering: 2023 Intevation GmbH +// SPDX-License-Identifier: BSD-3-Clause +// SPDX-FileCopyrightText: 2017 FIRST.ORG, INC. // // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE! diff --git a/csaf/cvss3enums.go b/csaf/cvss3enums.go index b8cf54f3..32e01e3a 100644 --- a/csaf/cvss3enums.go +++ b/csaf/cvss3enums.go @@ -1,10 +1,5 @@ -// This file is Free Software under the Apache-2.0 License -// without warranty, see README.md and LICENSES/Apache-2.0.txt for details. -// -// SPDX-License-Identifier: Apache-2.0 -// -// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) -// Software-Engineering: 2023 Intevation GmbH +// SPDX-License-Identifier: BSD-3-Clause +// SPDX-FileCopyrightText: 2017 FIRST.ORG, INC. // // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE! diff --git a/csaf/generate_cvss_enums.go b/csaf/generate_cvss_enums.go index 7c9b9fd6..c84ab158 100644 --- a/csaf/generate_cvss_enums.go +++ b/csaf/generate_cvss_enums.go @@ -14,21 +14,21 @@ import ( "bytes" "encoding/json" "flag" + "fmt" "go/format" "log" "os" + "regexp" "sort" "strings" "text/template" ) -const tmplText = `// This file is Free Software under the MIT License -// without warranty, see README.md and LICENSES/MIT.txt for details. -// -// SPDX-License-Identifier: MIT -// -// SPDX-FileCopyrightText: 2023 German Federal Office for Information Security (BSI) -// Software-Engineering: 2023 Intevation GmbH +// We from Intevation consider the source code parts in the following +// template file as too insignificant to be a piece of work that gains +// "copyrights" protection in the European Union. So the license(s) +// of the output files are fully determined by the input file. +const tmplText = `// {{ $.License }} // // THIS FILE IS MACHINE GENERATED. EDIT WITH CARE! @@ -69,6 +69,7 @@ type definition struct { } type schema struct { + License []string `json:"license"` Definitions map[string]*definition `json:"definitions"` } @@ -137,9 +138,22 @@ func main() { } sort.Strings(defs) + license := "determine license(s) from input file and replace this line" + + pattern := regexp.MustCompile(`Copyright \(c\) (\d+), FIRST.ORG, INC.`) + for _, line := range s.License { + if m := pattern.FindStringSubmatch(line); m != nil { + license = fmt.Sprintf( + "SPDX-License-Identifier: BSD-3-Clause\n"+ + "// SPDX-FileCopyrightText: %s FIRST.ORG, INC.", m[1]) + break + } + } + var source bytes.Buffer check(tmpl.Execute(&source, map[string]any{ + "License": license, "Prefix": *prefix, "Definitions": s.Definitions, "Keys": defs, diff --git a/csaf/schema/cvss-v2.0.json.license b/csaf/schema/cvss-v2.0.json.license new file mode 100644 index 00000000..dd033e8c --- /dev/null +++ b/csaf/schema/cvss-v2.0.json.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: BSD-3-Clause +SPDX-FileCopyrightText: 2017 FIRST.ORG, INC. diff --git a/csaf/schema/cvss-v3.0.json.license b/csaf/schema/cvss-v3.0.json.license new file mode 100644 index 00000000..dd033e8c --- /dev/null +++ b/csaf/schema/cvss-v3.0.json.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: BSD-3-Clause +SPDX-FileCopyrightText: 2017 FIRST.ORG, INC. diff --git a/csaf/schema/cvss-v3.1.json.license b/csaf/schema/cvss-v3.1.json.license new file mode 100644 index 00000000..f87ced89 --- /dev/null +++ b/csaf/schema/cvss-v3.1.json.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: BSD-3-Clause +SPDX-FileCopyrightText: 2021 FIRST.ORG, INC. From 56fadc3a80f66d0006203e9983138c5171b07fbf Mon Sep 17 00:00:00 2001 From: "Bernhard E. Reiter" Date: Fri, 21 Jun 2024 14:04:20 +0200 Subject: [PATCH 3/8] docs: fix typo in examples/aggregator.toml (#539) --- docs/examples/aggregator.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/aggregator.toml b/docs/examples/aggregator.toml index ae1723d1..2161079a 100644 --- a/docs/examples/aggregator.toml +++ b/docs/examples/aggregator.toml @@ -51,7 +51,7 @@ insecure = true # rate = 1.8 # insecure = true write_indices = true - # If aggregator.category == "aggreator", set for an entry that should + # If aggregator.category == "aggregator", set for an entry that should # be listed in addition: category = "lister" # ignore_pattern = [".*white.*", ".*red.*"] From 3084cdbc371f03adfe22c1640b53b43fed5a0563 Mon Sep 17 00:00:00 2001 From: koplas <54645365+koplas@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:35:30 +0200 Subject: [PATCH 4/8] Address comments --- util/csv_test.go | 6 ++---- util/file_test.go | 54 ++++++++++++++++++++++------------------------- util/hash_test.go | 6 ++---- util/json_test.go | 49 ++++++++++++++++-------------------------- 4 files changed, 47 insertions(+), 68 deletions(-) diff --git a/util/csv_test.go b/util/csv_test.go index a744b75b..575d83d9 100644 --- a/util/csv_test.go +++ b/util/csv_test.go @@ -17,15 +17,13 @@ func TestCSV(t *testing.T) { buf := new(bytes.Buffer) csvWriter := NewFullyQuotedCSWWriter(buf) for _, x := range [][]string{{"a", "b", "c"}, {"d", "e", "f"}} { - err := csvWriter.Write(x) - if err != nil { + if err := csvWriter.Write(x); err != nil { t.Error(err) } } csvWriter.Flush() - err := csvWriter.Error() - if err != nil { + if err := csvWriter.Error(); err != nil { t.Error(err) } for _, want := range []string{`"a","b","c"`, `"d","e","f"`} { diff --git a/util/file_test.go b/util/file_test.go index 320f3d45..28c51967 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -60,28 +60,25 @@ func TestConformingFileName(t *testing.T) { func TestIDMatchesFilename(t *testing.T) { pathEval := NewPathEval() - doc := make(map[string]interface{}) - doc["document"] = map[string]interface{}{ - "tracking": map[string]interface{}{ + doc := make(map[string]any) + doc["document"] = map[string]any{ + "tracking": map[string]any{ "id": "valid.json", }, } - err := IDMatchesFilename(pathEval, doc, "valid.json") - if err != nil { + if err := IDMatchesFilename(pathEval, doc, "valid.json"); err != nil { t.Errorf("IDMatchesFilename: Expected nil, got %q", err) } - err = IDMatchesFilename(pathEval, doc, "different_file_name.json") - if err == nil { + if err := IDMatchesFilename(pathEval, doc, "different_file_name.json"); err == nil { t.Error("IDMatchesFilename: Expected error, got nil") } - doc["document"] = map[string]interface{}{ - "tracking": map[string]interface{}{}, + doc["document"] = map[string]any{ + "tracking": map[string]any{}, } - err = IDMatchesFilename(pathEval, doc, "valid.json") - if err == nil { + if err := IDMatchesFilename(pathEval, doc, "valid.json"); err == nil { t.Error("IDMatchesFilename: Expected error, got nil") } } @@ -130,8 +127,7 @@ func TestNWriter(t *testing.T) { func TestWriteToFile(t *testing.T) { filename := filepath.Join(t.TempDir(), "test_file") wt := bytes.NewBufferString("test_data") - err := WriteToFile(filename, wt) - if err != nil { + if err := WriteToFile(filename, wt); err != nil { t.Error(err) } fileData, err := os.ReadFile(filename) @@ -149,12 +145,10 @@ func TestMakeUniqFile(t *testing.T) { if err != nil { t.Error(err) } - _, err = file.Write([]byte("test_data")) - if err != nil { + if _, err = file.Write([]byte("test_data")); err != nil { t.Error(err) } - err = file.Close() - if err != nil { + if err = file.Close(); err != nil { t.Error(err) } } @@ -185,16 +179,20 @@ func Test_mkUniq(t *testing.T) { func TestDeepCopy(t *testing.T) { dir := t.TempDir() - os.MkdirAll(filepath.Join(dir, "src/folder0"), 0755) - os.MkdirAll(filepath.Join(dir, "dst"), 0755) - os.MkdirAll(filepath.Join(dir, "dst1"), 0755) - err := os.WriteFile(filepath.Join(dir, "src/folder0/test_file"), []byte("test_data"), 0755) - if err != nil { - t.Error(err) + if err := os.MkdirAll(filepath.Join(dir, "src/folder0"), 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "dst"), 0755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "dst1"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "src/folder0/test_file"), []byte("test_data"), 0755); err != nil { + t.Fatal(err) } - err = DeepCopy(filepath.Join(dir, "dst"), filepath.Join(dir, "src")) - if err != nil { + if err := DeepCopy(filepath.Join(dir, "dst"), filepath.Join(dir, "src")); err != nil { t.Error(err) } @@ -207,13 +205,11 @@ func TestDeepCopy(t *testing.T) { t.Errorf("DeepCopy: Expected test_data, got %v", fileData) } - err = DeepCopy("/path/does/not/exist", filepath.Join(dir, "src")) - if err == nil { + if err = DeepCopy("/path/does/not/exist", filepath.Join(dir, "src")); err == nil { t.Error("DeepCopy: Expected error, got nil") } - err = DeepCopy(filepath.Join(dir, "dst1"), "/path/does/not/exist") - if err == nil { + if err = DeepCopy(filepath.Join(dir, "dst1"), "/path/does/not/exist"); err == nil { t.Error("DeepCopy: Expected error, got nil") } } diff --git a/util/hash_test.go b/util/hash_test.go index ed0f0b22..d6908915 100644 --- a/util/hash_test.go +++ b/util/hash_test.go @@ -64,8 +64,7 @@ func TestWriteHashToFile(t *testing.T) { nameArg := "name" want := "deadbeef " + nameArg + "\n" - err := WriteHashToFile(filePath, nameArg, hashArg, []byte{}) - if err != nil { + if err := WriteHashToFile(filePath, nameArg, hashArg, []byte{}); err != nil { t.Error(err) } testFile, err := os.Open(filePath) @@ -90,8 +89,7 @@ func TestWriteHashSumToFile(t *testing.T) { nameArg := "name" want := "deadbeef " + nameArg + "\n" - err := WriteHashSumToFile(filePath, nameArg, sum) - if err != nil { + if err := WriteHashSumToFile(filePath, nameArg, sum); err != nil { t.Error(err) } testFile, err := os.Open(filePath) diff --git a/util/json_test.go b/util/json_test.go index 452fabe9..ba18171c 100644 --- a/util/json_test.go +++ b/util/json_test.go @@ -31,7 +31,7 @@ func TestPathEval_Compile(t *testing.T) { t.Error("PathEval_Compile: Expected cached eval") } - got, err := eval.EvalInt(context.Background(), map[string]interface{}{"foo": 5}) + got, err := eval.EvalInt(context.Background(), map[string]any{"foo": 5}) if err != nil { t.Error(err) } @@ -46,7 +46,7 @@ func TestPathEval_Eval(t *testing.T) { if err == nil { t.Error("PathEval_Eval: Expected error, got nil") } - got, err := pathEval.Eval("foo", map[string]interface{}{"foo": 5}) + got, err := pathEval.Eval("foo", map[string]any{"foo": 5}) if err != nil { t.Error(err) } @@ -59,8 +59,7 @@ func TestReMarshalMatcher(t *testing.T) { var intDst int var uintSrc uint = 2 remarshalFunc := ReMarshalMatcher(&intDst) - err := remarshalFunc(uintSrc) - if err != nil { + if err := remarshalFunc(uintSrc); err != nil { t.Error(err) } if intDst != 2 { @@ -71,8 +70,7 @@ func TestReMarshalMatcher(t *testing.T) { func TestBoolMatcher(t *testing.T) { var boolDst bool boolFunc := BoolMatcher(&boolDst) - err := boolFunc(true) - if err != nil { + if err := boolFunc(true); err != nil { t.Error(err) } @@ -80,8 +78,7 @@ func TestBoolMatcher(t *testing.T) { t.Error("BoolMatcher: Expected true got false") } - err = boolFunc(1) - if err == nil { + if err := boolFunc(1); err == nil { t.Error("BoolMatcher: Expected error, got nil") } } @@ -89,8 +86,7 @@ func TestBoolMatcher(t *testing.T) { func TestStringMatcher(t *testing.T) { var stringDst string stringFunc := StringMatcher(&stringDst) - err := stringFunc("test") - if err != nil { + if err := stringFunc("test"); err != nil { t.Error(err) } @@ -98,8 +94,7 @@ func TestStringMatcher(t *testing.T) { t.Errorf("StringMatcher: Expected test, got %v", stringDst) } - err = stringFunc(1) - if err == nil { + if err := stringFunc(1); err == nil { t.Error("StringMatcher: Expected error, got nil") } } @@ -107,8 +102,7 @@ func TestStringMatcher(t *testing.T) { func TestStringTreeMatcher(t *testing.T) { var stringTreeDst []string stringTreeFunc := StringTreeMatcher(&stringTreeDst) - err := stringTreeFunc([]any{"a", "a", "b"}) - if err != nil { + if err := stringTreeFunc([]any{"a", "a", "b"}); err != nil { t.Error(err) } @@ -117,13 +111,11 @@ func TestStringTreeMatcher(t *testing.T) { t.Errorf("StringTreeMatcher: Expected %v, got %v", wantAnySlice, stringTreeDst) } - err = stringTreeFunc([]string{"a", "a", "b"}) - if err == nil { + if err := stringTreeFunc([]string{"a", "a", "b"}); err == nil { t.Error("StringTreeMatcher: Expected error, got nil") } - err = stringTreeFunc(1) - if err == nil { + if err := stringTreeFunc(1); err == nil { t.Error("StringTreeMatcher: Expected error, got nil") } } @@ -131,8 +123,7 @@ func TestStringTreeMatcher(t *testing.T) { func TestTimeMatcher(t *testing.T) { var timeDst time.Time timeFunc := TimeMatcher(&timeDst, time.RFC3339) - err := timeFunc("2024-03-18T12:57:48.236Z") - if err != nil { + if err := timeFunc("2024-03-18T12:57:48.236Z"); err != nil { t.Error(err) } wantTime := time.Date(2024, time.March, 18, 12, 57, 48, 236_000_000, time.UTC) @@ -140,13 +131,11 @@ func TestTimeMatcher(t *testing.T) { t.Errorf("TimeMatcher: Expected %v, got %v", wantTime, timeDst) } - err = timeFunc("") - if err == nil { + if err := timeFunc(""); err == nil { t.Error("TimeMatcher: Expected error, got nil") } - err = timeFunc(1) - if err == nil { + if err := timeFunc(1); err == nil { t.Error("TimeMatcher: Expected error, got nil") } } @@ -155,8 +144,7 @@ func TestPathEval_Extract(t *testing.T) { pathEval := NewPathEval() var result string matcher := StringMatcher(&result) - err := pathEval.Extract("foo", matcher, true, map[string]interface{}{"foo": "bar"}) - if err != nil { + if err := pathEval.Extract("foo", matcher, true, map[string]any{"foo": "bar"}); err != nil { t.Error(err) } if result != "bar" { @@ -166,13 +154,12 @@ func TestPathEval_Extract(t *testing.T) { func TestPathEval_Match(t *testing.T) { var got string - doc := map[string]interface{}{"foo": "bar"} + doc := map[string]any{"foo": "bar"} pe := NewPathEval() pem := PathEvalMatcher{Expr: "foo", Action: StringMatcher(&got)} - err := pe.Match([]PathEvalMatcher{pem}, doc) - if err != nil { + if err := pe.Match([]PathEvalMatcher{pem}, doc); err != nil { t.Error(err) } if got != "bar" { @@ -182,7 +169,7 @@ func TestPathEval_Match(t *testing.T) { func TestPathEval_Strings(t *testing.T) { pe := NewPathEval() - doc := map[string]interface{}{"foo": "bar"} + doc := map[string]any{"foo": "bar"} want := []string{"bar"} got, err := pe.Strings([]string{"foo"}, true, doc) @@ -196,7 +183,7 @@ func TestPathEval_Strings(t *testing.T) { } func TestAsStrings(t *testing.T) { - arg := []interface{}{"foo", "bar"} + arg := []any{"foo", "bar"} want := []string{"foo", "bar"} got, valid := AsStrings(arg) From 5c6736b178b113f6abc2cad6efd9301d5fbbe18e Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Mon, 24 Jun 2024 11:57:38 +0200 Subject: [PATCH 5/8] Remove data races in downloader caused by shared use of json path eval. (#547) --- cmd/csaf_downloader/downloader.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index c5c3e021..a0cf34e5 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -40,7 +40,6 @@ import ( type downloader struct { cfg *config keys *crypto.KeyRing - eval *util.PathEval validator csaf.RemoteValidator forwarder *forwarder mkdirMu sync.Mutex @@ -73,7 +72,6 @@ func newDownloader(cfg *config) (*downloader, error) { return &downloader{ cfg: cfg, - eval: util.NewPathEval(), validator: validator, }, nil } @@ -218,17 +216,20 @@ func (d *downloader) download(ctx context.Context, domain string) error { return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err) } + expr := util.NewPathEval() + if err := d.loadOpenPGPKeys( client, lpmd.Document, base, + expr, ); err != nil { return err } afp := csaf.NewAdvisoryFileProcessor( client, - d.eval, + expr, lpmd.Document, base) @@ -297,9 +298,10 @@ func (d *downloader) loadOpenPGPKeys( client util.Client, doc any, base *url.URL, + expr *util.PathEval, ) error { - src, err := d.eval.Eval("$.public_openpgp_keys", doc) + src, err := expr.Eval("$.public_openpgp_keys", doc) if err != nil { // no keys. return nil @@ -421,6 +423,7 @@ func (d *downloader) downloadWorker( dateExtract = util.TimeMatcher(&initialReleaseDate, time.RFC3339) lower = strings.ToLower(string(label)) stats = stats{} + expr = util.NewPathEval() ) // Add collected stats back to total. @@ -588,7 +591,7 @@ nextAdvisory: // Validate if filename is conforming. filenameCheck := func() error { - if err := util.IDMatchesFilename(d.eval, doc, filename); err != nil { + if err := util.IDMatchesFilename(expr, doc, filename); err != nil { stats.filenameFailed++ return fmt.Errorf("filename not conforming %s: %s", file.URL(), err) } @@ -651,7 +654,7 @@ nextAdvisory: continue } - if err := d.eval.Extract( + if err := expr.Extract( `$.document.tracking.initial_release_date`, dateExtract, false, doc, ); err != nil { slog.Warn("Cannot extract initial_release_date from advisory", From a46c286cf482451e8f395d367ef8ad3c705cdfd4 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Fri, 19 Apr 2024 14:04:12 +0200 Subject: [PATCH 6/8] fix: don't drop error messages from loading provider-metadata.json previously in case case of trying last resort dns, all other error messages were dropped --- csaf/providermetaloader.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 203f2b35..0c4fc3bd 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -173,6 +173,8 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata // We have a candidate. if wellknownResult.Valid() { wellknownGood = wellknownResult + } else { + pmdl.messages.AppendUnique(wellknownResult.Messages) } // Next load the PMDs from security.txt @@ -220,25 +222,28 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata } } // Take the good well-known. - wellknownGood.Messages.AppendUnique(pmdl.messages) + wellknownGood.Messages = pmdl.messages return wellknownGood } // Don't have well-known. Take first good from security.txt. ignoreExtras() - secGoods[0].Messages.AppendUnique(pmdl.messages) + secGoods[0].Messages = pmdl.messages return secGoods[0] } // If we have a good well-known take it. if wellknownGood != nil { - wellknownGood.Messages.AppendUnique(pmdl.messages) + wellknownGood.Messages = pmdl.messages return wellknownGood } // Last resort: fall back to DNS. dnsURL := "https://csaf.data.security." + domain - return pmdl.loadFromURL(dnsURL) + dnsURLResult := pmdl.loadFromURL(dnsURL) + pmdl.messages.AppendUnique(dnsURLResult.Messages) // keep order of messages consistent (i.e. last occurred message is last element) + dnsURLResult.Messages = pmdl.messages + return dnsURLResult } // loadFromSecurity loads the PMDs mentioned in the security.txt. From 51dc9b5bcb26c74bc3e46f3c9cf0e7d190cc41d1 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Fri, 19 Apr 2024 14:06:56 +0200 Subject: [PATCH 7/8] refactor: deduplicate filtering pmd results from security.txt already done in `loadFromSecurity` --- csaf/providermetaloader.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/csaf/providermetaloader.go b/csaf/providermetaloader.go index 0c4fc3bd..b21ddc61 100644 --- a/csaf/providermetaloader.go +++ b/csaf/providermetaloader.go @@ -178,20 +178,7 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata } // Next load the PMDs from security.txt - secResults := pmdl.loadFromSecurity(domain) - - // Filter out the results which are valid. - var secGoods []*LoadedProviderMetadata - - for _, result := range secResults { - if len(result.Messages) > 0 { - // If there where validation issues append them - // to the overall report - pmdl.messages.AppendUnique(pmdl.messages) - } else { - secGoods = append(secGoods, result) - } - } + secGoods := pmdl.loadFromSecurity(domain) // Mention extra CSAF entries in security.txt. ignoreExtras := func() { @@ -246,7 +233,7 @@ func (pmdl *ProviderMetadataLoader) Load(domain string) *LoadedProviderMetadata return dnsURLResult } -// loadFromSecurity loads the PMDs mentioned in the security.txt. +// loadFromSecurity loads the PMDs mentioned in the security.txt. Only valid PMDs are returned. func (pmdl *ProviderMetadataLoader) loadFromSecurity(domain string) []*LoadedProviderMetadata { // If .well-known fails try legacy location. From 1e531de82d35ab549fa4b07f828f21a38554c3a5 Mon Sep 17 00:00:00 2001 From: Marius Goetze Date: Mon, 15 Jul 2024 10:52:13 +0200 Subject: [PATCH 8/8] fix: don't require debug level to print error details on failed loading of provider metadata json --- cmd/csaf_aggregator/processor.go | 14 +++++++++----- cmd/csaf_downloader/downloader.go | 13 ++++++++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go index 9f10a774..5cb36283 100644 --- a/cmd/csaf_aggregator/processor.go +++ b/cmd/csaf_aggregator/processor.go @@ -89,17 +89,21 @@ func (w *worker) locateProviderMetadata(domain string) error { lpmd := loader.Load(domain) - if w.processor.cfg.Verbose { + if !lpmd.Valid() { for i := range lpmd.Messages { - w.log.Info( + w.log.Error( "Loading provider-metadata.json", "domain", domain, "message", lpmd.Messages[i].Message) } - } - - if !lpmd.Valid() { return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) + } else if w.processor.cfg.Verbose { + for i := range lpmd.Messages { + w.log.Debug( + "Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) + } } w.metadataProvider = lpmd.Document diff --git a/cmd/csaf_downloader/downloader.go b/cmd/csaf_downloader/downloader.go index a0cf34e5..e370f55a 100644 --- a/cmd/csaf_downloader/downloader.go +++ b/cmd/csaf_downloader/downloader.go @@ -199,7 +199,14 @@ func (d *downloader) download(ctx context.Context, domain string) error { lpmd := loader.Load(domain) - if d.cfg.verbose() { + if !lpmd.Valid() { + for i := range lpmd.Messages { + slog.Error("Loading provider-metadata.json", + "domain", domain, + "message", lpmd.Messages[i].Message) + } + return fmt.Errorf("no valid provider-metadata.json found for '%s': ", domain) + } else if d.cfg.verbose() { for i := range lpmd.Messages { slog.Debug("Loading provider-metadata.json", "domain", domain, @@ -207,10 +214,6 @@ func (d *downloader) download(ctx context.Context, domain string) error { } } - if !lpmd.Valid() { - return fmt.Errorf("no valid provider-metadata.json found for '%s'", domain) - } - base, err := url.Parse(lpmd.URL) if err != nil { return fmt.Errorf("invalid URL '%s': %v", lpmd.URL, err)