Implement better search for provider-metadata.json

* Decouple loading of provider metadata from processor and moved in the base library. * Integrate new code into checker and aggregator * Adhere to csd02 revision of CSAF 2.0. resolve #60
gocsaf · May 31, 2022 · 527a6f6 · 527a6f6
1 parent dca6f2c
commit 527a6f6
Show file tree

Hide file tree

Showing 3 changed files with 236 additions and 207 deletions.
diff --git a/cmd/csaf_aggregator/processor.go b/cmd/csaf_aggregator/processor.go
@@ -9,14 +9,10 @@
 package main
 
 import (
-	"encoding/json"
-	"errors"
-	"io"
+	"fmt"
 	"log"
-	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/ProtonMail/gopenpgp/v2/crypto"
 	"github.com/csaf-poc/csaf_distribution/csaf"
@@ -75,76 +71,22 @@ func (w *worker) createDir() (string, error) {
 	return dir, err
 }
 
-// httpsDomain prefixes a domain with 'https://'.
-func httpsDomain(domain string) string {
-	if strings.HasPrefix(domain, "https://") {
-		return domain
-	}
-	return "https://" + domain
-}
-
-var providerMetadataLocations = [...]string{
-	".well-known/csaf",
-	"security/data/csaf",
-	"advisories/csaf",
-	"security/csaf",
-}
-
 func (w *worker) locateProviderMetadata(domain string) error {
 
-	w.metadataProvider = nil
-
-	download := func(r io.Reader) error {
-		if err := json.NewDecoder(r).Decode(&w.metadataProvider); err != nil {
-			log.Printf("error: %s\n", err)
-			return errNotFound
-		}
-		return nil
-	}
-
-	hd := httpsDomain(domain)
-	for _, loc := range providerMetadataLocations {
-		url := hd + "/" + loc
-		if err := downloadJSON(w.client, url, download); err != nil {
-			if err == errNotFound {
-				continue
-			}
-			return err
-		}
-		if w.metadataProvider != nil {
-			w.loc = loc
-			return nil
-		}
-	}
-
-	// Read from security.txt
-
-	path := hd + "/.well-known/security.txt"
-	res, err := w.client.Get(path)
-	if err != nil {
-		return err
-	}
+	lpmd := csaf.LoadProviderMetadataForDomain(
+		w.client, domain, func(format string, args ...interface{}) {
+			log.Printf(
+				"Looking for provider-metadata.json of '"+domain+"': "+format+"\n", args...)
+		})
 
-	if res.StatusCode != http.StatusOK {
-		return errNotFound
+	if lpmd == nil {
+		return fmt.Errorf("no provider-metadata.json found for '%s'", domain)
 	}
 
-	if err := func() error {
-		defer res.Body.Close()
-		urls, err := csaf.ExtractProviderURL(res.Body, false)
-		if err != nil {
-			return err
-		}
-		if len(urls) == 0 {
-			return errors.New("no provider-metadata.json found in secturity.txt")
-		}
-		w.loc = urls[0]
-		return nil
-	}(); err != nil {
-		return err
-	}
+	w.metadataProvider = lpmd.Document
+	w.loc = lpmd.URL
 
-	return downloadJSON(w.client, w.loc, download)
+	return nil
 }
 
 // removeOrphans removes the directories that are not in the providers list.

diff --git a/cmd/csaf_checker/processor.go b/cmd/csaf_checker/processor.go
@@ -795,111 +795,6 @@ func (p *processor) checkListing(string) error {
 	return nil
 }
 
-var providerMetadataLocations = [...]string{
-	".well-known/csaf",
-	"security/data/csaf",
-	"advisories/csaf",
-	"security/csaf",
-}
-
-// locateProviderMetadata searches for provider-metadata.json at various
-// locations mentioned in "7.1.7 Requirement 7: provider-metadata.json".
-func (p *processor) locateProviderMetadata(
-	domain string,
-	found func(string, io.Reader) error,
-) error {
-
-	client := p.httpClient()
-	tryURL := func(url string) (bool, error) {
-		log.Printf("Trying: %v\n", url)
-		res, err := client.Get(url)
-
-		if err != nil || res.StatusCode != http.StatusOK ||
-			res.Header.Get("Content-Type") != "application/json" {
-			// ignore this as it is expected.
-			return false, nil
-		}
-
-		if err := func() error {
-			defer res.Body.Close()
-			return found(url, res.Body)
-		}(); err != nil {
-			return false, err
-		}
-		return true, nil
-	}
-
-	for _, loc := range providerMetadataLocations {
-		url := "https://" + domain + "/" + loc + "/provider-metadata.json"
-		ok, err := tryURL(url)
-		if err != nil {
-			if err == errContinue {
-				continue
-			}
-			return err
-		}
-		if ok {
-			return nil
-		}
-	}
-
-	// Read from security.txt
-
-	path := "https://" + domain + "/.well-known/security.txt"
-	log.Printf("Searching in: %v\n", path)
-	res, err := client.Get(path)
-	if err == nil && res.StatusCode == http.StatusOK {
-		loc, err := func() (string, error) {
-			defer res.Body.Close()
-			return p.extractProviderURL(res.Body)
-		}()
-
-		if err != nil {
-			log.Printf("did not find provider URL in /.well-known/security.txt, error: %v\n", err)
-		}
-
-		if loc != "" {
-			if _, err = tryURL(loc); err == errContinue {
-				err = nil
-			}
-			return err
-		}
-	}
-
-	// Read from DNS path
-
-	path = "https://csaf.data.security." + domain
-	ok, err := tryURL(path)
-	if err != nil {
-		return err
-	}
-	if ok {
-		return nil
-	}
-
-	return errStop
-}
-
-func (p *processor) extractProviderURL(r io.Reader) (string, error) {
-	urls, err := csaf.ExtractProviderURL(r, true)
-	if err != nil {
-		return "", err
-	}
-	if len(urls) == 0 {
-		return "", errors.New("no provider-metadata.json found")
-	}
-
-	if len(urls) > 1 {
-		p.badSecurity.use()
-		p.badSecurity.add("Found %d CSAF entries in security.txt", len(urls))
-	}
-	if !strings.HasPrefix(urls[0], "https://") {
-		p.badSecurity.use()
-		p.badSecurity.add("CSAF URL does not start with https://: %s", urls[0])
-	}
-	return urls[0], nil
-}
-
 // checkProviderMetadata checks provider-metadata.json. If it exists,
 // decodes, and validates against the JSON schema.
 // According to the result, the respective error messages added to
@@ -909,44 +804,20 @@ func (p *processor) checkProviderMetadata(domain string) error {
 
 	p.badProviderMetadata.use()
 
-	found := func(url string, content io.Reader) error {
-
-		// Calculate checksum for later comparison.
-		hash := sha256.New()
-
-		tee := io.TeeReader(content, hash)
-		if err := json.NewDecoder(tee).Decode(&p.pmd); err != nil {
-			p.badProviderMetadata.add("%s: Decoding JSON failed: %v", url, err)
-			return errContinue
-		}
-
-		p.pmd256 = hash.Sum(nil)
-
-		errors, err := csaf.ValidateProviderMetadata(p.pmd)
-		if err != nil {
-			return err
-		}
-		if len(errors) > 0 {
-			p.badProviderMetadata.add("%s: Validating against JSON schema failed:", url)
-			for _, msg := range errors {
-				p.badProviderMetadata.add(strings.ReplaceAll(msg, `%`, `%%`))
-			}
-			p.badProviderMetadata.add("STOPPING here - cannot perform other checks.")
-			return errStop
-		}
-		p.pmdURL = url
-		return nil
-	}
+	client := p.httpClient()
 
-	if err := p.locateProviderMetadata(domain, found); err != nil {
-		return err
-	}
+	lpmd := csaf.LoadProviderMetadataForDomain(client, domain, p.badProviderMetadata.add)
 
-	if p.pmdURL == "" {
-		p.badProviderMetadata.add("No provider-metadata.json found.")
+	if lpmd == nil {
+		p.badProviderMetadata.add("No valid provider-metadata.json found.")
 		p.badProviderMetadata.add("STOPPING here - cannot perform other checks.")
 		return errStop
 	}
+
+	p.pmdURL = lpmd.URL
+	p.pmd256 = lpmd.Hash
+	p.pmd = lpmd.Document
+
 	return nil
 }