Skip to content

Commit

Permalink
Move downloader to lib/downloader
Browse files Browse the repository at this point in the history
  • Loading branch information
@koplas
koplas committed Jun 18, 2024
1 parent d909e9d commit 3bfb421
Show file tree
Hide file tree
Showing 7 changed files with 175 additions and 170 deletions.
80 changes: 70 additions & 10 deletions cmd/csaf_downloader/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ package main

import (
"context"
"github.com/csaf-poc/csaf_distribution/v3/lib/downloader"
"os"
"os/signal"

Expand All @@ -19,36 +20,95 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)

func run(cfg *config, domains []string) error {
d, err := newDownloader(cfg)
const (
defaultWorker = 2
defaultPreset = "mandatory"
defaultForwardQueue = 5
defaultValidationMode = downloader.ValidationStrict
defaultLogFile = "downloader.log"
defaultLogLevel = slog.LevelInfo
)

// configPaths are the potential file locations of the Config file.
var configPaths = []string{
"~/.config/csaf/downloader.toml",
"~/.csaf_downloader.toml",
"csaf_downloader.toml",
}

// parseArgsConfig parses the command line and if needed a config file.
func parseArgsConfig() ([]string, *downloader.Config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[downloader.Config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *downloader.Config) string { return cfg.Config },
Usage: "[OPTIONS] domain...",
HasVersion: func(cfg *downloader.Config) bool { return cfg.Version },
SetDefaults: func(cfg *downloader.Config) {
cfg.Worker = defaultWorker
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
cfg.ForwardQueue = defaultForwardQueue
cfg.LogFile = &logFile
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *downloader.Config) {
if cfg.Worker == 0 {
cfg.Worker = defaultWorker
}
if cfg.RemoteValidatorPresets == nil {
cfg.RemoteValidatorPresets = []string{defaultPreset}
}
switch cfg.ValidationMode {
case downloader.ValidationStrict, downloader.ValidationUnsafe:
default:
cfg.ValidationMode = downloader.ValidationStrict
}
if cfg.LogFile == nil {
cfg.LogFile = &logFile
}
if cfg.LogLevel == nil {
cfg.LogLevel = logLevel
}
},
}
return p.Parse()
}

func run(cfg *downloader.Config, domains []string) error {
d, err := downloader.NewDownloader(cfg)
if err != nil {
return err
}
defer d.close()
defer d.Close()

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ctx, stop := signal.NotifyContext(ctx, os.Interrupt)
defer stop()

if cfg.ForwardURL != "" {
f := newForwarder(cfg)
go f.run()
f := downloader.NewForwarder(cfg)
go f.Run()
defer func() {
f.log()
f.close()
f.Log()
f.Close()
}()
d.forwarder = f
d.Forwarder = f
}

return d.run(ctx, domains)
return d.Run(ctx, domains)
}

func main() {

domains, cfg, err := parseArgsConfig()
options.ErrorCheck(err)
options.ErrorCheck(cfg.prepare())
options.ErrorCheck(cfg.Prepare())

if len(domains) == 0 {
slog.Warn("No domains given.")
Expand Down
108 changes: 25 additions & 83 deletions cmd/csaf_downloader/config.go → lib/downloader/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
// SPDX-FileCopyrightText: 2022 German Federal Office for Information Security (BSI) <https://www.bsi.bund.de>
// Software-Engineering: 2022 Intevation GmbH <https://intevation.de>

package main
package downloader

import (
"crypto/tls"
Expand All @@ -26,23 +26,15 @@ import (
"github.com/csaf-poc/csaf_distribution/v3/internal/options"
)

const (
defaultWorker = 2
defaultPreset = "mandatory"
defaultForwardQueue = 5
defaultValidationMode = validationStrict
defaultLogFile = "downloader.log"
defaultLogLevel = slog.LevelInfo
)

type validationMode string
type ValidationMode string

Check failure on line 29 in lib/downloader/config.go

View workflow job for this annotation

GitHub Actions / build 

exported type ValidationMode should have comment or be unexported

const (
validationStrict = validationMode("strict")
validationUnsafe = validationMode("unsafe")
ValidationStrict = ValidationMode("strict")

Check failure on line 32 in lib/downloader/config.go

View workflow job for this annotation

GitHub Actions / build 

exported const ValidationStrict should have comment (or a comment on this block) or be unexported
ValidationUnsafe = ValidationMode("unsafe")
)

type config struct {
// Config provides the download configuration.
type Config struct {
Directory string `short:"d" long:"directory" description:"DIRectory to store the downloaded files in" value-name:"DIR" toml:"directory"`
Insecure bool `long:"insecure" description:"Do not check TLS certificates from provider" toml:"insecure"`
IgnoreSignatureCheck bool `long:"ignore_sigcheck" description:"Ignore signature check results, just warn on mismatch" toml:"ignore_sigcheck"`
Expand All @@ -63,7 +55,7 @@ type config struct {
RemoteValidatorPresets []string `long:"validator_preset" description:"One or more PRESETS to validate remotely" value-name:"PRESETS" toml:"validator_preset"`

//lint:ignore SA5008 We are using choice twice: strict, unsafe.
ValidationMode validationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"`
ValidationMode ValidationMode `long:"validation_mode" short:"m" choice:"strict" choice:"unsafe" value-name:"MODE" description:"MODE how strict the validation is" toml:"validation_mode"`

ForwardURL string `long:"forward_url" description:"URL of HTTP endpoint to forward downloads to" value-name:"URL" toml:"forward_url"`
ForwardHeader http.Header `long:"forward_header" description:"One or more extra HTTP header fields used by forwarding" toml:"forward_header"`
Expand All @@ -80,60 +72,10 @@ type config struct {
ignorePattern filter.PatternMatcher
}

// configPaths are the potential file locations of the config file.
var configPaths = []string{
"~/.config/csaf/downloader.toml",
"~/.csaf_downloader.toml",
"csaf_downloader.toml",
}

// parseArgsConfig parses the command line and if need a config file.
func parseArgsConfig() ([]string, *config, error) {
var (
logFile = defaultLogFile
logLevel = &options.LogLevel{Level: defaultLogLevel}
)
p := options.Parser[config]{
DefaultConfigLocations: configPaths,
ConfigLocation: func(cfg *config) string { return cfg.Config },
Usage: "[OPTIONS] domain...",
HasVersion: func(cfg *config) bool { return cfg.Version },
SetDefaults: func(cfg *config) {
cfg.Worker = defaultWorker
cfg.RemoteValidatorPresets = []string{defaultPreset}
cfg.ValidationMode = defaultValidationMode
cfg.ForwardQueue = defaultForwardQueue
cfg.LogFile = &logFile
cfg.LogLevel = logLevel
},
// Re-establish default values if not set.
EnsureDefaults: func(cfg *config) {
if cfg.Worker == 0 {
cfg.Worker = defaultWorker
}
if cfg.RemoteValidatorPresets == nil {
cfg.RemoteValidatorPresets = []string{defaultPreset}
}
switch cfg.ValidationMode {
case validationStrict, validationUnsafe:
default:
cfg.ValidationMode = validationStrict
}
if cfg.LogFile == nil {
cfg.LogFile = &logFile
}
if cfg.LogLevel == nil {
cfg.LogLevel = logLevel
}
},
}
return p.Parse()
}

// UnmarshalText implements [encoding.TextUnmarshaler].
func (vm *validationMode) UnmarshalText(text []byte) error {
switch m := validationMode(text); m {
case validationStrict, validationUnsafe:
func (vm *ValidationMode) UnmarshalText(text []byte) error {
switch m := ValidationMode(text); m {
case ValidationStrict, ValidationUnsafe:
*vm = m
default:
return fmt.Errorf(`invalid value %q (expected "strict" or "unsafe)"`, m)
Expand All @@ -142,8 +84,8 @@ func (vm *validationMode) UnmarshalText(text []byte) error {
}

// UnmarshalFlag implements [flags.UnmarshalFlag].
func (vm *validationMode) UnmarshalFlag(value string) error {
var v validationMode
func (vm *ValidationMode) UnmarshalFlag(value string) error {
var v ValidationMode
if err := v.UnmarshalText([]byte(value)); err != nil {
return err
}
Expand All @@ -152,18 +94,18 @@ func (vm *validationMode) UnmarshalFlag(value string) error {
}

// ignoreFile returns true if the given URL should not be downloaded.
func (cfg *config) ignoreURL(u string) bool {
func (cfg *Config) ignoreURL(u string) bool {
return cfg.ignorePattern.Matches(u)
}

// verbose is considered a log level equal or less debug.
func (cfg *config) verbose() bool {
func (cfg *Config) verbose() bool {
return cfg.LogLevel.Level <= slog.LevelDebug
}

// prepareDirectory ensures that the working directory
// exists and is setup properly.
func (cfg *config) prepareDirectory() error {
func (cfg *Config) prepareDirectory() error {
// If not given use current working directory.
if cfg.Directory == "" {
dir, err := os.Getwd()
Expand Down Expand Up @@ -197,7 +139,7 @@ func dropSubSeconds(_ []string, a slog.Attr) slog.Attr {
}

// prepareLogging sets up the structured logging.
func (cfg *config) prepareLogging() error {
func (cfg *Config) prepareLogging() error {
var w io.Writer
if cfg.LogFile == nil || *cfg.LogFile == "" {
log.Println("using STDERR for logging")
Expand Down Expand Up @@ -230,7 +172,7 @@ func (cfg *config) prepareLogging() error {
}

// compileIgnorePatterns compiles the configure patterns to be ignored.
func (cfg *config) compileIgnorePatterns() error {
func (cfg *Config) compileIgnorePatterns() error {
pm, err := filter.NewPatternMatcher(cfg.IgnorePattern)
if err != nil {
return err
Expand All @@ -240,7 +182,7 @@ func (cfg *config) compileIgnorePatterns() error {
}

// prepareCertificates loads the client side certificates used by the HTTP client.
func (cfg *config) prepareCertificates() error {
func (cfg *Config) prepareCertificates() error {
cert, err := certs.LoadCertificate(
cfg.ClientCert, cfg.ClientKey, cfg.ClientPassphrase)
if err != nil {
Expand All @@ -250,13 +192,13 @@ func (cfg *config) prepareCertificates() error {
return nil
}

// prepare prepares internal state of a loaded configuration.
func (cfg *config) prepare() error {
for _, prepare := range []func(*config) error{
(*config).prepareDirectory,
(*config).prepareLogging,
(*config).prepareCertificates,
(*config).compileIgnorePatterns,
// Prepare prepares internal state of a loaded configuration.
func (cfg *Config) Prepare() error {
for _, prepare := range []func(*Config) error{
(*Config).prepareDirectory,
(*Config).prepareLogging,
(*Config).prepareCertificates,
(*Config).compileIgnorePatterns,
} {
if err := prepare(cfg); err != nil {
return err
Expand Down
Loading

0 comments on commit 3bfb421

Please sign in to comment.