Skip to content

Commit

Permalink
Add concurrency control parameters (#144)
Browse files Browse the repository at this point in the history 
Adds some concurrency control parameters which can be useful to tweaking performance/resource usage and dealing with long-running queries.
  • Loading branch information
@ccampo133
ccampo133 authored Aug 28, 2024
1 parent f616f1c commit 087671d
Show file tree
Hide file tree
Showing 6 changed files with 257 additions and 130 deletions.
51 changes: 29 additions & 22 deletions cmd/dmap/repo_scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"reflect"
"strings"
"time"

"github.com/alecthomas/kong"
"github.com/gobwas/glob"
Expand All @@ -15,21 +16,24 @@ import (
)

type RepoScanCmd struct {
Type string `help:"Type of repository to connect to (postgres|mysql|oracle|sqlserver|snowflake|redshift|denodo)." enum:"postgres,mysql,oracle,sqlserver,snowflake,redshift,denodo" required:""`
Host string `help:"Hostname of the repository." required:""`
Port uint16 `help:"Port of the repository." required:""`
User string `help:"Username to connect to the repository." required:""`
Password string `help:"Password to connect to the repository." required:""`
RepoID string `help:"The ID of the repository used by the Dmap service to identify the data repository. For RDS or Redshift, this is the ARN of the database. Optional, but required to publish the scan results Dmap service."`
Database string `help:"Name of the database to connect to. If not specified, the default database is used (if possible)."`
Advanced map[string]any `help:"Advanced configuration for the repository, semicolon separated (e.g. key1=value1;key2=value2). Please see the documentation for details on how to provide this argument for specific repository types."`
IncludePaths GlobFlag `help:"List of glob patterns to include when introspecting the database(s), semicolon separated (e.g. foo*;bar*;*.baz)." default:"*"`
ExcludePaths GlobFlag `help:"List of glob patterns to exclude when introspecting the database(s), semicolon separated (e.g. foo*;bar*;*.baz)."`
MaxOpenConns uint `help:"Maximum number of open connections to the database." default:"10"`
SampleSize uint `help:"Number of rows to sample from the repository (per table)." default:"5"`
Offset uint `help:"Offset to start sampling each table from." default:"0"`
LabelYamlFile string `help:"Filename of the yaml file containing the custom set of data labels (e.g. /path/to/labels.yaml). If omitted, a set of predefined labels is used."`
Silent bool `help:"Do not print the results to stdout." short:"s"`
Type string `help:"Type of repository to connect to (postgres|mysql|oracle|sqlserver|snowflake|redshift|denodo)." enum:"postgres,mysql,oracle,sqlserver,snowflake,redshift,denodo" required:""`
Host string `help:"Hostname of the repository." required:""`
Port uint16 `help:"Port of the repository." required:""`
User string `help:"Username to connect to the repository." required:""`
Password string `help:"Password to connect to the repository." required:""`
RepoID string `help:"The ID of the repository used by the Dmap service to identify the data repository. For RDS or Redshift, this is the ARN of the database. Optional, but required to publish the scan results Dmap service."`
Database string `help:"Name of the database to connect to. If not specified, the default database is used (if possible)."`
Advanced map[string]any `help:"Advanced configuration for the repository, semicolon separated (e.g. key1=value1;key2=value2). Please see the documentation for details on how to provide this argument for specific repository types."`
IncludePaths GlobFlag `help:"List of glob patterns to include when introspecting the database(s), semicolon separated (e.g. foo*;bar*;*.baz)." default:"*"`
ExcludePaths GlobFlag `help:"List of glob patterns to exclude when introspecting the database(s), semicolon separated (e.g. foo*;bar*;*.baz)."`
MaxOpenConns uint `help:"Maximum number of open connections to the database." default:"10"`
MaxParallelDbs uint `help:"Maximum number of parallel databases scanned at once. If zero, there is no limit." default:"0"`
MaxConcurrency uint `help:"Maximum number of concurrent query goroutines. If zero, there is no limit." default:"0"`
QueryTimeout time.Duration `help:"Maximum time a query can run before being cancelled. If zero, there is no timeout." default:"0s"`
SampleSize uint `help:"Number of rows to sample from the repository (per table)." default:"5"`
Offset uint `help:"Offset to start sampling each table from." default:"0"`
LabelYamlFile string `help:"Filename of the yaml file containing the custom set of data labels (e.g. /path/to/labels.yaml). If omitted, a set of predefined labels is used."`
Silent bool `help:"Do not print the results to stdout." short:"s"`
}

func (cmd *RepoScanCmd) Validate() error {
Expand Down Expand Up @@ -69,13 +73,16 @@ func (cmd *RepoScanCmd) Run(globals *Globals) error {
cfg := sql.ScannerConfig{
RepoType: cmd.Type,
RepoConfig: sql.RepoConfig{
Host: cmd.Host,
Port: cmd.Port,
User: cmd.User,
Password: cmd.Password,
Database: cmd.Database,
MaxOpenConns: cmd.MaxOpenConns,
Advanced: cmd.Advanced,
Host: cmd.Host,
Port: cmd.Port,
User: cmd.User,
Password: cmd.Password,
Database: cmd.Database,
MaxOpenConns: cmd.MaxOpenConns,
MaxParallelDbs: cmd.MaxParallelDbs,
MaxConcurrency: cmd.MaxConcurrency,
QueryTimeout: cmd.QueryTimeout,
Advanced: cmd.Advanced,
},
IncludePaths: cmd.IncludePaths,
ExcludePaths: cmd.ExcludePaths,
Expand Down
38 changes: 19 additions & 19 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,16 @@ require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/99designs/keyring v1.2.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/apache/arrow/go/v15 v15.0.2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
Expand All @@ -58,7 +58,7 @@ require (
github.com/danieljoos/wincred v1.2.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand All @@ -79,7 +79,7 @@ require (
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_golang v1.20.2 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
Expand All @@ -90,19 +90,19 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/yashtewari/glob-intersection v0.2.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.23.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/sdk v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/term v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
Expand Down
Loading

0 comments on commit 087671d

Please sign in to comment.