From 4cc5e1a82864a0a02aec1534501a59ff254cceb3 Mon Sep 17 00:00:00 2001 From: Alexander Waldenmaier <70207700+alexw9988@users.noreply.github.com> Date: Tue, 23 Mar 2021 16:09:18 +0100 Subject: [PATCH] Add insecure option, include bucket name in AWS credentials, best-effort+zoom flag (#18) * implement minZ maxZ logic * add tiles_c * revert image.go changes * actually loop sources backwards and reimplement opaque check * actually loop sources backwards and reimplement opaque check * insecure https * add insecure option * include bucketname in AWS credentials env variables * fix zoom level mismatch skip in combination with bestEffort flag Co-authored-by: Alexander Waldenmaier --- S3Backend/s3backend.go | 13 ++++++++----- main.go | 13 ++++++++----- tileset.go | 12 ++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/S3Backend/s3backend.go b/S3Backend/s3backend.go index 5fa7fed..a0c4673 100644 --- a/S3Backend/s3backend.go +++ b/S3Backend/s3backend.go @@ -3,10 +3,11 @@ package S3Backend import ( "bytes" "context" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "os" "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) type S3Backend struct { @@ -15,12 +16,14 @@ type S3Backend struct { BasePath string } -func NewS3Backend(path string) (*S3Backend, error) { +func NewS3Backend(path string, insecure bool) (*S3Backend, error) { pathComponents := strings.Split(path[5:], "/") + accessKey := os.Getenv(pathComponents[0] + "_" + pathComponents[1] + "_ACCESS_KEY_ID") + secretKey := os.Getenv(pathComponents[0] + "_" + pathComponents[1] + "_SECRET_ACCESS_KEY") minioClient, err := minio.New(pathComponents[0], &minio.Options{ - Creds: credentials.NewStaticV4(os.Getenv(pathComponents[0]+"_ACCESS_KEY_ID"), os.Getenv(pathComponents[0]+"_SECRET_ACCESS_KEY"), ""), - Secure: true, + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: !insecure, }) if err != nil { diff --git a/main.go b/main.go index 2ec1b61..d767f25 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,8 @@ func atomicAverage(target *int64, channel *chan time.Duration) { } } +var insecure bool + func main() { numWorkers := flag.Int("parallel", 2, "Number of parallel threads to use for processing") quiet := flag.Bool("quiet", false, "Don't output progress information") @@ -49,6 +51,7 @@ func main() { report := flag.Bool("report", false, "Enable periodic reports (every min); intended for non-interactive environments") bestEffort := flag.Bool("best-effort", false, "Best-effort merging: ignore erroneous tilesets completely and silently skip single failed tiles.") zoom := flag.String("zoom", "", "Restrict/manually set zoom levels to work on, in the form of 'minZ-maxZ' (e.g. '1-8'). If this option is specified, prioritile does not try to automatically detect the zoom levels of the target but rather uses these hardcoded ones.") + insecure = *flag.Bool("insecure", false, "Do not use https for remote sources/target") flag.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: prioritile [-zoom '1-8'] [-debug] [-report] [-best-effort] [-parallel=2] /tiles/target/ /tiles/source1/ [s3://foo/tiles/source2/ [...]]") fmt.Fprintln(os.Stderr, "") @@ -63,9 +66,9 @@ func main() { fmt.Fprintln(os.Stderr, "- Resolution of corresponding tiles matches") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "S3 disk backends are supported as source and target by prefixing the tile") - fmt.Fprintln(os.Stderr, "directories with 's3://', e.g. 's3://example.com/source/'.") - fmt.Fprintln(os.Stderr, "S3 authentication information is read from environment variables prefixed with the target hostname:") - fmt.Fprintln(os.Stderr, "example.com_ACCESS_KEY_ID, example.com_SECRET_ACCESS_KEY") + fmt.Fprintln(os.Stderr, "directories with 's3://', e.g. 's3://example.com/foobucket/'.") + fmt.Fprintln(os.Stderr, "S3 authentication information is read from environment variables prefixed with the target hostname and bucketname:") + fmt.Fprintln(os.Stderr, "example.com_foobucket_ACCESS_KEY_ID, example.com_foobucket_SECRET_ACCESS_KEY") fmt.Fprintln(os.Stderr, "") flag.PrintDefaults() } @@ -111,7 +114,7 @@ func main() { } } - sources, errs := discoverTilesets(flag.Args()[1:], target.MinZ, target.MaxZ) + sources, errs := discoverTilesets(flag.Args()[1:], target, *bestEffort) if errs != nil && !*bestEffort { log.Fatalf("could not discover tilesets: %v", errs) } @@ -312,7 +315,7 @@ func main() { func stringToBackend(pathSpec string) (StorageBackend, error) { if strings.HasPrefix(pathSpec, "s3://") { - backend, err := S3Backend.NewS3Backend(pathSpec) + backend, err := S3Backend.NewS3Backend(pathSpec, insecure) if err != nil { return nil, err } diff --git a/tileset.go b/tileset.go index 8a2fb95..a326f52 100644 --- a/tileset.go +++ b/tileset.go @@ -17,12 +17,10 @@ func (t TilesetDescriptor) String() string { return fmt.Sprintf("%d-%d", t.MaxZ, t.MinZ) } -func discoverTilesets(paths []string, minZ int, maxZ int) ([]TilesetDescriptor, []error) { +func discoverTilesets(paths []string, target TilesetDescriptor, bestEffort bool) ([]TilesetDescriptor, []error) { var tilesets []TilesetDescriptor var errors []error - // XXX if discovery for the target tileset fails, the first source might be used as a target, which is somewhat - // undesirable, I believe for _, path := range paths { backend, err := stringToBackend(path) if err != nil { @@ -30,16 +28,18 @@ func discoverTilesets(paths []string, minZ int, maxZ int) ([]TilesetDescriptor, continue } - tileset, err := discoverTileset(backend, minZ, maxZ) + tileset, err := discoverTileset(backend, target.MinZ, target.MaxZ) if err != nil { errors = append(errors, fmt.Errorf("could not discover tileset: %v in %s", err, path)) continue } - if len(tilesets) > 0 && (tilesets[0].MaxZ != tileset.MaxZ || tilesets[0].MinZ != tileset.MinZ) { + if len(tilesets) > 0 && (target.MaxZ != tileset.MaxZ || target.MinZ != tileset.MinZ) { errors = append(errors, fmt.Errorf("zoom level mismatch for target and source %s", path)) - continue + if !bestEffort { + continue + } } tilesets = append(tilesets, tileset) }