Add --scan-target option for downsync, get and unpack (#188)

* Rename regex include/exclude names for clarity
* Add option for enabling/disabling scanning of target
* release notes

.github/workflows/create-release.yml

@@ -125,6 +125,7 @@ jobs:
         release_name: Release ${{ github.ref }}
         body: |
           Changes in this Release
+          - **ADDED** `--scan-target` option for `downsync`, `get` and `unpack`. Default is on, disable with `--no-scan-target`
           - **UPDATED** Updated longtail to v0.2.9
         draft: false
         prerelease: false

commands/cmd_downsync.go

@@ -23,7 +23,8 @@ func downsync(
 	validate bool,
 	versionLocalStoreIndexPath string,
 	includeFilterRegEx string,
-	excludeFilterRegEx string) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	excludeFilterRegEx string,
+	scanTarget bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
 	const fname = "downsync"
 	log := logrus.WithFields(logrus.Fields{
 		"fname":                      fname,
@@ -38,6 +39,7 @@ func downsync(
 		"versionLocalStoreIndexPath": versionLocalStoreIndexPath,
 		"includeFilterRegEx":         includeFilterRegEx,
 		"excludeFilterRegEx":         excludeFilterRegEx,
+		"scanTarget":                 scanTarget,
 	})
 	log.Debug(fname)
 
@@ -72,7 +74,7 @@ func downsync(
 	defer fs.Dispose()
 
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
-	if targetIndexPath == "" {
+	if scanTarget && targetIndexPath == "" {
 		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs)
 	}
 
@@ -344,8 +346,9 @@ type DownsyncCmd struct {
 	RetainPermissionsOption
 	ValidateTargetOption
 	VersionLocalStoreIndexPathOption
-	DownsyncIncludeRegExOption
-	DownsyncExcludeRegExOption
+	TargetPathIncludeRegExOption
+	TargetPathExcludeRegExOption
+	ScanTargetOption
 }
 
 func (r *DownsyncCmd) Run(ctx *Context) error {
@@ -360,7 +363,8 @@ func (r *DownsyncCmd) Run(ctx *Context) error {
 		r.Validate,
 		r.VersionLocalStoreIndexPath,
 		r.IncludeFilterRegEx,
-		r.ExcludeFilterRegEx)
+		r.ExcludeFilterRegEx,
+		r.ScanTarget)
 	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
 	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
 	return err

commands/cmd_get.go

@@ -20,7 +20,8 @@ func get(
 	retainPermissions bool,
 	validate bool,
 	includeFilterRegEx string,
-	excludeFilterRegEx string) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
+	excludeFilterRegEx string,
+	scanTarget bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
 	const fname = "get"
 	log := logrus.WithFields(logrus.Fields{
 		"fname":              fname,
@@ -33,6 +34,7 @@ func get(
 		"validate":           validate,
 		"includeFilterRegEx": includeFilterRegEx,
 		"excludeFilterRegEx": excludeFilterRegEx,
+		"scanTarget":         scanTarget,
 	})
 	log.Debug(fname)
 
@@ -82,7 +84,8 @@ func get(
 		validate,
 		versionLocalStoreIndexPath,
 		includeFilterRegEx,
-		excludeFilterRegEx)
+		excludeFilterRegEx,
+		scanTarget)
 
 	storeStats = append(storeStats, downSyncStoreStats...)
 	timeStats = append(timeStats, downSyncTimeStats...)
@@ -98,8 +101,9 @@ type GetCmd struct {
 	VersionLocalStoreIndexPathOption
 	CachePathOption
 	RetainPermissionsOption
-	DownsyncIncludeRegExOption
-	DownsyncExcludeRegExOption
+	TargetPathIncludeRegExOption
+	TargetPathExcludeRegExOption
+	ScanTargetOption
 }
 
 func (r *GetCmd) Run(ctx *Context) error {
@@ -112,7 +116,8 @@ func (r *GetCmd) Run(ctx *Context) error {
 		r.RetainPermissions,
 		r.Validate,
 		r.IncludeFilterRegEx,
-		r.ExcludeFilterRegEx)
+		r.ExcludeFilterRegEx,
+		r.ScanTarget)
 	ctx.StoreStats = append(ctx.StoreStats, storeStats...)
 	ctx.TimeStats = append(ctx.TimeStats, timeStats...)
 	return err

commands/cmd_pack.go

@@ -198,8 +198,8 @@ type PackCmd struct {
 	TargetBlockSizeOption
 	CompressionOption
 	HashingOption
-	UpsyncIncludeRegExOption
-	UpsyncExcludeRegExOption
+	SourcePathIncludeRegExOption
+	SourcePathExcludeRegExOption
 }
 
 func (r *PackCmd) Run(ctx *Context) error {

commands/cmd_unpack.go

@@ -19,6 +19,7 @@ func unpack(
 	targetIndexPath string,
 	includeFilterRegEx string,
 	excludeFilterRegEx string,
+	scanTarget bool,
 	retainPermissions bool,
 	validate bool) ([]longtailutils.StoreStat, []longtailutils.TimeStat, error) {
 	const fname = "unpack"
@@ -30,7 +31,9 @@ func unpack(
 		"targetIndexPath":    targetIndexPath,
 		"includeFilterRegEx": includeFilterRegEx,
 		"excludeFilterRegEx": excludeFilterRegEx,
+		"scanTarget":         scanTarget,
 		"retainPermissions":  retainPermissions,
+		"validate":           validate,
 	})
 	log.Debug(fname)
 
@@ -65,7 +68,7 @@ func unpack(
 	defer fs.Dispose()
 
 	targetFolderScanner := longtailutils.AsyncFolderScanner{}
-	if targetIndexPath == "" {
+	if scanTarget && targetIndexPath == "" {
 		targetFolderScanner.Scan(resolvedTargetFolderPath, pathFilter, fs)
 	}
 
@@ -285,10 +288,11 @@ type UnpackCmd struct {
 	SourcePath      string `name:"source-path" help:"Source folder path" required:""`
 	TargetPath      string `name:"target-path" help:"Target file uri"`
 	TargetIndexPath string `name:"target-index-path" help:"Optional pre-computed index of target-path"`
+	TargetPathIncludeRegExOption
+	TargetPathExcludeRegExOption
+	ScanTargetOption
 	RetainPermissionsOption
 	ValidateTargetOption
-	DownsyncIncludeRegExOption
-	DownsyncExcludeRegExOption
 }
 
 func (r *UnpackCmd) Run(ctx *Context) error {
@@ -299,6 +303,7 @@ func (r *UnpackCmd) Run(ctx *Context) error {
 		r.TargetIndexPath,
 		r.IncludeFilterRegEx,
 		r.ExcludeFilterRegEx,
+		r.ScanTarget,
 		r.RetainPermissions,
 		r.Validate)
 	ctx.StoreStats = append(ctx.StoreStats, storeStats...)

commands/cmd_upsync.go

@@ -271,8 +271,8 @@ type UpsyncCmd struct {
 	StorageURIOption
 	CompressionOption
 	HashingOption
-	UpsyncIncludeRegExOption
-	UpsyncExcludeRegExOption
+	SourcePathIncludeRegExOption
+	SourcePathExcludeRegExOption
 }
 
 func (r *UpsyncCmd) Run(ctx *Context) error {

commands/options.go

@@ -16,19 +16,19 @@ type HashingOption struct {
 	Hashing string `name:"hash-algorithm" help:"Hash algorithm [meow blake2 blake3]" enum:"meow,blake2,blake3" default:"blake3"`
 }
 
-type UpsyncIncludeRegExOption struct {
+type SourcePathIncludeRegExOption struct {
 	IncludeFilterRegEx string `name:"include-filter-regex" help:"Optional include regex filter for assets in --source-path. Separate regexes with **"`
 }
 
-type DownsyncIncludeRegExOption struct {
+type TargetPathIncludeRegExOption struct {
 	IncludeFilterRegEx string `name:"include-filter-regex" help:"Optional include regex filter for assets in --target-path on downsync. Separate regexes with **"`
 }
 
-type UpsyncExcludeRegExOption struct {
+type SourcePathExcludeRegExOption struct {
 	ExcludeFilterRegEx string `name:"exclude-filter-regex" help:"Optional exclude regex filter for assets in --source-path on upsync. Separate regexes with **"`
 }
 
-type DownsyncExcludeRegExOption struct {
+type TargetPathExcludeRegExOption struct {
 	ExcludeFilterRegEx string `name:"exclude-filter-regex" help:"Optional exclude regex filter for assets in --target-path on downsync. Separate regexes with **"`
 }
 
@@ -91,3 +91,7 @@ type MaxChunksPerBlockOption struct {
 type TargetBlockSizeOption struct {
 	TargetBlockSize uint32 `name:"target-block-size" help:"Target block size" default:"8388608"`
 }
+
+type ScanTargetOption struct {
+	ScanTarget bool `name:"scan-target" help:"Enables scanning of target folder before write. Disable it to only add/write content to a folder" default:"true" negatable:""`
+}