diff --git a/NEWS.rst b/NEWS.rst index 01aa69c52..bc4c6e324 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -19,6 +19,8 @@ bundle_adjust (:numref:`bundle_adjust`): parallel_stereo (:numref:`parallel_stereo`): * Added Kaguya processing example (:numref:`kaguya_tc`). + * When a run finished successfully, combine the data from subdirectories and + delete these. See ``--keep-only`` for more options. * Fixed a failure when processing images that have very large blocks (on the order of several tens of thousands of pixels along some dimension, as shown by ``gdalinfo``). A warning, progress bar, and timing info is displayed. diff --git a/docs/tools/parallel_stereo.rst b/docs/tools/parallel_stereo.rst index f673aa2de..d5661a22c 100644 --- a/docs/tools/parallel_stereo.rst +++ b/docs/tools/parallel_stereo.rst @@ -28,10 +28,6 @@ virtual mosaics of files created by individual processes, with the actual files in subdirectories; ASP and GDAL tools are able to use these virtual files in the same way as regular binary TIF files. -The option ``--keep-only`` may be essential for large runs. It will -condense a run by converting VRT files to TIF, and will remove a lot -of auxiliary files. - See :numref:`pbs_slurm` for how to set up this tool for PBS and SLURM systems. @@ -231,19 +227,16 @@ Command-line options windows, as that would invalidate the run. See :numref:`bathy_reuse_run` for an example. ---keep-only - Keep only files with these suffixes in the output prefix - directory. Files that are internally in VRT format will be - converted to TIF. All subdirectories will be deleted. Will be - invoked after triangulation, if that step is reached. - Specify as a string in quotes. Example (this will keep - only the files needed to re-create ``PC.tif`` and this file as - well): ``'.exr L.tif F.tif PC.tif'``. To delete ``PC.tif`` - after it is not needed, such as after DEM generation, reinvoke - this program with ``--entry-point 6`` and do not mention ``PC.tif`` - as a file to keep, but mention ``DEM.tif``, ``IntersectionErr.tif``, - etc. - +--keep-only + If set to ``all_combined``, which is the default, at the end of a + successful run combine the results from subdirectories into ``.tif`` + files with the given output prefix, and delete the + subdirectories. If set to ``essential``, keep only ``PC.tif`` and the + files needed to recreate it (those ending with ``.exr``, ``-L.tif``, + ``-F.tif``). If set to ``unchanged``, keep the run directory as it + is. For fine-grained control, specify a quoted list of suffixes of + files to keep, such as ``".exr .match -L.tif -PC.tif"``. + --verbose Display the commands being executed. diff --git a/src/asp/Camera/LinescanASTERModel.h b/src/asp/Camera/LinescanASTERModel.h index edb0880af..b5fba9be5 100644 --- a/src/asp/Camera/LinescanASTERModel.h +++ b/src/asp/Camera/LinescanASTERModel.h @@ -38,6 +38,8 @@ namespace asp { // ASTER User Handbook Version 2 // https://asterweb.jpl.nasa.gov/content/03_data/04_Documents/aster_user_guide_v2.pdf + // https://lpdaac.usgs.gov/documents/175/ASTER_L1_Product_Specifications.pdf + // The following paper has a blurb on the ASTER linescan camera. // Quantifying ice loss in the eastern Himalayas since 1974 using // declassified spy satellite imagery diff --git a/src/asp/Camera/RPCModel.cc b/src/asp/Camera/RPCModel.cc index 7e41cd0e4..342c69df7 100644 --- a/src/asp/Camera/RPCModel.cc +++ b/src/asp/Camera/RPCModel.cc @@ -501,6 +501,7 @@ namespace asp { invJ[0][1] = -J[0][1]; invJ[1][0] = -J[1][0]; invJ[1][1] = J[0][0]; + // TODO(oalexan1): What if det is close to 0? invJ /= det; // Newton's method for F(x) = y is diff --git a/src/asp/Tools/aster2asp.cc b/src/asp/Tools/aster2asp.cc index a485b53de..d29fa9f0d 100644 --- a/src/asp/Tools/aster2asp.cc +++ b/src/asp/Tools/aster2asp.cc @@ -27,6 +27,8 @@ // ASTER User Handbook Version 2 // https://asterweb.jpl.nasa.gov/content/03_data/04_Documents/aster_user_guide_v2.pdf // +// https://lpdaac.usgs.gov/documents/175/ASTER_L1_Product_Specifications.pdf + // IMPROVEMENT OF DEM GENERATION FROM ASTER IMAGES USING SATELLITE // JITTER ESTIMATION AND OPEN SOURCE IMPLEMENTATION // Luc Girod, Christopher Nutha, and Andreas Kaab diff --git a/src/asp/Tools/parallel_stereo b/src/asp/Tools/parallel_stereo index 2b6f459e3..5f21349c5 100644 --- a/src/asp/Tools/parallel_stereo +++ b/src/asp/Tools/parallel_stereo @@ -142,7 +142,9 @@ def create_subproject_dirs(settings, **kw): # creating symbolic links to actual files in parent run directory. # Don't symlink to RD.tif or PC.tif as those will be files which # actually need to be created in each subdirectory. + # Return the created subdirectories. + dirs = [] out_prefix = settings['out_prefix'][0] # Save the list of subdirectories to disk. This is used in stereo_blend. @@ -158,6 +160,7 @@ def create_subproject_dirs(settings, **kw): for tile in produce_tiles(settings, opt.job_size_w, opt.job_size_h): subproject_dir = tile_dir(out_prefix, tile) + dirs.append(subproject_dir) tile_prefix = subproject_dir + "/" + tile.name_str() if opt.dryrun: print("mkdir -p %s" % subproject_dir) @@ -180,6 +183,8 @@ def create_subproject_dirs(settings, **kw): os.symlink(rel_src, dst_f) fout.close() + + return dirs def rename_files(settings, postfix_in, postfix_out, **kw): """ @@ -297,7 +302,7 @@ def build_vrt(prog, settings, georef, postfix, tile_postfix, contract_tiles=Fals goodFilename = filename break if goodFilename == "": - raise Exception('No tiles were generated') + raise Exception('No tiles were generated.') # Do gdalinfo on the good tile to get metadata args = [goodFilename] @@ -697,32 +702,42 @@ def isVrt(filename): return False -def keepOnlySpecified(keep_only, out_prefix): +def keepOnlySpecified(keep_only, out_prefix, subdirs): """ Merge all VRTs and wipe any files the user does not want to keep. """ - if keep_only is None: - return - - if '/' not in out_prefix: - raise Exception('Cannot use the --keep-only option as the output prefix ' + \ - 'does not produce a subdirectory. This is on purpose, ' + \ - 'to avoid unintended mass deletion of user files.') + + if keep_only == "unchanged": + return # keep everything unchanged + elif keep_only == "essential": + keep_only = ".exr -L.tif -F.tif -PC.tif" + + # List all the files stereo may create. We will not attempt to delete + # any files except these (and subdirs) + all_files = set() + for ext in ".vwip .match cropped.tif -L.tif -R.tif ask.tif .exr sub.tif -D.tif -RD.tif -B.tif -F.tif -PC.tif".split(): + for f in glob.glob(out_prefix + "*" + ext): + all_files.add(f) + # List all the files with the suffixes the user wants to keep keep_files = set() - for keep in keep_only.split(): - for f in glob.glob(out_prefix + "*" + keep): - keep_files.add(f) + if keep_only == "all_combined": + keep_files = all_files.copy() + else: + for keep in keep_only.split(): + for f in glob.glob(out_prefix + "*" + keep): + keep_files.add(f) - all_files = set() - for f in glob.glob(out_prefix + "*"): - all_files.add(f) - + if len(keep_files) == 0: + # Sometimes the user can make a mistake here. If not sure, don't do anything. + print("No files to keep. This is likely a user error. Not deleting any files.") + return + for f in keep_files: print("Keeping: " + f) if isVrt(f): f_out = os.path.splitext(f)[0] + "_merged.tif" - print("Convert " + f + " to .tif format.") + print("Convert " + f + " from VRT to TIF format.") cmd = ['gdal_translate', '-co', 'compress=lzw', '-co', 'TILED=yes', '-co', 'INTERLEAVE=BAND', '-co', 'BLOCKXSIZE=256', '-co', 'BLOCKYSIZE=256', '-co', 'BIGTIFF=IF_SAFER', f, f_out] @@ -733,7 +748,8 @@ def keepOnlySpecified(keep_only, out_prefix): print("Renaming: " + f_out + " to " + f) shutil.move(f_out, f) - for f in all_files: + # Delete the other files and directories + for f in list(all_files) + subdirs: if f not in keep_files: print("Deleting: " + f) if os.path.isdir(f): @@ -802,10 +818,18 @@ if __name__ == '__main__': '--left-image-crop-win, etc, when running this.') p.add_argument('--prev-run-prefix', dest='prev_run_prefix', default=None, help='Start at the triangulation stage while reusing the data from this prefix. The new run can use different cameras, bundle adjustment prefix, or bathy planes (if applicable). Do not change crop windows, as that would invalidate the run.') - p.add_argument('--keep-only', dest='keep_only', - help='Keep only produced files with these suffixes. Files that are internally in VRT format will be converted to TIF. All subdirectories will be deleted. Will be invoked after triangulation, if that step is reached. Specify as a string in quotes. Example (this will keep only the files needed to re-create PC.tif and this file as well): ".exr L.tif F.tif PC.tif".') + p.add_argument('--keep-only', dest='keep_only', default="all_combined", + help='If set to "all_combined", which is the default, at the end of a ' + \ + 'successful run combine the results from subdirectories into .tif ' + \ + 'files with the given output prefix, and delete the ' + \ + 'subdirectories. If set to "essential", keep only PC.tif and the ' + \ + 'files needed to recreate it (those ending with .exr, -L.tif, -F.tif). ' + \ + 'If set to "unchanged", keep the run directory as it ' + \ + 'is. For fine-grained control, specify a quoted list of suffixes of ' + \ + 'files to keep, such as ".exr .match -L.tif -PC.tif".') p.add_argument('--sparse-disp-options', dest='sparse_disp_options', - help='Options to pass directly to sparse_disp. Use quotes around this string.') + help='Options to pass directly to sparse_disp. Use quotes around ' + \ + 'this string.') p.add_argument('-v', '--version', dest='version', default=False, action='store_true', help='Display the version of software.') p.add_argument('-s', '--stereo-file', dest='stereo_file', default='./stereo.default', @@ -897,7 +921,7 @@ if __name__ == '__main__': sep = "," settings = run_and_parse_output("stereo_parse", args, sep, opt.verbose) out_prefix = settings['out_prefix'][0] - + # See if to resume at triangulation if opt.tile_id is None and opt.prev_run_prefix is not None: print("Starting at the triangulation stage while reusing a previous run.") @@ -1011,9 +1035,13 @@ if __name__ == '__main__': num_pairs = int(settings['num_stereo_pairs'][0]) if num_pairs > 1: - # Run multivew logic. + # Run multiview logic. - if opt.keep_only is not None: + if opt.keep_only == "all_combined": + # Change the default to "unchanged" + opt.keep_only = "unchanged" + if opt.keep_only != "unchanged": + # If the user specified something else raise Exception('Option --keep-only does not work with multiview stereo.') # Bugfix: avoid confusing the logic below @@ -1126,7 +1154,9 @@ if __name__ == '__main__': skip_refine_step = (int(settings['subpixel_mode'][0]) > 6) # Blending (when using local_epipolar alignment, or SGM/MGM, or external algorithms) - if using_padded_tiles: + if not using_padded_tiles: + print("No work to do at the blending step.") + else: step = Step.blend if (opt.entry_point <= step): if (opt.stop_point <= step): @@ -1172,13 +1202,18 @@ if __name__ == '__main__': if (opt.entry_point <= step): if (opt.stop_point <= step): sys.exit() - - build_vrt('stereo_rfne', settings, georef, "-RD.tif", "-RD.tif") + try: + build_vrt('stereo_rfne', settings, georef, "-RD.tif", "-RD.tif") + except Exception as e: + # Make the error message more informative + raise Exception('Failed to build a VRT from */*RD.tif files. Must redo at least the refinement step. Additional error message: ' + str(e)) + normal_run('stereo_fltr', args, msg='%d: Filtering' % step) create_subproject_dirs(settings) # symlink F.tif # Triangulation step = Step.tri + subdirs = [] if (opt.entry_point <= step): if (opt.stop_point <= step): sys.exit() @@ -1210,20 +1245,20 @@ if __name__ == '__main__': asp_cmd_utils.wipe_option(parallel_args, '--num-matches-from-disp-triplets', 1) # symlink the files just created - create_subproject_dirs(settings) + subdirs = create_subproject_dirs(settings) # Run triangulation on multiple machines spawn_to_nodes(step, settings, parallel_args) build_vrt('stereo_tri', settings, georef, "-PC.tif", "-PC.tif") # mosaic - if (opt.entry_point >= Step.tri or opt.stop_point > Step.tri): - # Allow this logic to be called with --entry-step 6, which will just - # wipe files. For example, after point2dem is invoked, this can - # be used to delete PC.tif. - if opt.keep_only is not None: - keepOnlySpecified(opt.keep_only, out_prefix) + # If the run concluded successfully, merge and wipe + if (opt.stop_point > Step.tri): + if len(subdirs) == 0: + # If this was not created by now, do it now + subdirs = create_subproject_dirs(settings) + keepOnlySpecified(opt.keep_only, out_prefix, subdirs) - # End main process case + # End main process case else: # This process was spawned by GNU Parallel with a given