Remove most X509/globus links and hooks from PyCBC (gwastro#4375)

* Remove most X509/globus links and hooks from PyCBC

* Need to update this one as well

* Don't need these options now

* Update docs/workflow/pycbc_make_coinc_search_workflow.rst

Co-authored-by: Gareth S Cabourn Davies <gareth.cabourndavies@ligo.org>

---------

Co-authored-by: Gareth S Cabourn Davies <gareth.cabourndavies@ligo.org>

bin/pycbc_submit_dax

@@ -17,9 +17,7 @@ exec 2>&1
 
 LOCAL_PEGASUS_DIR=""
 PEGASUS_PROPERTIES=""
-NO_CREATE_PROXY=0
 NO_QUERY_DB=0
-NO_GRID=""
 SUBMIT_DAX="--submit"
 HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}"
 
@@ -54,15 +52,13 @@ while true ; do
         "") shift 2 ;;
         *) echo $2 >> extra-properties.conf ; shift 2 ;;
       esac ;;
-    -K|--no-create-proxy) NO_CREATE_PROXY=1 ; shift ;;
     -Q|--no-query-db) NO_QUERY_DB=1 ; shift ;;
     -n|--no-submit) SUBMIT_DAX="" ; shift ;;
     -l|--local-dir)
       case "$2" in
         "") shift 2 ;;
         *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
       esac ;;
-    -G|--no-grid) NO_GRID="--forward nogrid" ; shift ;;
     -h|--help)
       echo "usage: pycbc_submit_dax [-h] [optional arguments]"
       echo
@@ -72,13 +68,9 @@ while true ; do
       echo "                               the pegasus properties file"
       echo "  -P, --append-pegasus-property STRING add the extra property"
       echo "                                          specified by the argument"
-      echo "  -K, --no-create-proxy   Do not run ligo-proxy-init and assume"
-      echo "                             that the user has a valid grid proxy"
       echo "  -n, --no-submit         Plan the DAX but do not submit it"
       echo "  -l, --local-dir         Directory to put condor files under"
       echo "  -Q, --no-query-db       Don't query the pegasus DB."
-      echo "  -G, --no-grid           Disable checks for grid proxy and"
-      echo "                             GLOBUS_LOCATION in pegasus-plan"
       echo
       echo "If the environment variable TMPDIR is set then this is prepended to the "
       echo "path to the temporary workflow execute directory passed to pegasus-plan."
@@ -95,49 +87,6 @@ while true ; do
   esac
 done
 
-if [ $NO_CREATE_PROXY == 0 ]; then
-  # Force the user to create a new grid proxy
-  LIGO_USER_NAME=""
-  while true; do
-    read -p "Enter your LIGO.ORG username in (e.g. albert.einstein): " LIGO_USER_NAME
-    echo
-    if [ ! -z $LIGO_USER_NAME ] ; then
-      break
-    fi
-  done
-  unset X509_USER_PROXY
-  ligo-proxy-init -p $LIGO_USER_NAME || exit 1
-else
-  if [ ! -z ${X509_USER_PROXY} ] ; then
-    if [ -f ${X509_USER_PROXY} ] ; then
-      cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u`
-    fi
-  unset X509_USER_PROXY
-  fi
-fi
-
-if [ -z "${NO_GRID}" ] ; then
-  #Check that the proxy is valid
-  set +e
-  ecp-cert-info -exists
-  RESULT=$?
-  set -e
-  if [ ${RESULT} -eq 0 ] ; then
-    PROXY_TYPE=`ecp-cert-info -type | tr -d ' '`
-    if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
-      ecp-cert-info
-    else
-      cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig
-      grid-proxy-init -hours 276 -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig
-      rm -f /tmp/x509up_u`id -u`.orig
-      ecp-cert-info
-    fi
-  else
-    echo "Error: Could not find a valid grid proxy to submit workflow."
-    exit 1
-  fi
-fi
-
 #Make a directory for the submit files
 SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`
 
@@ -170,7 +119,7 @@ cat extra-properties.conf >> pegasus-properties.conf
 # it deserves!
 STORED_PLANNER_ARGS=`cat additional_planner_args.dat`
 
-pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX $NO_GRID ${STORED_PLANNER_ARGS}
+pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX ${STORED_PLANNER_ARGS}
 
 echo
 
@@ -186,42 +135,6 @@ chmod 755 debug
 echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop
 chmod 755 stop
 
-if [ -z "${NO_GRID}" ] ; then
-  cat << EOF > start
-#!/bin/bash
-
-if [ -f /tmp/x509up_u\`id -u\` ] ; then
-  unset X509_USER_PROXY
-else
-  if [ ! -z \${X509_USER_PROXY} ] ; then
-    if [ -f \${X509_USER_PROXY} ] ; then
-      cp -a \${X509_USER_PROXY} /tmp/x509up_u\`id -u\`
-    fi
-  fi
-  unset X509_USER_PROXY
-fi
-
-# Check that the proxy is valid
-ecp-cert-info -exists
-RESULT=\${?}
-if [ \${RESULT} -eq 0 ] ; then
-  PROXY_TYPE=\`ecp-cert-info -type | tr -d ' '\`
-  if [ x\${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
-    ecp-cert-info
-  else
-    cp /tmp/x509up_u\`id -u\` /tmp/x509up_u\`id -u\`.orig
-    grid-proxy-init -hours 276 -cert /tmp/x509up_u\`id -u\`.orig -key /tmp/x509up_u\`id -u\`.orig
-    rm -f /tmp/x509up_u\`id -u\`.orig
-    ecp-cert-info
-  fi
-else
-  echo "Error: Could not find a valid grid proxy to submit workflow."
-  exit 1
-fi
-
-EOF
-fi
-
 echo "pegasus-run $SUBMIT_DIR/work \$@" > start
 
 chmod 755 start

docs/hwinj.rst

@@ -181,7 +181,7 @@ You can plot the ASCII waveform files with an X11 connection. It's strongly reco
 
 where ``${OUTPUT_PATH}`` is the path to the output plot.
 
-If you are using ``ssh`` or ``gsissh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example ::
+If you are using ``ssh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example ::
 
   gsissh -Y ldas-pcdev1.ligo.caltech.edu
 

docs/workflow/datafind.rst

@@ -83,7 +83,7 @@ When using any of the AT_RUNTIME sub-modules the following other configuration o
 
 * datafind-X1-frame-type = NAME - REQUIRED. Where X1 is replaced by the ifo name for each ifo. The NAME should be the full frame type, which is used when querying the database.
 * datafind-ligo-datafind-server = URL - OPTIONAL. If provided use this server when querying for frames. If not provided, which is recommended for most applications, then the LIGO_DATAFIND_SERVER environment variable will be used to determine this.
-* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this).
+* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this). **WARNING** gsiftp copying is largely deprecated. This option can still work, if you have a valid X509 certificate, but if this is used in a production setting we should investigate the use case. Please contact Ian if using this!
 
 When using the PREGENERATED sub-module the following configuartion options apply in the [workflow-datafind] section:
 

docs/workflow/initialization.rst

@@ -193,7 +193,7 @@ This section should contain the names of each of the executables that will be us
   tmpltbank = /full/path/to/lalapps_tmpltbank
   inspiral = /full/path/to/lalapps_inspiral
 
-Note that one can give gsiftp or http/https paths here and the workflow generator will download the code to the workflow directory when it is run.
+Note that one can give remote URLs here and the workflow generator will download the code to the workflow directory when it is run.
 
 One can also give a URL indicating singularity as the scheme. This will indicate that the executable will be run within a singularity container, and therefore the executables would not be directly accessible from the head node::
 

docs/workflow/pycbc_make_offline_search_workflow.rst

@@ -643,9 +643,9 @@ full URL where the file can be found, and ``SITE`` is the site on which that URL
 resides.
 
 The URI in the ``PHYSICAL_FILE_URL`` can be any of the URIs that Pegasus
-recognizes. The URIs ``file://``, ``gsiftp://``, and ``http://`` are likely
+recognizes. The URIs ``file://``,  ``https://`` are likely
 the most useful. Pegasus will take care of adding transfer jobs for
-``gsiftp://`` and ``http://`` URIs, if the data is not available locally.
+``https://`` URIs, if the data is not available locally.
 
 The string ``SITE`` is a hint that tells Pegasus on which site the
 ``PHYSICAL_FILE_URL`` can be found. The ``SITE`` string should be one of the
@@ -672,7 +672,7 @@ might not be obvious from the hostname in the ``PHYSICAL_FILE_URL``.
 The following rule should be helpful when chosing the ``SITE`` string:
 
 * If you are re-using a file that is available locally with a ``file://`` URI in its ``PHYSICAL_FILE_URL`` (or has an implicit ``file://`` URI since the ``PHYSICAL_FILE_URL`` starts with a ``/``) then the string ``SITE`` should be set to ``local``.
-* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will be either ``gsiftp://`` or ``http://`` depending on how the file can be accessed.
+* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will begin with the scheme (e.g. ``https://``) depending on how the file can be accessed.
 
 To illustrate this, an example of a simple cache file containing four files for re-use from the ``local`` site is::
 
@@ -685,23 +685,6 @@ Note that the ``LOGICAL_FILE_NAME`` for the veto files is just the name of the
 file, but for the two inspiral files it contains the subdirectory that the
 workflow uses to organize the files by GPS time. In the case of this file Pegasus will delete from the workflow the jobs that create the files ``H1-VETOTIME_CAT3-1169107218-1066800.xml``, ``L1-VETOTIME_CAT3-1169107218-1066800.xml``, ``116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf``, and ``116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf`` when it plans the workflow. Insted, the data will be re-used from the URLs specified in the cache. Since ``site="local"`` for these files, Pegasus expects that the files all exist on the host where the workflow is run from.
 
-To re-use data from a remote cluster, the URLs must contain a file transfer
-mechanism and the ``SITE`` should be set to ``remote``. For example, if the
-files listed in the example above are available on
-``sugwg-condor.phy.syr.edu`` and you want to re-use them in a workflow on the
-AEI Atlas cluster, then the cache file would contain::
-
-    H1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/H1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote"
-    L1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/L1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote"
-    116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf pool="remote"
-    116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf pool="remote"
-
-Note that the URL now contains ``gsiftp://sugwg-condor.phy.syr.edu`` rather
-than ``file://localhost`` and the files are listes as ``pool="remote"`` rather
-than ``pool="local"``. Pegasus will re-use these data files adding in
-file transfer jobs to the workflow to move them into the appropriate
-locations.
-
 Once a cache file has been constructed, to enable data re-use, you follow the
 standard instructions for planning and submitting the workflow in the section
 :ref:`coincworkflowplan`, but add the ``--cache-file`` argument that points to

examples/search/submit.sh

@@ -1,3 +1,3 @@
-pycbc_submit_dax --no-grid --no-create-proxy \
+pycbc_submit_dax \
 --local-dir ./ \
 --no-query-db \

pycbc/frame/frame.py

@@ -272,7 +272,7 @@ def frame_paths(frame_type, start_time, end_time, server=None, url_type='file'):
         attempt is made to use a local datafind server.
     url_type : string
         Returns only frame URLs with a particular scheme or head such
-        as "file" or "gsiftp". Default is "file", which queries locally
+        as "file" or "https". Default is "file", which queries locally
         stored frames. Option can be disabled if set to None.
     Returns
     -------

pycbc/workflow/configuration.py

@@ -90,8 +90,7 @@ def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True):
         output_fp.close()
 
     else:
-        # TODO: We could support other schemes such as gsiftp by
-        # calling out to globus-url-copy
+        # TODO: We could support other schemes as needed
         errmsg = "Unknown URL scheme: %s\n" % (u.scheme)
         errmsg += "Currently supported are: file, http, and https."
         raise ValueError(errmsg)

pycbc/workflow/core.py

@@ -197,7 +197,7 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None,
             # need to do anything here, as I cannot easily check it exists.
             exe_path = exe_url.path
         else:
-            # Could be http, gsiftp, etc. so it needs fetching if run now
+            # Could be http, https, etc. so it needs fetching if run now
             self.needs_fetching = True
             if self.needs_fetching and not self.installed:
                 err_msg = "Non-file path URLs cannot be used unless the "

pycbc/workflow/datafind.py

@@ -751,15 +751,6 @@ def convert_cachelist_to_filelist(datafindcache_list):
                     # Datafind returned a URL valid on the osg as well
                     # so add the additional PFNs to allow OSG access.
                     currFile.add_pfn(frame.url, site='osg')
-                    currFile.add_pfn(frame.url.replace(
-                        'file:///cvmfs/oasis.opensciencegrid.org/',
-                        'root://xrootd-local.unl.edu/user/'), site='osg')
-                    currFile.add_pfn(frame.url.replace(
-                        'file:///cvmfs/oasis.opensciencegrid.org/',
-                        'gsiftp://red-gridftp.unl.edu/user/'), site='osg')
-                    currFile.add_pfn(frame.url.replace(
-                        'file:///cvmfs/oasis.opensciencegrid.org/',
-                        'gsiftp://ldas-grid.ligo.caltech.edu/hdfs/'), site='osg')
                 elif frame.url.startswith(
                     'file:///cvmfs/gwosc.osgstorage.org/'):
                     # Datafind returned a URL valid on the osg as well

pycbc/workflow/pegasus_files/pegasus-properties.conf

@@ -17,14 +17,12 @@ pegasus.dir.storage.mapper.replica=File
 pegasus.dir.storage.mapper.replica.file=output.map
 
 # Add Replica selection options so that it will try URLs first, then 
-# XrootD for OSG, then gridftp, then anything else
+# XrootD for OSG, then anything else
 # FIXME: This feels like a *site* property, not a global
 pegasus.selector.replica=Regex
 pegasus.selector.replica.regex.rank.1=file://(?!.*(cvmfs)).*
 pegasus.selector.replica.regex.rank.2=file:///cvmfs/.*
 pegasus.selector.replica.regex.rank.3=root://.*
-pegasus.selector.replica.regex.rank.4=gsiftp://red-gridftp.unl.edu.*
-pegasus.selector.replica.regex.rank.5=gridftp://.*
 pegasus.selector.replica.regex.rank.6=.\*
 
 dagman.maxpre=1

pycbc/workflow/pegasus_sites.py

@@ -202,7 +202,9 @@ def add_osg_site(sitecat, cp):
     site.add_profiles(Namespace.CONDOR, key="should_transfer_files",
                       value="Yes")
     site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output",
-                      value="ON_EXIT_OR_EVICT")
+                      value="ON_SUCCESS")
+    site.add_profiles(Namespace.CONDOR, key="success_exit_code",
+                      value="0")
     site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid",
                       value="True")
     site.add_profiles(Namespace.CONDOR, key="getenv",
@@ -211,8 +213,6 @@ def add_osg_site(sitecat, cp):
                       value="False")
     site.add_profiles(Namespace.CONDOR, key="+SingularityCleanEnv",
                       value="True")
-    site.add_profiles(Namespace.CONDOR, key="use_x509userproxy",
-                      value="True")
     site.add_profiles(Namespace.CONDOR, key="Requirements",
                       value="(HAS_SINGULARITY =?= TRUE) && "
                             "(HAS_LIGO_FRAMES =?= True) && "

pycbc/workflow/pegasus_workflow.py

@@ -38,37 +38,6 @@
 PEGASUS_FILE_DIRECTORY = os.path.join(os.path.dirname(__file__),
                                       'pegasus_files')
 
-GRID_START_TEMPLATE = '''#!/bin/bash
-
-if [ -f /tmp/x509up_u`id -u` ] ; then
-  unset X509_USER_PROXY
-else
-  if [ ! -z ${X509_USER_PROXY} ] ; then
-    if [ -f ${X509_USER_PROXY} ] ; then
-      cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u`
-    fi
-  fi
-  unset X509_USER_PROXY
-fi
-
-# Check that the proxy is valid
-ecp-cert-info -exists
-RESULT=${?}
-if [ ${RESULT} -eq 0 ] ; then
-  PROXY_TYPE=`ecp-cert-info -type | tr -d ' '`
-  if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
-    ecp-cert-info
-  else
-    cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig
-    grid-proxy-init -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig
-    rm -f /tmp/x509up_u`id -u`.orig
-    ecp-cert-info
-  fi
-else
-  echo "Error: Could not find a valid grid proxy to submit workflow."
-  exit 1
-fi
-'''
 
 class ProfileShortcuts(object):
     """ Container of common methods for setting pegasus profile information
@@ -731,9 +700,6 @@ def plan_and_submit(self, submit_now=True):
             fp.write('pegasus-remove {}/work $@'.format(submitdir))
 
         with open('start', 'w') as fp:
-            if self.cp.has_option('pegasus_profile', 'pycbc|check_grid'):
-                fp.write(GRID_START_TEMPLATE)
-                fp.write('\n')
             fp.write('pegasus-run {}/work $@'.format(submitdir))
 
         os.chmod('status', 0o755)