From 6a2fc81fde7c9c4b07edf20dbf1778c78df8e07a Mon Sep 17 00:00:00 2001 From: Ian Harry Date: Wed, 12 Jul 2023 09:50:15 +0100 Subject: [PATCH] Remove most X509/globus links and hooks from PyCBC (#4375) * Remove most X509/globus links and hooks from PyCBC * Need to update this one as well * Don't need these options now * Update docs/workflow/pycbc_make_coinc_search_workflow.rst Co-authored-by: Gareth S Cabourn Davies --------- Co-authored-by: Gareth S Cabourn Davies --- bin/pycbc_submit_dax | 89 +------------------ docs/hwinj.rst | 2 +- docs/workflow/datafind.rst | 2 +- docs/workflow/initialization.rst | 2 +- .../pycbc_make_offline_search_workflow.rst | 23 +---- examples/search/submit.sh | 2 +- pycbc/frame/frame.py | 2 +- pycbc/workflow/configuration.py | 3 +- pycbc/workflow/core.py | 2 +- pycbc/workflow/datafind.py | 9 -- .../pegasus_files/pegasus-properties.conf | 4 +- pycbc/workflow/pegasus_sites.py | 6 +- pycbc/workflow/pegasus_workflow.py | 34 ------- 13 files changed, 15 insertions(+), 165 deletions(-) diff --git a/bin/pycbc_submit_dax b/bin/pycbc_submit_dax index 4f9bd3ef20e..8f33203a946 100644 --- a/bin/pycbc_submit_dax +++ b/bin/pycbc_submit_dax @@ -17,9 +17,7 @@ exec 2>&1 LOCAL_PEGASUS_DIR="" PEGASUS_PROPERTIES="" -NO_CREATE_PROXY=0 NO_QUERY_DB=0 -NO_GRID="" SUBMIT_DAX="--submit" HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}" @@ -54,7 +52,6 @@ while true ; do "") shift 2 ;; *) echo $2 >> extra-properties.conf ; shift 2 ;; esac ;; - -K|--no-create-proxy) NO_CREATE_PROXY=1 ; shift ;; -Q|--no-query-db) NO_QUERY_DB=1 ; shift ;; -n|--no-submit) SUBMIT_DAX="" ; shift ;; -l|--local-dir) @@ -62,7 +59,6 @@ while true ; do "") shift 2 ;; *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;; esac ;; - -G|--no-grid) NO_GRID="--forward nogrid" ; shift ;; -h|--help) echo "usage: pycbc_submit_dax [-h] [optional arguments]" echo @@ -72,13 +68,9 @@ while true ; do echo " the pegasus properties file" echo " -P, --append-pegasus-property STRING add the extra property" echo " specified by the argument" - echo " -K, --no-create-proxy Do not run ligo-proxy-init and assume" - echo " that the user has a valid grid proxy" echo " -n, --no-submit Plan the DAX but do not submit it" echo " -l, --local-dir Directory to put condor files under" echo " -Q, --no-query-db Don't query the pegasus DB." - echo " -G, --no-grid Disable checks for grid proxy and" - echo " GLOBUS_LOCATION in pegasus-plan" echo echo "If the environment variable TMPDIR is set then this is prepended to the " echo "path to the temporary workflow execute directory passed to pegasus-plan." @@ -95,49 +87,6 @@ while true ; do esac done -if [ $NO_CREATE_PROXY == 0 ]; then - # Force the user to create a new grid proxy - LIGO_USER_NAME="" - while true; do - read -p "Enter your LIGO.ORG username in (e.g. albert.einstein): " LIGO_USER_NAME - echo - if [ ! -z $LIGO_USER_NAME ] ; then - break - fi - done - unset X509_USER_PROXY - ligo-proxy-init -p $LIGO_USER_NAME || exit 1 -else - if [ ! -z ${X509_USER_PROXY} ] ; then - if [ -f ${X509_USER_PROXY} ] ; then - cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u` - fi - unset X509_USER_PROXY - fi -fi - -if [ -z "${NO_GRID}" ] ; then - #Check that the proxy is valid - set +e - ecp-cert-info -exists - RESULT=$? - set -e - if [ ${RESULT} -eq 0 ] ; then - PROXY_TYPE=`ecp-cert-info -type | tr -d ' '` - if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig - grid-proxy-init -hours 276 -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig - rm -f /tmp/x509up_u`id -u`.orig - ecp-cert-info - fi - else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 - fi -fi - #Make a directory for the submit files SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX` @@ -170,7 +119,7 @@ cat extra-properties.conf >> pegasus-properties.conf # it deserves! STORED_PLANNER_ARGS=`cat additional_planner_args.dat` -pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX $NO_GRID ${STORED_PLANNER_ARGS} +pegasus-plan --conf ./pegasus-properties.conf --dir $SUBMIT_DIR $SUBMIT_DAX ${STORED_PLANNER_ARGS} echo @@ -186,42 +135,6 @@ chmod 755 debug echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop chmod 755 stop -if [ -z "${NO_GRID}" ] ; then - cat << EOF > start -#!/bin/bash - -if [ -f /tmp/x509up_u\`id -u\` ] ; then - unset X509_USER_PROXY -else - if [ ! -z \${X509_USER_PROXY} ] ; then - if [ -f \${X509_USER_PROXY} ] ; then - cp -a \${X509_USER_PROXY} /tmp/x509up_u\`id -u\` - fi - fi - unset X509_USER_PROXY -fi - -# Check that the proxy is valid -ecp-cert-info -exists -RESULT=\${?} -if [ \${RESULT} -eq 0 ] ; then - PROXY_TYPE=\`ecp-cert-info -type | tr -d ' '\` - if [ x\${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u\`id -u\` /tmp/x509up_u\`id -u\`.orig - grid-proxy-init -hours 276 -cert /tmp/x509up_u\`id -u\`.orig -key /tmp/x509up_u\`id -u\`.orig - rm -f /tmp/x509up_u\`id -u\`.orig - ecp-cert-info - fi -else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 -fi - -EOF -fi - echo "pegasus-run $SUBMIT_DIR/work \$@" > start chmod 755 start diff --git a/docs/hwinj.rst b/docs/hwinj.rst index f09c89a448c..11da93a62c7 100644 --- a/docs/hwinj.rst +++ b/docs/hwinj.rst @@ -181,7 +181,7 @@ You can plot the ASCII waveform files with an X11 connection. It's strongly reco where ``${OUTPUT_PATH}`` is the path to the output plot. -If you are using ``ssh`` or ``gsissh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example :: +If you are using ``ssh`` to log into a cluster, you can provide the ``-Y`` option to open an X11 connection. For example :: gsissh -Y ldas-pcdev1.ligo.caltech.edu diff --git a/docs/workflow/datafind.rst b/docs/workflow/datafind.rst index 94d2de364a8..b3e4d288ad2 100644 --- a/docs/workflow/datafind.rst +++ b/docs/workflow/datafind.rst @@ -83,7 +83,7 @@ When using any of the AT_RUNTIME sub-modules the following other configuration o * datafind-X1-frame-type = NAME - REQUIRED. Where X1 is replaced by the ifo name for each ifo. The NAME should be the full frame type, which is used when querying the database. * datafind-ligo-datafind-server = URL - OPTIONAL. If provided use this server when querying for frames. If not provided, which is recommended for most applications, then the LIGO_DATAFIND_SERVER environment variable will be used to determine this. -* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this). +* datafind-backup-datafind-server = URL - OPTIONAL. This option is only available when using AT_RUNTIME_SINGLE_FRAMES or AT_RUNTIME_MULTIPLE_FRAMES. If given it will query a second datafind server (ie. a remote server) using gsiftp urltypes. This will then allow frames to be associated with both a file:// and gsiftp:// url, in the case that your local site is missing a frame file, or the file is not accessible, pegasus will copy the file from gsiftp://. **NOTE** This will not catch the case that the frame file is available at the **start** of a workflow but goes missing later. Pegasus can copy **all** frame files around at the start of the workflow, but you may not want this (remove symlink option from the basic_pegasus.conf if you want this). **WARNING** gsiftp copying is largely deprecated. This option can still work, if you have a valid X509 certificate, but if this is used in a production setting we should investigate the use case. Please contact Ian if using this! When using the PREGENERATED sub-module the following configuartion options apply in the [workflow-datafind] section: diff --git a/docs/workflow/initialization.rst b/docs/workflow/initialization.rst index c4c516666d2..ece0ff1c401 100644 --- a/docs/workflow/initialization.rst +++ b/docs/workflow/initialization.rst @@ -193,7 +193,7 @@ This section should contain the names of each of the executables that will be us tmpltbank = /full/path/to/lalapps_tmpltbank inspiral = /full/path/to/lalapps_inspiral -Note that one can give gsiftp or http/https paths here and the workflow generator will download the code to the workflow directory when it is run. +Note that one can give remote URLs here and the workflow generator will download the code to the workflow directory when it is run. One can also give a URL indicating singularity as the scheme. This will indicate that the executable will be run within a singularity container, and therefore the executables would not be directly accessible from the head node:: diff --git a/docs/workflow/pycbc_make_offline_search_workflow.rst b/docs/workflow/pycbc_make_offline_search_workflow.rst index 3a195b9f640..e7694321d8a 100644 --- a/docs/workflow/pycbc_make_offline_search_workflow.rst +++ b/docs/workflow/pycbc_make_offline_search_workflow.rst @@ -643,9 +643,9 @@ full URL where the file can be found, and ``SITE`` is the site on which that URL resides. The URI in the ``PHYSICAL_FILE_URL`` can be any of the URIs that Pegasus -recognizes. The URIs ``file://``, ``gsiftp://``, and ``http://`` are likely +recognizes. The URIs ``file://``, ``https://`` are likely the most useful. Pegasus will take care of adding transfer jobs for -``gsiftp://`` and ``http://`` URIs, if the data is not available locally. +``https://`` URIs, if the data is not available locally. The string ``SITE`` is a hint that tells Pegasus on which site the ``PHYSICAL_FILE_URL`` can be found. The ``SITE`` string should be one of the @@ -672,7 +672,7 @@ might not be obvious from the hostname in the ``PHYSICAL_FILE_URL``. The following rule should be helpful when chosing the ``SITE`` string: * If you are re-using a file that is available locally with a ``file://`` URI in its ``PHYSICAL_FILE_URL`` (or has an implicit ``file://`` URI since the ``PHYSICAL_FILE_URL`` starts with a ``/``) then the string ``SITE`` should be set to ``local``. -* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will be either ``gsiftp://`` or ``http://`` depending on how the file can be accessed. +* If you are re-using a file from another cluster, e.g. you are on the Syracuse cluster and want to re-use data from AEI Atlas cluster, then the string ``SITE`` should be set to ``remote`` for that file. In this case, the URI in ``PHYSICAL_FILE_URL`` will begin with the scheme (e.g. ``https://``) depending on how the file can be accessed. To illustrate this, an example of a simple cache file containing four files for re-use from the ``local`` site is:: @@ -685,23 +685,6 @@ Note that the ``LOGICAL_FILE_NAME`` for the veto files is just the name of the file, but for the two inspiral files it contains the subdirectory that the workflow uses to organize the files by GPS time. In the case of this file Pegasus will delete from the workflow the jobs that create the files ``H1-VETOTIME_CAT3-1169107218-1066800.xml``, ``L1-VETOTIME_CAT3-1169107218-1066800.xml``, ``116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf``, and ``116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf`` when it plans the workflow. Insted, the data will be re-used from the URLs specified in the cache. Since ``site="local"`` for these files, Pegasus expects that the files all exist on the host where the workflow is run from. -To re-use data from a remote cluster, the URLs must contain a file transfer -mechanism and the ``SITE`` should be set to ``remote``. For example, if the -files listed in the example above are available on -``sugwg-condor.phy.syr.edu`` and you want to re-use them in a workflow on the -AEI Atlas cluster, then the cache file would contain:: - - H1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/H1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote" - L1-VETOTIME_CAT3-1169107218-1066800.xml gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/results/1._analysis_time/1.01_segment_data/L1-VETOTIME_CAT3-1169107218-1066800.xml pool="remote" - 116912/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB0-1169120586-1662.hdf pool="remote" - 116912/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf gsiftp://sugwg-condor.phy.syr.edu/home/dbrown/projects/aligo/o2/analysis-4/o2-analysis-4/output/full_data/H1-INSPIRAL_FULL_DATA_JOB1-1169120586-1662.hdf pool="remote" - -Note that the URL now contains ``gsiftp://sugwg-condor.phy.syr.edu`` rather -than ``file://localhost`` and the files are listes as ``pool="remote"`` rather -than ``pool="local"``. Pegasus will re-use these data files adding in -file transfer jobs to the workflow to move them into the appropriate -locations. - Once a cache file has been constructed, to enable data re-use, you follow the standard instructions for planning and submitting the workflow in the section :ref:`coincworkflowplan`, but add the ``--cache-file`` argument that points to diff --git a/examples/search/submit.sh b/examples/search/submit.sh index 9b9411b1449..0bd1f677bb6 100644 --- a/examples/search/submit.sh +++ b/examples/search/submit.sh @@ -1,3 +1,3 @@ -pycbc_submit_dax --no-grid --no-create-proxy \ +pycbc_submit_dax \ --local-dir ./ \ --no-query-db \ diff --git a/pycbc/frame/frame.py b/pycbc/frame/frame.py index 5b4cc9201a8..08658e380cd 100644 --- a/pycbc/frame/frame.py +++ b/pycbc/frame/frame.py @@ -272,7 +272,7 @@ def frame_paths(frame_type, start_time, end_time, server=None, url_type='file'): attempt is made to use a local datafind server. url_type : string Returns only frame URLs with a particular scheme or head such - as "file" or "gsiftp". Default is "file", which queries locally + as "file" or "https". Default is "file", which queries locally stored frames. Option can be disabled if set to None. Returns ------- diff --git a/pycbc/workflow/configuration.py b/pycbc/workflow/configuration.py index 5a58f5fef6e..35c4038ce4a 100644 --- a/pycbc/workflow/configuration.py +++ b/pycbc/workflow/configuration.py @@ -90,8 +90,7 @@ def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True): output_fp.close() else: - # TODO: We could support other schemes such as gsiftp by - # calling out to globus-url-copy + # TODO: We could support other schemes as needed errmsg = "Unknown URL scheme: %s\n" % (u.scheme) errmsg += "Currently supported are: file, http, and https." raise ValueError(errmsg) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index e8019ec0d19..46536b476e8 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -197,7 +197,7 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, # need to do anything here, as I cannot easily check it exists. exe_path = exe_url.path else: - # Could be http, gsiftp, etc. so it needs fetching if run now + # Could be http, https, etc. so it needs fetching if run now self.needs_fetching = True if self.needs_fetching and not self.installed: err_msg = "Non-file path URLs cannot be used unless the " diff --git a/pycbc/workflow/datafind.py b/pycbc/workflow/datafind.py index 7eef3b3dc8c..948cd0f350d 100644 --- a/pycbc/workflow/datafind.py +++ b/pycbc/workflow/datafind.py @@ -751,15 +751,6 @@ def convert_cachelist_to_filelist(datafindcache_list): # Datafind returned a URL valid on the osg as well # so add the additional PFNs to allow OSG access. currFile.add_pfn(frame.url, site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'root://xrootd-local.unl.edu/user/'), site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'gsiftp://red-gridftp.unl.edu/user/'), site='osg') - currFile.add_pfn(frame.url.replace( - 'file:///cvmfs/oasis.opensciencegrid.org/', - 'gsiftp://ldas-grid.ligo.caltech.edu/hdfs/'), site='osg') elif frame.url.startswith( 'file:///cvmfs/gwosc.osgstorage.org/'): # Datafind returned a URL valid on the osg as well diff --git a/pycbc/workflow/pegasus_files/pegasus-properties.conf b/pycbc/workflow/pegasus_files/pegasus-properties.conf index fe78a9c3bf4..06fd37bc9fe 100644 --- a/pycbc/workflow/pegasus_files/pegasus-properties.conf +++ b/pycbc/workflow/pegasus_files/pegasus-properties.conf @@ -17,14 +17,12 @@ pegasus.dir.storage.mapper.replica=File pegasus.dir.storage.mapper.replica.file=output.map # Add Replica selection options so that it will try URLs first, then -# XrootD for OSG, then gridftp, then anything else +# XrootD for OSG, then anything else # FIXME: This feels like a *site* property, not a global pegasus.selector.replica=Regex pegasus.selector.replica.regex.rank.1=file://(?!.*(cvmfs)).* pegasus.selector.replica.regex.rank.2=file:///cvmfs/.* pegasus.selector.replica.regex.rank.3=root://.* -pegasus.selector.replica.regex.rank.4=gsiftp://red-gridftp.unl.edu.* -pegasus.selector.replica.regex.rank.5=gridftp://.* pegasus.selector.replica.regex.rank.6=.\* dagman.maxpre=1 diff --git a/pycbc/workflow/pegasus_sites.py b/pycbc/workflow/pegasus_sites.py index a9a5a195f78..59209b55678 100644 --- a/pycbc/workflow/pegasus_sites.py +++ b/pycbc/workflow/pegasus_sites.py @@ -202,7 +202,9 @@ def add_osg_site(sitecat, cp): site.add_profiles(Namespace.CONDOR, key="should_transfer_files", value="Yes") site.add_profiles(Namespace.CONDOR, key="when_to_transfer_output", - value="ON_EXIT_OR_EVICT") + value="ON_SUCCESS") + site.add_profiles(Namespace.CONDOR, key="success_exit_code", + value="0") site.add_profiles(Namespace.CONDOR, key="+OpenScienceGrid", value="True") site.add_profiles(Namespace.CONDOR, key="getenv", @@ -211,8 +213,6 @@ def add_osg_site(sitecat, cp): value="False") site.add_profiles(Namespace.CONDOR, key="+SingularityCleanEnv", value="True") - site.add_profiles(Namespace.CONDOR, key="use_x509userproxy", - value="True") site.add_profiles(Namespace.CONDOR, key="Requirements", value="(HAS_SINGULARITY =?= TRUE) && " "(HAS_LIGO_FRAMES =?= True) && " diff --git a/pycbc/workflow/pegasus_workflow.py b/pycbc/workflow/pegasus_workflow.py index 9edcb941435..b2fc880ced4 100644 --- a/pycbc/workflow/pegasus_workflow.py +++ b/pycbc/workflow/pegasus_workflow.py @@ -38,37 +38,6 @@ PEGASUS_FILE_DIRECTORY = os.path.join(os.path.dirname(__file__), 'pegasus_files') -GRID_START_TEMPLATE = '''#!/bin/bash - -if [ -f /tmp/x509up_u`id -u` ] ; then - unset X509_USER_PROXY -else - if [ ! -z ${X509_USER_PROXY} ] ; then - if [ -f ${X509_USER_PROXY} ] ; then - cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u` - fi - fi - unset X509_USER_PROXY -fi - -# Check that the proxy is valid -ecp-cert-info -exists -RESULT=${?} -if [ ${RESULT} -eq 0 ] ; then - PROXY_TYPE=`ecp-cert-info -type | tr -d ' '` - if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then - ecp-cert-info - else - cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig - grid-proxy-init -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig - rm -f /tmp/x509up_u`id -u`.orig - ecp-cert-info - fi -else - echo "Error: Could not find a valid grid proxy to submit workflow." - exit 1 -fi -''' class ProfileShortcuts(object): """ Container of common methods for setting pegasus profile information @@ -731,9 +700,6 @@ def plan_and_submit(self, submit_now=True): fp.write('pegasus-remove {}/work $@'.format(submitdir)) with open('start', 'w') as fp: - if self.cp.has_option('pegasus_profile', 'pycbc|check_grid'): - fp.write(GRID_START_TEMPLATE) - fp.write('\n') fp.write('pegasus-run {}/work $@'.format(submitdir)) os.chmod('status', 0o755)