From 6467368352c6e4da53f17d6f095620db1231984f Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 9 Mar 2017 14:37:44 +0100 Subject: [PATCH 01/21] Added gtools wrappers for uberftp. gcat, gls, gmkdir, gmv and grm all support host aliases and remote directory browsing. --- bin/gcat | 1 + bin/gls | 1 + bin/gmkdir | 1 + bin/gmv | 1 + bin/grm | 1 + bin/gtools | 1 + bin/gtools.bash | 457 ++++++++++++++ .../gtools-bash-completion.sh | 417 +++++++++++++ lib/gtransfer/gridftp.bashlib | 558 ++++++++++++++++++ 9 files changed, 1438 insertions(+) create mode 120000 bin/gcat create mode 120000 bin/gls create mode 120000 bin/gmkdir create mode 120000 bin/gmv create mode 120000 bin/grm create mode 120000 bin/gtools create mode 100755 bin/gtools.bash create mode 100644 etc/bash_completion.d/gtools-bash-completion.sh create mode 100644 lib/gtransfer/gridftp.bashlib diff --git a/bin/gcat b/bin/gcat new file mode 120000 index 0000000..6922b7b --- /dev/null +++ b/bin/gcat @@ -0,0 +1 @@ +gtools \ No newline at end of file diff --git a/bin/gls b/bin/gls new file mode 120000 index 0000000..6922b7b --- /dev/null +++ b/bin/gls @@ -0,0 +1 @@ +gtools \ No newline at end of file diff --git a/bin/gmkdir b/bin/gmkdir new file mode 120000 index 0000000..6922b7b --- /dev/null +++ b/bin/gmkdir @@ -0,0 +1 @@ +gtools \ No newline at end of file diff --git a/bin/gmv b/bin/gmv new file mode 120000 index 0000000..6922b7b --- /dev/null +++ b/bin/gmv @@ -0,0 +1 @@ +gtools \ No newline at end of file diff --git a/bin/grm b/bin/grm new file mode 120000 index 0000000..6922b7b --- /dev/null +++ b/bin/grm @@ -0,0 +1 @@ +gtools \ No newline at end of file diff --git a/bin/gtools b/bin/gtools new file mode 120000 index 0000000..ea4eec5 --- /dev/null +++ b/bin/gtools @@ -0,0 +1 @@ +gtools.bash \ No newline at end of file diff --git a/bin/gtools.bash b/bin/gtools.bash new file mode 100755 index 0000000..c6cff2a --- /dev/null +++ b/bin/gtools.bash @@ -0,0 +1,457 @@ +#!/bin/bash + +# gtools.bash - GridFTP tools multicall executable + +:<. + +COPYRIGHT + +################################################################################ +# DEFINES +################################################################################ + +readonly _gtools_exit_usage=64 +readonly _gtools_version="0.2.0" +readonly _program=$( basename $0 ) + +################################################################################ +# PATH CONFIGURATION +################################################################################ + +# path to configuration files (prefer git deploy!) +# For native OS packages: +if [[ -e "/etc/gtransfer" ]]; then + + gtransferConfigurationFilesPath="/etc/gtransfer" + # gtransfer is installed in "/usr/bin", hence the base path is "/usr" + gtransferBasePath="/usr" + gtransferLibPath="$gtransferBasePath/share" + gtransferLibexecPath="$gtransferBasePath/libexec/gtransfer" + +# For installation with "install.sh". +#sed#elif [[ -e "/etc" ]]; then +#sed# +#sed# gtransferConfigurationFilesPath="/etc" +#sed# gtransferBasePath= +#sed# gtransferLibPath="$gtransferBasePath/lib" +#sed# gtransferLibexecPath="$gtransferBasePath/libexec" + +# According to FHS 2.3, configuration files for packages located in "/opt" have +# to be placed here (if you use a provider super dir below "/opt" for the +# gtransfer files, please also use the same provider super dir below +# "/etc/opt"). +#elif [[ -e "/etc/opt//gtransfer" ]]; then +# +# gtransferConfigurationFilesPath="/etc/opt//gtransfer" +# gtransferBasePath="/opt//gtransfer" +# gtransferLibPath="$gtransferBasePath/lib" +elif [[ -e "/etc/opt/gtransfer" ]]; then + + gtransferConfigurationFilesPath="/etc/opt/gtransfer" + gtransferBasePath="/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + gtransferLibexecPath="$gtransferBasePath/libexec" + +# For user install in $HOME: +elif [[ -e "$HOME/opt/gtransfer" ]]; then + + gtransferConfigurationFilesPath="$HOME/.gtransfer" + gtransferBasePath="$HOME/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + gtransferLibexecPath="$gtransferBasePath/libexec" + +# For git deploy, use $BASH_SOURCE +elif [[ -e "$( dirname $BASH_SOURCE )/../etc" ]]; then + + gtransferConfigurationFilesPath="$( dirname $BASH_SOURCE )/../etc/gtransfer" + gtransferBasePath="$( dirname $BASH_SOURCE )/../" + gtransferLibPath="$gtransferBasePath/lib" + gtransferLibexecPath="$gtransferBasePath/libexec" +fi + +gtransferConfigurationFile="$gtransferConfigurationFilesPath/gtransfer.conf" + +# Set $_LIB so gtransfer and its libraries can find their includes +readonly _LIB="$gtransferLibPath" + +readonly _gtransfer_libraryPrefix="gtransfer" +readonly _GTRANSFER_LIBPATH="$_LIB/gtransfer" + +# On SLES these files are located in `$_LIB/gtransfer` +if [[ -e "$_GTRANSFER_LIBPATH/getPidForUrl.r" && \ + -e "$_GTRANSFER_LIBPATH/getUrlForPid.r" && \ + -e "$_GTRANSFER_LIBPATH/packBinsNew.py" ]]; then + + readonly _GTRANSFER_LIBEXECPATH="$_GTRANSFER_LIBPATH" +else + readonly _GTRANSFER_LIBEXECPATH="$gtransferLibexecPath" +fi + +################################################################################ +# INCLUDES +################################################################################ + +_neededLibraries=( "gtransfer/gridftp.bashlib" ) + +for _library in ${_neededLibraries[@]}; do + + if ! . "$_LIB/$_library" 2>/dev/null; then + echo "$_program: Library \"$_LIB/$_library\" couldn't be read or is corrupted." 1>&2 + exit 70 + fi +done + + +################################################################################ +# FUNCTIONS +################################################################################ + +gtools/gcat() +{ + local _url="$1" + + gridftp/cat "$_url" + + return +} + + +gtools/gcatHelpMsg() +{ + cat <<-HELP + Usage: gcat + HELP + + return +} + + +gtools/gmv() +{ + local _oldUrl="$1" + local _newUrl="$2" + + gridftp/rename "$_oldUrl" "$_newUrl" + + return +} + + +gtools/gmvHelpMsg() +{ + cat <<-HELP + Usage: gmv + + Only works when both and point to the same remote GridFTP service! + HELP + + return +} + + +gtools/grm() +{ + local _url="$1" + + gridftp/removeFile "$_url" + + return +} + + +gtools/grmHelpMsg() +{ + cat <<-HELP + Usage: grm + + Also works for empty directories! + HELP + + return +} + + +gtools/gmkdir() +{ + local _url="$1" + + gridftp/mkdir "$_url" + + return +} + + +gtools/gmkdirHelpMsg() +{ + cat <<-HELP + Usage: gmkdir + + Behaves like `mkdir -p [...]'! + HELP + + return +} + +gtools/gls() +{ + local _url="$1" + + gridftp/ls "$_url" + + return +} + + +gtools/glsHelpMsg() +{ + cat <<-HELP + Usage: gls + HELP + + return +} + + +gtools/versionMsg() +{ + echo "gtools v${_gtools_version} (gridftp.bashlib v${_gridftp_Version})" + + return +} + + +gtools/usageMsg() +{ + cat <<-USAGE + Usage: gtools [function [arguments]...] + or: function [arguments]... + + gtools is a multi-call shell script that combines various GridFTP + functionality into a single executable. Most people will create a + link to gtools for each function they wish to use and gtools will + act like whatever is was invoked as. + + Currently defined functions: + gls, gmkdir, gmv, grm + USAGE + + return +} + + +################################################################################ +# MAIN +################################################################################ + +# Short hands +case $( basename "$0" ) in + +"gcat") + exec gtools cat "$@" + ;; + +"gls") + exec gtools ls "$@" + ;; + +"gmkdir") + exec gtools mkdir "$@" + ;; + +"gmv") + exec gtools mv "$@" + ;; + +"grm") + exec gtools rm "$@" + ;; + +*) + : + ;; +esac + +# correct number of params? +if [[ "$#" -lt "1" ]]; then + # no, so output a usage message + gtools/usageMsg + exit $_gtools_exit_usage +fi + +# read in all parameters +while [[ "$1" != "" ]]; do + + # only valid params used? + if [[ "$1" != "cat" && \ + "$1" != "ls" && \ + "$1" != "mkdir" && \ + "$1" != "mv" && \ + "$1" != "rm" && \ + "$1" != "--help" && "$1" != "-h" && \ + "$1" != "--version" && "$1" != "-V" \ + ]]; then + # no, so output a usage message + gtools/usageMsg + exit $_gtools_exit_usage + fi + + if [[ "$1" == "cat" ]]; then + + shift 1 + + if [[ "$1" == "" ]]; then + + gtools/gcatHelpMsg + exit $_gtools_exit_usage + + elif [[ "$1" == "--help" || \ + "$1" == "-h" ]]; then + + gtools/gcatHelpMsg + + elif [[ "$1" == "--version" || \ + "$1" == "-V" ]]; then + + gtools/versionMsg + else + gtools/gcat "$1" + fi + + exit + + elif [[ "$1" == "ls" ]]; then + + shift 1 + + if [[ "$1" == "" ]]; then + + gtools/glsHelpMsg + exit $_gtools_exit_usage + + elif [[ "$1" == "--help" || \ + "$1" == "-h" ]]; then + + gtools/glsHelpMsg + + elif [[ "$1" == "--version" || \ + "$1" == "-V" ]]; then + + gtools/versionMsg + else + gtools/gls "$1" + fi + + exit + + elif [[ "$1" == "mkdir" ]]; then + + shift 1 + + if [[ "$1" == "" ]]; then + + gtools/gmkdirHelpMsg + exit $_gtools_exit_usage + + elif [[ "$1" == "--help" || \ + "$1" == "-h" ]]; then + + gtools/gmkdirHelpMsg + + elif [[ "$1" == "--version" || \ + "$1" == "-V" ]]; then + + gtools/versionMsg + else + gtools/gmkdir "$1" + fi + + exit + + elif [[ "$1" == "rm" ]]; then + + shift 1 + + if [[ "$1" == "" ]]; then + + gtools/grmHelpMsg + exit $_gtools_exit_usage + + elif [[ "$1" == "--help" || \ + "$1" == "-h" ]]; then + + gtools/grmHelpMsg + + elif [[ "$1" == "--version" || \ + "$1" == "-V" ]]; then + + gtools/versionMsg + else + gtools/grm "$1" + fi + + exit + + elif [[ "$1" == "mv" ]]; then + + shift 1 + + if [[ "$1" == "" ]]; then + + gtools/gmvHelpMsg + exit $_gtools_exit_usage + + elif [[ "$1" == "--help" || \ + "$1" == "-h" ]]; then + + gtools/gmvHelpMsg + + elif [[ "$1" == "--version" || \ + "$1" == "-V" ]]; then + + gtools/versionMsg + else + if [[ "$2" == "" ]]; then + + echo "$_program: missing." + gtools/gmvHelpMsg + exit $_gtools_exit_usage + else + gtools/gmv "$1" "$2" + fi + fi + + exit + + elif [[ "$1" == "--help" || "$1" == "-h" ]]; then + + shift 1 + + gtools/usageMsg + + exit + + elif [[ "$1" == "--version" || "$1" == "-V" ]]; then + + shift 1 + + gtools/versionMsg + + exit + fi +done + +exit diff --git a/etc/bash_completion.d/gtools-bash-completion.sh b/etc/bash_completion.d/gtools-bash-completion.sh new file mode 100644 index 0000000..c8bcd4c --- /dev/null +++ b/etc/bash_completion.d/gtools-bash-completion.sh @@ -0,0 +1,417 @@ +:<. + +This product includes software developed by members of the DEISA project +www.deisa.org. DEISA is an EU FP7 integrated infrastructure initiative under +contract number RI-222919. + +COPYRIGHT + +gtools_complete_urls() +{ + ######################################################################## + # HELPER FUNCTIONS #################################################### + ######################################################################## + getPathFromURL() + { + # determines the path portion from the URL: + # + # gsiftp://venus.milkyway.universe:2811(/path/to/)file + # + # usage: + #+ getPathFromURL "URL" + + local URL="$1" + + #local path=$(echo "$URL" | sed -e "s|$( getURLWithoutPath $URL )||") + # local path? + if echo $URL | grep "^.*://" &>/dev/null; then + # no + local tmp=$( echo "$URL" | cut -d '/' -f '4-' ) + # add leading '/' + tmp="/$tmp" + else + # yes + tmp=$URL + fi + + # strip any file portion from path + path=$(echo $tmp | grep -o '/.*/') + if [[ $path == "" ]]; then + path="/" + fi + + echo "$path" + } + + getPathFromAliasUrl() + { + local url="$1" + + # if there's currentl only the alias, assume "/" + if ! echo "$url" | grep "/" &>/dev/null; then + echo "/" + return + fi + + local path=${url#*/} + + # strip any file portion from path + #path=${path%/*} + # add leading '/' + path="/$path" + # strip any file portion from path + path=$(echo $path | grep -o '/.*/') + + #if [[ "$path" != "" ]]; then + # echo "/$path/" + #else + # echo "/" + #fi + if [[ $path == "" ]]; then + path="/" + fi + + echo "$path" + } + + getURLWithoutPath() + { + # determines the URL portion that consists of the protocol id, the + #+ domain name and the port, or "file://": + # + # (gsiftp://venus.milkyway.universe:2811)/path/to/file + # (file://)/path/to/local/file + # + # usage: + #+ getURLWithoutPath "URL" + + local URL="$1" + + # TODO: + #+ support URLs not containing any port descriptions: + # + # done! + + :<<-COMMENT + from: + " + echo gsiftp://venus.milkyway.universe/path/to/file | sed "s;\(gsiftp://[^/]*\)/.*;\1;" + " + + or + + " + echo gsiftp://venus.milkyway.universe/path/to/file | cut -d '/' -f "1-3" + " + + returns: + " + gsiftp://venus.milkyway.universe + " + COMMENT + + #local tmp=$(echo "$URL" | grep -o "gsiftp://.*:[[:digit:]]*") + local tmp="" + # URL starting with "/", then this is a local path (equal to + #+ "file://$URL". + if [[ ${URL:0:1} == "/" ]]; then + #echo "DEBUG: 1" + tmp="file://" + # valid URL + else + #echo "DEBUG: 3" + tmp=$( echo $URL | cut -d '/' -f "1-3" ) + fi + #if [[ "$tmp" == "" ]]; then + # tmp=$( echo "$URL" | grep -o "file://" ) + #fi + + # does $tmp start with 'gsiftp://'? + if echo $tmp | grep '^gsiftp://' &>/dev/null; then + # if yes, check if port is provided + if echo $tmp | grep -o ':[[:digit:]].*' &>/dev/null; then + # port provided by user, don't modify string + : + else + # no port provided, add default gsiftp port + tmp="${tmp}:2811" + fi + # does $tmp start with 'ftp://'? + elif echo $tmp | grep '^ftp://' &>/dev/null; then + # if yes, check if port is provided + if echo $tmp | grep -o ':[[:digit:]].*' &>/dev/null; then + # port provided by user, don't modify string + : + else + # no port provided, add default ftp port + tmp="${tmp}:21" + fi + # does $tmp start with 'http://'? + elif echo $tmp | grep '^http://' &>/dev/null; then + # if yes, check if port is provided + if echo $tmp | grep -o ':[[:digit:]].*' &>/dev/null; then + # port provided by user, don't modify string + : + else + # no port provided, add default ftp port + tmp="${tmp}:80" + fi + # does $tmp start with 'https://'? + elif echo $tmp | grep '^https://' &>/dev/null; then + # if yes, check if port is provided + if echo $tmp | grep -o ':[[:digit:]].*' &>/dev/null; then + # port provided by user, don't modify string + : + else + # no port provided, add default ftp port + tmp="${tmp}:443" + fi + fi + + local URLWithoutPath=$tmp + + echo "$URLWithoutPath" + } + ######################################################################## + + local cur prev opts + + # defuse ":" in completions, as the ":" implies specific readline + #+ behaviour. + COMP_WORDBREAKS=${COMP_WORDBREAKS//:} + # also defuse "@" (used when usernames are provided in URLs) + COMP_WORDBREAKS=${COMP_WORDBREAKS//@} + + COMPREPLY=() + + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if hash globus-url-copy &>/dev/null; then + # complete remote paths + if echo "$cur" | grep '^gsiftp://.*:.*' &>/dev/null; then + + userhost=$( getURLWithoutPath "${cur}" ) + #echo -e "\n$userhost" 1>&2 + userpath=$( getPathFromURL "${cur}" ) + #echo "$userpath" 1>&2 + + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) + + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) + + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) + return 0 + + elif echo "$cur" | grep '^ftp://.*:.*' &>/dev/null; then + + userhost=$( getURLWithoutPath "${cur}" ) + userpath=$( getPathFromURL "${cur}" ) + + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) + + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) + + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) + return 0 + fi + + if hash halias &>/dev/null; then + + alias="${cur%%/*}" ## remove path + user="${alias%%@*}" + alias="${alias#*@}" ## remove "user@" + + if halias --is-alias "$alias" &>/dev/null; then + + userhost=$( halias --dealias "$alias" ) + + if [[ "$user" != "$alias" ]]; then + + userhost=${userhost/:\/\//:\/\/$user@} + fi + #echo -e "\n$userhost A$alias U$user" 1>&2 + userpath=$( getPathFromAliasUrl "$cur" ) + #echo "$userpath" 1>&2 + + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) + + if [[ "$user" != "$alias" ]]; then + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${user}@${alias}${userpath}${path}; done ) + else + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${alias}${userpath}${path}; done ) + fi + + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) + return 0 + fi + fi + fi + + # only complete source URL host parts if dpath is available + if hash dpath &>/dev/null; then + # complete source URL host parts + local sites=$( dpath --list-sources ) + else + local sites="" + fi + + if hash halias &>/dev/null; then + local aliases=$( halias --list ) + else + local aliases="" + fi + + if [[ "$sites" != "" && "$aliases" != "" ]]; then + COMPREPLY=( $(compgen -W "${sites} ${aliases}" -- ${cur}) ) + elif [[ "$sites" != "" ]]; then + COMPREPLY=( $(compgen -W "${sites}" -- ${cur}) ) + elif [[ "$aliases" != "" ]]; then + COMPREPLY=( $(compgen -W "${aliases}" -- ${cur}) ) + fi + + return 0 +} + + +gtools_complete_general_options() +{ + local cur prev opts + + # defuse ":" in completions, as the ":" implies specific readline + #+ behaviour. + COMP_WORDBREAKS=${COMP_WORDBREAKS//:} + # also defuse "@" (used when usernames are provided in URLs) + COMP_WORDBREAKS=${COMP_WORDBREAKS//@} + + COMPREPLY=() + + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + COMPREPLY=( $( compgen -W '--help -h --version -V' -- "$cur" ) ) + COMPREPLY=( "${COMPREPLY[@]/%/ }" ) + return 0 +} + + +gtools_complete_functions() +{ + local cur prev opts + + # defuse ":" in completions, as the ":" implies specific readline + #+ behaviour. + COMP_WORDBREAKS=${COMP_WORDBREAKS//:} + # also defuse "@" (used when usernames are provided in URLs) + COMP_WORDBREAKS=${COMP_WORDBREAKS//@} + + COMPREPLY=() + + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + COMPREPLY=( $( compgen -W "gcat gls gmkdir gmv grm" -- ${cur} ) ) + return 0 +} + + +_gtools() +{ + local cur prev opts + + # defuse ":" in completions, as the ":" implies specific readline + #+ behaviour. + COMP_WORDBREAKS=${COMP_WORDBREAKS//:} + # also defuse "@" (used when usernames are provided in URLs) + COMP_WORDBREAKS=${COMP_WORDBREAKS//@} + + COMPREPLY=() + + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + #echo "${#COMP_WORDS[@]}" 1>&2 + + if [[ $( basename $prev) == "gtools" ]]; then + + gtools_complete_functions + return 0 + fi + + if [[ $( basename $prev ) == "gcat" || \ + $( basename $prev ) == "gls" || \ + $( basename $prev ) == "gmkdir" || \ + $( basename $prev ) == "grm" ]]; then + + case $cur in + -*) + gtools_complete_general_options + return 0 + ;; + *) + #+ pass through + ;; + esac + + gtools_complete_urls + return 0 + + elif [[ $( basename ${COMP_WORDS[0]} ) == "gmv" ]]; then + + case $cur in + -*) + gtools_complete_general_options + return 0 + ;; + *) + #+ pass through + ;; + esac + + if [[ ${#COMP_WORDS[@]} -le 3 ]]; then + + gtools_complete_urls + return 0 + fi + + elif [[ $( basename ${COMP_WORDS[1]} ) == "gmv" ]]; then + + case $cur in + -*) + gtools_complete_general_options + return 0 + ;; + *) + #+ pass through + ;; + esac + + if [[ ${#COMP_WORDS[@]} -le 4 ]]; then + + gtools_complete_urls + return 0 + fi + fi + + return 0 +} + +complete -o nospace -F _gtools gtools gcat gls gmkdir gmv grm + diff --git a/lib/gtransfer/gridftp.bashlib b/lib/gtransfer/gridftp.bashlib new file mode 100644 index 0000000..a149a0c --- /dev/null +++ b/lib/gtransfer/gridftp.bashlib @@ -0,0 +1,558 @@ +#!/bin/bash -x + +# gridftp.bashlib - Utilities for gsatellite jobs + +:<. + +COPYRIGHT + +################################################################################ +# DEFINES +################################################################################ + +readonly _gridftp_Version="0.4.0" + +readonly _READY_MARKER="00_TRANSFER_READY" +readonly _DEST_MARKER="00_TRANSFER_DESTINATION" + + +################################################################################ +# FUNCTIONS +################################################################################ + +# Private: Try to expand the given GridFTP URL +# +# $1 (_gridftpURL) - GridFTP URL to expand +# +# Yields the possibly expanded GridFTP URL. +# +# Returns 0 on success, 1 otherwise. +gridftp/expandUrl() +{ + local _gridftpUrl="$1" + local _expandedGridftpUrl="" + + local _alias="" + local _user="" + local _host="" + local _path="" + + if echo "$_gridftpUrl" | grep '^gsiftp://.*:.*' &>/dev/null; then + + # no expansion needed + _expandedGridftpUrl="$_gridftpUrl" + echo "$_expandedGridftpUrl" + return 0 + + elif hash halias &>/dev/null; then + + _alias="${_gridftpUrl%%/*}" ## remove path + _user="${_alias%%@*}" + _alias="${_alias#*@}" ## remove "user@" + + #echo "A $_alias U $_user" 1>&2 + + if halias --is-alias "$_alias" &>/dev/null; then + + _host=$( halias --dealias "$_alias" ) + + #echo "H $_host" 1>&2 + + if [[ "$_user" != "$_alias" ]]; then + + _host=${_host/:\/\//:\/\/$_user@} + _expandedGridftpUrl="${_gridftpUrl/${_user}@${_alias}/$_host}" + else + _expandedGridftpUrl="${_gridftpUrl/$_alias/$_host}" + fi + else + _expandedGridftpUrl="$_gridftpUrl" + fi + + echo "$_expandedGridftpUrl" + return 0 + else + return 1 + fi +} + + +# Private: Check if file/directory exists. +# +# $1 (_gridftpFileUrl) - GridFTP URL for the file/directory to check +# +# Returns 0 if file exists, 1 otherwise and 2 if there is another problem. +gridftp/fileExists() +{ + local _gridftpFileUrl="$1" + + local _expandedUrl="" + local _command="" + local _output="" + + _expandedUrl=$( gridftp/expandUrl "$_gridftpFileUrl" ) + + if [[ $? -ne 0 ]]; then + + return 2 + fi + + _command="uberftp -ls "$_expandedUrl"" + + _output=$( $_command 2>&1 ) + + _returnVal=$? + + # File does exist + if [[ $_returnVal -eq 0 ]]; then + + return 0 + + # File does not exist or other error + elif [[ $_returnVal -eq 1 ]]; then + + # No GSI proxy credential available + if echo "$_output" | grep 'Failed to acquire credentials' &>/dev/null; then + + return 2 + + # Wrong TCP port used or used TCP port is firewalled + elif echo "$_output" | grep 'Failed to read()' &>/dev/null; then + + return 2 + + # File does not exist + else + return 1 + fi + + # Any other error + else + return 2 + fi +} + + +# Private: List file or directory contents in GridFTP URL +# +# $1 (_gridftpFileUrl) - GridFTP URL for the file/directory +# +# Returns 0 if successful, 1 otherwise and 2 if there is another problem. +gridftp/ls() +{ + local _gridftpFileUrl="$1" + + local _expandedUrl="" + local _command="" + local _output="" + + _expandedUrl=$( gridftp/expandUrl "$_gridftpFileUrl" ) + + if [[ $? -ne 0 ]]; then + + return 2 + fi + + _command="uberftp -ls "$_expandedUrl"" + + _output=$( $_command 2>&1 ) + + _returnVal=$? + + # File or dir does exist + if [[ $_returnVal -eq 0 ]]; then + + echo "$_output" + return 0 + + # File does not exist or other error + elif [[ $_returnVal -eq 1 ]]; then + + errorMsg "Couldn't list given URL." \ + "$_command" \ + "$_output" + return 1 + + # Any other error + else + return 2 + fi +} + + +# Private: Cat contents of file given in GridFTP URL +# +# $1 (_gridftpFileUrl) - GridFTP URL for the file +# +# Returns 0 if successful, 1 otherwise and 2 if there is another problem. +gridftp/cat() +{ + local _gridftpFileUrl="$1" + + local _expandedUrl="" + local _command="" + local _output="" + + _expandedUrl=$( gridftp/expandUrl "$_gridftpFileUrl" ) + + if [[ $? -ne 0 ]]; then + + return 2 + fi + + _command="uberftp -cat "$_expandedUrl"" + + _output=$( $_command 2>&1 ) + + _returnVal=$? + + # File or dir does exist + if [[ $_returnVal -eq 0 ]]; then + + echo "$_output" + return 0 + + # File does not exist or other error + elif [[ $_returnVal -eq 1 ]]; then + + errorMsg "Couldn't cat given URL." \ + "$_command" \ + "$_output" + return 1 + + # Any other error + else + return 2 + fi +} + + +# Private: Emit error message for failed GridFTP actions (including +# failed command and its output) +# +# $1 (_msg) - The error message to emit +# $2 (_command) - The failed command +# $3 (_output) - The output of the failed command +# +# Returns 0 on success, 1 otherwise. +errorMsg() +{ + local _msg="$1" + local _command="$2" + local _output="$3" + + if [[ "$_command" == "" && \ + "$_output" == "" ]]; then + + cat >&2 <<-EOM + E: $_msg + EOM + else + cat >&2 <<-EOM + E: $_msg + E: Failed command was: "$_command". + E: Output was: " + $( echo "$_output" | cat -v ) + " + EOM + fi + + if [[ $? -eq 0 ]]; then + + return 0 + else + return 1 + fi +} + + +# Private: Create directory identified by GridFTP URL +# +# $1 (_gridftpFileUrl) - GridFTP URL for the directory +# +# Behaves like `mkdir -p [...]`, but requires one additional GridFTP action +# (gridftp/mkdir()) for each directory component. Existing directories are +# ignored by uberftp's mkdir command, which allows to avoid a prior check for +# existence of a directory component. +# +# Returns 0 if successful, 1 otherwise and 2 if there is another problem. +gridftp/mkdir() +{ + local _gridftpFileUrl="$1" + + local _expandedUrl="" + local _index="4" ## index 4 includes the top dir in the remote URL + ## gsiftp://host.domain.tld:2811//[/...] + ## 12 3 4 + local _tmpUrl="" + + local _command="" + local _output="" + + _expandedUrl=$( gridftp/expandUrl "$_gridftpFileUrl" ) + + if [[ $? -ne 0 ]]; then + + return 2 + fi + + # shortcut if last sub directory is already existing + if gridftp/fileExists "$_expandedUrl"; then + + return 0 + fi + + while [[ 1 ]]; do + + _tmpUrl=$( echo $_expandedUrl | cut -d '/' -f "1-${_index}" ) + + #echo "U $_tmpUrl" 1>&2 + + _command="uberftp -mkdir "$_tmpUrl"" + + _output=$( $_command 2>&1 ) + + _returnVal=$? + + # Directory creation successful + if [[ $_returnVal -eq 0 ]]; then + + : + + # Directory couldn't be created or other error + elif [[ $_returnVal -eq 1 ]]; then + + errorMsg "Couldn't create directory for given URL." \ + "$_command" \ + "$_output" + return 1 + + # Any other error + else + return 2 + fi + + if [[ "$_tmpUrl" == "$_expandedUrl" ]]; then + + break + fi + + _index=$(( $_index + 1 )) + done + + return 0 +} + + +# Private: Remove file given via GridFTP URL +# +# $1 (_gridftpUrl) - The GridFTP URL (=file) +# +# Returns 0 on success, 1 otherwise. +gridftp/removeFile() +{ + local _gridftpUrl="$1" + + local _expandedUrl="" + + _expandedUrl=$( gridftp/expandUrl "$_gridftpUrl" ) + + if [[ $? -ne 0 ]]; then + + return 1 + fi + + _command="uberftp -rm "$_expandedUrl"" + + _output=$( $_command ) + + if [[ $? -ne 0 ]]; then + + errorMsg "Couldn't remove destination URL." \ + "$_command" \ + "$_output" + return 1 + else + return 0 + fi +} + + +# Private: Rename file/directory given via GridFTP URL +# +# $1 (_GridftpUrlOld) - The old GridFTP URL (=file/directory) +# $2 (_GridftpUrlNew) - The new GridFTP URL (=file/directory) +# +# Only works on the same remote system! +# +# Returns 0 on success, 1 otherwise, 2 if the remote GridFTP servers are +# not the same. +gridftp/rename() +{ + local _gridftpUrlOld="$1" + local _gridftpUrlNew="$2" + + local _expandedUrlOld="" + local _expandedUrlNew="" + + local _tmpUrlOld="" + local _tmpUrlNew="" + + _expandedUrlOld=$( gridftp/expandUrl "$_gridftpUrlOld" ) + + if [[ $? -ne 0 ]]; then + + return 1 + fi + + _expandedUrlNew=$( gridftp/expandUrl "$_gridftpUrlNew" ) + + if [[ $? -ne 0 ]]; then + + return 1 + fi + + _tmpUrlOld=$( echo $_expandedUrlOld | cut -d '/' -f "1-3" ) + _tmpUrlNew=$( echo $_expandedUrlNew | cut -d '/' -f "1-3" ) + + if [[ "$_tmpUrlOld" != "$_tmpUrlNew" ]]; then + + echo "E: Remote GridFTP servers have to be identical for rename operation!" 1>&2 + return 2 + fi + + # Now remove everything from the new URL so that only the file/directory and the path + # relative to the home directory is remaining + _expandedUrlNew=${_expandedUrlNew/*\/~\/} + + _command="uberftp -rename "$_expandedUrlOld" "$_expandedUrlNew"" + + _output=$( $_command 2>&1 ) + + if [[ $? -ne 0 ]]; then + + errorMsg "Couldn't rename to destination URL." \ + "$_command" \ + "$_output" + return 1 + else + return 0 + fi +} + + +# Public: Check if transfer is ready to start. +# +# $1 (_source) - The source URL (=dir) for the transfer +# +# Returns 0 on success (=transfer is ready), 1 otherwise. +gridftp/transferReady() +{ + local _source="$1" + + if gridftp/fileExists "${_source}/$_DEST_MARKER" && \ + gridftp/fileExists "${_source}/$_READY_MARKER"; then + + return 0 + else + return 1 + fi +} + + +# Public: Start the transfer by creating and submitting a gt job to +# gsatellite. +# +# $1 (_source) - The source URL (=dir) for the transfer +# +# Prints the job's ID and returns 0 on success, 1 otherwise. +gridftp/startTransfer() +{ + local _source="$1" + + local _expandedUrl="" + _expandedUrl=$( gridftp/expandUrl "$_source" ) + + if [[ $? -ne 0 ]]; then + + return 1 + fi + + local _uberftpOutput="" + local _destination="" + + local _command="" + local _output="" + + local _gtJobFile="" + local _gtJobId="" + + # (1) Get destination directory ############################################# + _command="uberftp -cat "${_expandedUrl}/${_DEST_MARKER}"" + _output=$( $_command 2>&1 ) + + if [[ $? -ne 0 ]]; then + + errorMsg "Couldn't determine destination URL. Cannot continue." \ + "$_command" \ + "$_output" + return 1 + else + _destination="$_output" + fi + + # (2) Remove marker files ################################################### + for _markerFile in $_READY_MARKER $_DEST_MARKER; do + + gridftp/renameFile "${_expandedUrl}/${_markerFile}" "${_expandedUrl}/${_markerFile}__" + #gridftp/removeFile "${_expandedUrl}/${_markerFile}" + done + + # (3) Create job file ####################################################### + _gtJobFile=$( mktemp --tmpdir="/tmp" XXXXXXXX ) + + cat > "$_gtJobFile" <<-EOF + #!/bin/bash + #GSAT -T gtransfer + # + # Created by job #${GSAT_O_JOBID} + # + + gt -s "${_source}*" -d "${_destination}" -e -r -v -o seq + + EOF + + # (4) Submit job ############################################################ + _command="gqsub "$_gtJobFile"" + _output=$( $_command ) + + if [[ $? -ne 0 ]]; then + + errorMsg "Job submission failed. Cannot continue." \ + "$_command" \ + "$_output" + return 1 + else + rm -f "$_gtJobFile" + _gtJobId="$_output" + echo "$_gtJobId" + return 0 + fi +} From bca612d7acaedb70f63b11b77b99381ed667cb1f Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 9 Mar 2017 16:04:38 +0100 Subject: [PATCH 02/21] Added manpage for gtools and updated gtools help message. --- bin/gtools.bash | 4 +-- share/doc/gtools.1.md | 76 +++++++++++++++++++++++++++++++++++++++++ share/man/man1/gtools.1 | 65 +++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 share/doc/gtools.1.md create mode 100644 share/man/man1/gtools.1 diff --git a/bin/gtools.bash b/bin/gtools.bash index c6cff2a..be039e1 100755 --- a/bin/gtools.bash +++ b/bin/gtools.bash @@ -25,7 +25,7 @@ COPYRIGHT ################################################################################ readonly _gtools_exit_usage=64 -readonly _gtools_version="0.2.0" +readonly _gtools_version="0.3.0" readonly _program=$( basename $0 ) ################################################################################ @@ -246,7 +246,7 @@ gtools/usageMsg() act like whatever is was invoked as. Currently defined functions: - gls, gmkdir, gmv, grm + (g)cat, (g)ls, (g)mkdir, (g)mv, (g)rm USAGE return diff --git a/share/doc/gtools.1.md b/share/doc/gtools.1.md new file mode 100644 index 0000000..3a8e45e --- /dev/null +++ b/share/doc/gtools.1.md @@ -0,0 +1,76 @@ +% GTOOLS(1) gtools 0.3.0 | User Commands +% Frank Scheiner +% Mar 09, 2017 + + +# NAME # + +**gtools** + + +# SYNOPSIS # + +**gtools [function [arguments]...]** or + +**function [arguments]...** if symlinked + + +# DESCRIPTION # + +**gtools** is a multi-call shell script that combines various GridFTP +functionality into a single executable. Most people will create a +link to gtools for each function they wish to use and gtools will +act like whatever is was invoked as. + + +# COMMON OPTIONS # + +All **gtools** functions provide a terse runtime description of their behavior +when invoked without arguments. + +# FUNCTIONS # + +Currently defined functions: + +(g)cat, (g)ls, (g)mkdir, (g)mv, (g)rm + +All functions support host aliases and remote directory browsing via bash completion. Use functions without the **g** prefix when using them via **gtools**, use functions with the **g** prefix when using them directly via (sym)links. + +# FUNCTION DESCRIPTIONS # + +## (g)cat ## + +**gcat _url_** + +Print to stdout the contents of the remote file given in _url_. + +## (g)ls ## + +**gls _url_** + +List the remote file or the contents of the remote directory given in _url_. + +## (g)mkdir ## + +**gmkdir _url_** + +Create the remote directory given in _url_. Creates non-existing directories +recursively but needs one GridFTP operation per directory. + +## (g)mv ## + +**gmv _url1_ _url2_** + +Move/rename the remote file or directory in _url1_ to the given path and/or +name in _url2_. + +## (g)rm ## + +**grm _url_** + +Remove the remote file or empty directory given in _url_. + + +# SEE ALSO # + +**uberftp(1c)**, **halias(1)** diff --git a/share/man/man1/gtools.1 b/share/man/man1/gtools.1 new file mode 100644 index 0000000..a974157 --- /dev/null +++ b/share/man/man1/gtools.1 @@ -0,0 +1,65 @@ +.TH "GTOOLS" "1" "Mar 09, 2017" "gtools 0.3.0" "User Commands" +.SH NAME +.PP +\f[B]gtools\f[] +.SH SYNOPSIS +.PP +\f[B]gtools [function [arguments]...]\f[] or +.PP +\f[B]function [arguments]...\f[] if symlinked +.SH DESCRIPTION +.PP +\f[B]gtools\f[] is a multi\-call shell script that combines various +GridFTP functionality into a single executable. +Most people will create a link to gtools for each function they wish to +use and gtools will act like whatever is was invoked as. +.SH COMMON OPTIONS +.PP +All \f[B]gtools\f[] functions provide a terse runtime description of +their behavior when invoked without arguments. +.SH FUNCTIONS +.PP +Currently defined functions: +.PP +(g)cat, (g)ls, (g)mkdir, (g)mv, (g)rm +.PP +All functions support host aliases and remote directory browsing via +bash completion. +Use functions without the \f[B]g\f[] prefix when using them via +\f[B]gtools\f[], use functions with the \f[B]g\f[] prefix when using +them directly via (sym)links. +.SH FUNCTION DESCRIPTIONS +.SS (g)cat +.PP +\f[B]gcat \f[I]url\f[]\f[] +.PP +Print to stdout the contents of the remote file given in \f[I]url\f[]. +.SS (g)ls +.PP +\f[B]gls \f[I]url\f[]\f[] +.PP +List the remote file or the contents of the remote directory given in +\f[I]url\f[]. +.SS (g)mkdir +.PP +\f[B]gmkdir \f[I]url\f[]\f[] +.PP +Create the remote directory given in \f[I]url\f[]. +Creates non\-existing directories recursively but needs one GridFTP +operation per directory. +.SS (g)mv +.PP +\f[B]gmv \f[I]url1\f[] \f[I]url2\f[]\f[] +.PP +Move/rename the remote file or directory in \f[I]url1\f[] to the given +path and/or name in \f[I]url2\f[]. +.SS (g)rm +.PP +\f[B]grm \f[I]url\f[]\f[] +.PP +Remove the remote file or empty directory given in \f[I]url\f[]. +.SH SEE ALSO +.PP +\f[B]uberftp(1c)\f[], \f[B]halias(1)\f[] +.SH AUTHORS +Frank Scheiner. From 065060ea1aeb15c05b05f250f20b26d578b070d8 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Tue, 14 Mar 2017 07:37:55 +0100 Subject: [PATCH 03/21] Finalized gtools manpage. --- share/doc/gtools.1.md | 26 ++++++++++++++++---------- share/man/man1/gtools.1 | 29 +++++++++++++++++++---------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/share/doc/gtools.1.md b/share/doc/gtools.1.md index 3a8e45e..c5ace01 100644 --- a/share/doc/gtools.1.md +++ b/share/doc/gtools.1.md @@ -1,6 +1,6 @@ % GTOOLS(1) gtools 0.3.0 | User Commands % Frank Scheiner -% Mar 09, 2017 +% Mar 14, 2017 # NAME # @@ -18,9 +18,9 @@ # DESCRIPTION # **gtools** is a multi-call shell script that combines various GridFTP -functionality into a single executable. Most people will create a -link to gtools for each function they wish to use and gtools will -act like whatever is was invoked as. +functionality (provided by **uberftp(1c)**) into a single executable. Most +people will create a link to gtools for each function they wish to use and +gtools will act like whatever is was invoked as. # COMMON OPTIONS # @@ -34,7 +34,11 @@ Currently defined functions: (g)cat, (g)ls, (g)mkdir, (g)mv, (g)rm -All functions support host aliases and remote directory browsing via bash completion. Use functions without the **g** prefix when using them via **gtools**, use functions with the **g** prefix when using them directly via (sym)links. +All functions support host aliases and remote directory browsing via bash +completion. Function names are without the **g[...]** prefix when used as +arguments of **gtools**. When used drectly (via symlinks) the **g[...]** prefix +should be used to differentiate them from the similar OS tools (**cat(1)**, +**ls(1)**, **mkdir(1)**, **mv(1)** and **rm(1)**). # FUNCTION DESCRIPTIONS # @@ -42,35 +46,37 @@ All functions support host aliases and remote directory browsing via bash comple **gcat _url_** -Print to stdout the contents of the remote file given in _url_. +Print to stdout the contents of the remote file given in _url_ (like **cat [...]**). ## (g)ls ## **gls _url_** List the remote file or the contents of the remote directory given in _url_. +Output is similar to **ls -la [...]**. ## (g)mkdir ## **gmkdir _url_** Create the remote directory given in _url_. Creates non-existing directories -recursively but needs one GridFTP operation per directory. +recursively (like **mkdir -p [...]**) but needs one GridFTP operation per directory! ## (g)mv ## **gmv _url1_ _url2_** Move/rename the remote file or directory in _url1_ to the given path and/or -name in _url2_. +name in _url2_. Only works when both _url1_ and _url2_ point to the same remote +GridFTP service! ## (g)rm ## **grm _url_** -Remove the remote file or empty directory given in _url_. +Remove the remote file (like **rm [...]**) or empty directory (like **rmdir [...]**) given in _url_. # SEE ALSO # -**uberftp(1c)**, **halias(1)** +**uberftp(1c)**, **halias(1)**, **cat(1)**, **ls(1)**, **mkdir(1)**, **rm(1)**, **rmdir(1)** diff --git a/share/man/man1/gtools.1 b/share/man/man1/gtools.1 index a974157..499fdf4 100644 --- a/share/man/man1/gtools.1 +++ b/share/man/man1/gtools.1 @@ -1,4 +1,4 @@ -.TH "GTOOLS" "1" "Mar 09, 2017" "gtools 0.3.0" "User Commands" +.TH "GTOOLS" "1" "Mar 14, 2017" "gtools 0.3.0" "User Commands" .SH NAME .PP \f[B]gtools\f[] @@ -10,7 +10,8 @@ .SH DESCRIPTION .PP \f[B]gtools\f[] is a multi\-call shell script that combines various -GridFTP functionality into a single executable. +GridFTP functionality (provided by \f[B]uberftp(1c)\f[]) into a single +executable. Most people will create a link to gtools for each function they wish to use and gtools will act like whatever is was invoked as. .SH COMMON OPTIONS @@ -25,41 +26,49 @@ Currently defined functions: .PP All functions support host aliases and remote directory browsing via bash completion. -Use functions without the \f[B]g\f[] prefix when using them via -\f[B]gtools\f[], use functions with the \f[B]g\f[] prefix when using -them directly via (sym)links. +Function names are without the \f[B]g[...]\f[] prefix when used as +arguments of \f[B]gtools\f[]. +When used drectly (via symlinks) the \f[B]g[...]\f[] prefix should be +used to differentiate them from the similar OS tools (\f[B]cat(1)\f[], +\f[B]ls(1)\f[], \f[B]mkdir(1)\f[], \f[B]mv(1)\f[] and \f[B]rm(1)\f[]). .SH FUNCTION DESCRIPTIONS .SS (g)cat .PP \f[B]gcat \f[I]url\f[]\f[] .PP -Print to stdout the contents of the remote file given in \f[I]url\f[]. +Print to stdout the contents of the remote file given in \f[I]url\f[] +(like \f[B]cat [...]\f[]). .SS (g)ls .PP \f[B]gls \f[I]url\f[]\f[] .PP List the remote file or the contents of the remote directory given in \f[I]url\f[]. +Output is similar to \f[B]ls \-la [...]\f[]. .SS (g)mkdir .PP \f[B]gmkdir \f[I]url\f[]\f[] .PP Create the remote directory given in \f[I]url\f[]. -Creates non\-existing directories recursively but needs one GridFTP -operation per directory. +Creates non\-existing directories recursively (like \f[B]mkdir \-p +[...]\f[]) but needs one GridFTP operation per directory! .SS (g)mv .PP \f[B]gmv \f[I]url1\f[] \f[I]url2\f[]\f[] .PP Move/rename the remote file or directory in \f[I]url1\f[] to the given path and/or name in \f[I]url2\f[]. +Only works when both \f[I]url1\f[] and \f[I]url2\f[] point to the same +remote GridFTP service! .SS (g)rm .PP \f[B]grm \f[I]url\f[]\f[] .PP -Remove the remote file or empty directory given in \f[I]url\f[]. +Remove the remote file (like \f[B]rm [...]\f[]) or empty directory (like +\f[B]rmdir [...]\f[]) given in \f[I]url\f[]. .SH SEE ALSO .PP -\f[B]uberftp(1c)\f[], \f[B]halias(1)\f[] +\f[B]uberftp(1c)\f[], \f[B]halias(1)\f[], \f[B]cat(1)\f[], +\f[B]ls(1)\f[], \f[B]mkdir(1)\f[], \f[B]rm(1)\f[], \f[B]rmdir(1)\f[] .SH AUTHORS Frank Scheiner. From b2e2ae92b6653be861f8dcfd0fb4362bfbcfaa5a Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Tue, 14 Mar 2017 08:10:11 +0100 Subject: [PATCH 04/21] Updated years in copyright statement. --- bin/gtools.bash | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/gtools.bash b/bin/gtools.bash index be039e1..1560323 100755 --- a/bin/gtools.bash +++ b/bin/gtools.bash @@ -4,7 +4,7 @@ :< Date: Tue, 14 Mar 2017 08:27:40 +0100 Subject: [PATCH 05/21] Renamed gtools bash completion file. --- etc/bash_completion.d/{gtools-bash-completion.sh => gtools.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etc/bash_completion.d/{gtools-bash-completion.sh => gtools.sh} (100%) diff --git a/etc/bash_completion.d/gtools-bash-completion.sh b/etc/bash_completion.d/gtools.sh similarity index 100% rename from etc/bash_completion.d/gtools-bash-completion.sh rename to etc/bash_completion.d/gtools.sh From 0399eb4d2ea9b66a693cddf17ffa40ddb1d56bee Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Tue, 14 Mar 2017 09:00:38 +0100 Subject: [PATCH 06/21] Updated installer script and spec file for RPM packaging. --- gtransfer.spec | 30 ++++++++++++++++++++++++++---- install.bash | 28 +++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/gtransfer.spec b/gtransfer.spec index e411f8f..4507628 100644 --- a/gtransfer.spec +++ b/gtransfer.spec @@ -1,5 +1,5 @@ Name: gtransfer -Version: 0.7.1 +Version: 0.8.0 Release: 1%{?dist} Summary: Advanced data transfer tool for GridFTP Group: base @@ -32,7 +32,7 @@ mkdir -p %{buildroot}%{_bindir} # SLES does not have a "libexec" dir! %if 0%{?rhel} mkdir -p %{buildroot}%{_libexecdir}/%{name} -%endif +%endif mkdir -p %{buildroot}%{_datadir}/%{name}/pids mkdir -p %{buildroot}%{_mandir}/man1 mkdir -p %{buildroot}%{_mandir}/man5 @@ -51,6 +51,7 @@ cp etc/gtransfer/pids/irodsMicroService_mappingFile %{buildroot}%{_sysconfdir}/% cp etc/gtransfer/aliases.conf %{buildroot}%{_sysconfdir}/%{name}/ cp etc/bash_completion.d/gtransfer.sh %{buildroot}%{_sysconfdir}/bash_completion.d/ +cp etc/bash_completion.d/gtools.sh %{buildroot}%{_sysconfdir}/bash_completion.d/ ################################################################################ # * Bash libraries @@ -63,6 +64,7 @@ cp lib/gtransfer/urlTransfer.bashlib %{buildroot}%{_datadir}/%{name}/ cp lib/gtransfer/alias.bashlib %{buildroot}%{_datadir}/%{name}/ cp lib/gtransfer/pids/irodsMicroService.bashlib %{buildroot}%{_datadir}/%{name}/pids/ cp lib/gtransfer/multipathing.bashlib %{buildroot}%{_datadir}/%{name}/ +cp lib/gtransfer/gridftp.bashlib %{buildroot}%{_datadir}/%{name}/ ################################################################################ # * Tools and symlinks @@ -72,12 +74,19 @@ cp bin/datapath.bash %{buildroot}%{_bindir}/ cp bin/defaultparam.bash %{buildroot}%{_bindir}/ cp bin/halias.bash %{buildroot}%{_bindir}/ cp bin/gtransfer-version.bash %{buildroot}%{_bindir}/ +cp bin/gtools.bash %{buildroot}%{_bindir}/ ln -s gtransfer.bash %{buildroot}%{_bindir}/gtransfer ln -s gtransfer.bash %{buildroot}%{_bindir}/gt ln -s datapath.bash %{buildroot}%{_bindir}/dpath ln -s defaultparam.bash %{buildroot}%{_bindir}/dparam ln -s halias.bash %{buildroot}%{_bindir}/halias ln -s gtransfer-version.bash %{buildroot}%{_bindir}/gt-version +ln -s gtools.bash %{buildroot}%{_bindir}/gtools +ln -s gtools.bash %{buildroot}%{_bindir}/gcat +ln -s gtools.bash %{buildroot}%{_bindir}/gls +ln -s gtools.bash %{buildroot}%{_bindir}/gmkdir +ln -s gtools.bash %{buildroot}%{_bindir}/gmv +ln -s gtools.bash %{buildroot}%{_bindir}/grm ################################################################################ # * Additional (internal) tools @@ -119,12 +128,14 @@ rm -rf %{buildroot} %config %{_sysconfdir}/%{name}/pids/irodsMicroService_mappingFile %config %{_sysconfdir}/%{name}/aliases.conf %config %{_sysconfdir}/bash_completion.d/gtransfer.sh +%config %{_sysconfdir}/bash_completion.d/gtools.sh -%doc README.md COPYING ChangeLog share/doc/dparam.1.md share/doc/dparam.5.md share/doc/dpath.1.md share/doc/dpath.5.md share/doc/gtransfer.1.md share/doc/halias.1.md share/doc/host-aliases.md share/doc/persistent-identifiers.md share/doc/images/multi-step-transfer.png share/doc/images/multipathing-transfer.png +%doc README.md COPYING ChangeLog share/doc/dparam.1.md share/doc/dparam.5.md share/doc/dpath.1.md share/doc/dpath.5.md share/doc/gtransfer.1.md share/doc/halias.1.md share/doc/gtools.1.md share/doc/host-aliases.md share/doc/persistent-identifiers.md share/doc/images/multi-step-transfer.png share/doc/images/multipathing-transfer.png %{_sysconfdir}/%{name} %{_sysconfdir}/bash_completion.d %{_sysconfdir}/bash_completion.d/gtransfer.sh +%{_sysconfdir}/bash_completion.d/gtools.sh %{_bindir}/gtransfer.bash %{_bindir}/gtransfer @@ -137,6 +148,13 @@ rm -rf %{buildroot} %{_bindir}/halias %{_bindir}/gtransfer-version.bash %{_bindir}/gt-version +%{_bindir}/gtools.bash +%{_bindir}/gtools +%{_bindir}/gcat +%{_bindir}/gls +%{_bindir}/gmkdir +%{_bindir}/gmv +%{_bindir}/grm %if 0%{?rhel} %{_libexecdir}/%{name} @@ -154,6 +172,7 @@ rm -rf %{buildroot} %{_datadir}/%{name}/alias.bashlib %{_datadir}/%{name}/pids/irodsMicroService.bashlib %{_datadir}/%{name}/multipathing.bashlib +%{_datadir}/%{name}/gridftp.bashlib %if 0%{?suse_version} %{_datadir}/%{name} %{_datadir}/%{name}/getPidForUrl.r @@ -168,8 +187,12 @@ rm -rf %{buildroot} %{_mandir}/man1/dparam.1.gz %{_mandir}/man5/dparam.5.gz %{_mandir}/man1/halias.1.gz +%{_mandir}/man1/gtools.1.gz %changelog +* Tue Mar 14 2017 Frank Scheiner 0.8.0-1 +- Updated source package and version number to new release. Also added new gtools to the toolkit. + * Thu Sep 08 2016 Frank Scheiner 0.7.1-1 - Updated source version number to new patch level. @@ -250,4 +273,3 @@ rm -rf %{buildroot} * Fri Sep 21 2012 Frank Scheiner 0.0.9-1 - Initial RPM package build for OBS - diff --git a/install.bash b/install.bash index 62fd9ec..1377fbd 100755 --- a/install.bash +++ b/install.bash @@ -61,7 +61,8 @@ if [[ "$(basename $0)" == "install.bash" ]]; then cp ./bin/gtransfer.bash \ ./bin/datapath.bash \ ./bin/defaultparam.bash \ - ./bin/halias.bash "$binDir" + ./bin/halias.bash \ + ./bin/gtools.bash "$binDir" # ...reconfigure paths inside of the scripts and... # + reconfigure path to configuration files @@ -71,17 +72,25 @@ if [[ "$(basename $0)" == "install.bash" ]]; then sed -e "s||$prefixDir/gtransfer|g" -e 's/#sed#//g' -i "$binDir/datapath.bash" sed -e "s||$prefixDir/gtransfer|g" -e 's/#sed#//g' -i "$binDir/defaultparam.bash" sed -e "s||$prefixDir/gtransfer|g" -e 's/#sed#//g' -i "$binDir/halias.bash" + sed -e "s||$prefixDir/gtransfer|g" -e 's/#sed#//g' -i "$binDir/gtools.bash" # ...make links and... if [[ $userInstall -eq 1 ]]; then linkPath="$HOME/bin" - + ln -s "$binDir/gtransfer.bash" "$linkPath/gtransfer" ln -s "$binDir/gtransfer.bash" "$linkPath/gt" ln -s "$binDir/datapath.bash" "$linkPath/dpath" ln -s "$binDir/defaultparam.bash" "$linkPath/dparam" ln -s "$binDir/halias.bash" "$linkPath/halias" + ln -s "$binDir/gtransfer-version.bash" "$linkPath/gt-version" + ln -s "$binDir/gtools.bash" "$linkPath/gtools" + ln -s "$binDir/gtools.bash" "$linkPath/gcat" + ln -s "$binDir/gtools.bash" "$linkPath/gls" + ln -s "$binDir/gtools.bash" "$linkPath/gmkdir" + ln -s "$binDir/gtools.bash" "$linkPath/gmv" + ln -s "$binDir/gtools.bash" "$linkPath/grm" else linkPath="$binDir" @@ -91,6 +100,13 @@ if [[ "$(basename $0)" == "install.bash" ]]; then ln -s "datapath.bash" "$linkPath/dpath" ln -s "defaultparam.bash" "$linkPath/dparam" ln -s "halias.bash" "$linkPath/halias" + ln -s "gtransfer-version.bash" "$linkPath/gt-version" + ln -s "gtools.bash" "$linkPath/gtools" + ln -s "gtools.bash" "$linkPath/gcat" + ln -s "gtools.bash" "$linkPath/gls" + ln -s "gtools.bash" "$linkPath/gmkdir" + ln -s "gtools.bash" "$linkPath/gmv" + ln -s "gtools.bash" "$linkPath/grm" fi # ...copy README and manpages. @@ -132,9 +148,15 @@ elif [[ "$(basename $0)" == "uninstall.bash" ]]; then rm "$HOME/bin/dpath" rm "$HOME/bin/dparam" rm "$HOME/bin/halias" + rm "$HOME/gt-version" + rm "$HOME/bin/gtools" + rm "$HOME/bin/gcat" + rm "$HOME/bin/gls" + rm "$HOME/bin/gmkdir" + rm "$HOME/bin/gmv" + rm "$HOME/bin/grm" # remove gtransfer dir rm -r "$prefixDir/gtransfer" fi fi - From 1dd177b6f2f1e1fb72920308320c462de4038822 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 16 Mar 2017 09:26:25 +0100 Subject: [PATCH 07/21] Relinked gtools short hands to the gtools bash script instead of the symlink to the gtools bash script. --- bin/gcat | 2 +- bin/gls | 2 +- bin/gmkdir | 2 +- bin/gmv | 2 +- bin/grm | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/gcat b/bin/gcat index 6922b7b..ea4eec5 120000 --- a/bin/gcat +++ b/bin/gcat @@ -1 +1 @@ -gtools \ No newline at end of file +gtools.bash \ No newline at end of file diff --git a/bin/gls b/bin/gls index 6922b7b..ea4eec5 120000 --- a/bin/gls +++ b/bin/gls @@ -1 +1 @@ -gtools \ No newline at end of file +gtools.bash \ No newline at end of file diff --git a/bin/gmkdir b/bin/gmkdir index 6922b7b..ea4eec5 120000 --- a/bin/gmkdir +++ b/bin/gmkdir @@ -1 +1 @@ -gtools \ No newline at end of file +gtools.bash \ No newline at end of file diff --git a/bin/gmv b/bin/gmv index 6922b7b..ea4eec5 120000 --- a/bin/gmv +++ b/bin/gmv @@ -1 +1 @@ -gtools \ No newline at end of file +gtools.bash \ No newline at end of file diff --git a/bin/grm b/bin/grm index 6922b7b..ea4eec5 120000 --- a/bin/grm +++ b/bin/grm @@ -1 +1 @@ -gtools \ No newline at end of file +gtools.bash \ No newline at end of file From e8902fba4c5b8cf553536d999f46d0cac5c71097 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 16 Mar 2017 09:29:04 +0100 Subject: [PATCH 08/21] Increased gt toolkit version number and added gtools. --- bin/gtransfer-version.bash | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/gtransfer-version.bash b/bin/gtransfer-version.bash index b82082a..41e91b1 100755 --- a/bin/gtransfer-version.bash +++ b/bin/gtransfer-version.bash @@ -5,7 +5,7 @@ :< Date: Thu, 16 Mar 2017 10:17:33 +0100 Subject: [PATCH 09/21] Updated years in copyright statement. Removed old unneeded commented code and also changed format of comments and tabs. --- bin/datapath.bash | 628 +++++++++++++++++++++++++++------------------- 1 file changed, 369 insertions(+), 259 deletions(-) diff --git a/bin/datapath.bash b/bin/datapath.bash index 598dedb..c93deb3 100755 --- a/bin/datapath.bash +++ b/bin/datapath.bash @@ -1,10 +1,10 @@ #!/bin/bash -# dpath - data path creation, editing, listing, etc. +# dpath - data path creation, editing, listing, etc. :</etc" ]]; then +#sed# #sed# gtransferConfigurationFilesPath="/etc" #sed# gtransferBasePath= #sed# gtransferLibPath="$gtransferBasePath/lib" -# According to FHS 2.3, configuration files for packages located in "/opt" have -#+ to be placed here (if you use a provider super dir below "/opt" for the -#+ gtransfer files, please also use the same provider super dir below -#+ "/etc/opt"). +# According to FHS 2.3, configuration files for packages located in "/opt" have +# to be placed here (if you use a provider super dir below "/opt" for the +# gtransfer files, please also use the same provider super dir below +# "/etc/opt"). #elif [[ -e "/etc/opt//gtransfer" ]]; then +# # gtransferConfigurationFilesPath="/etc/opt//gtransfer" # gtransferBasePath="/opt//gtransfer" # gtransferLibPath="$gtransferBasePath/lib" + elif [[ -e "/etc/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="/etc/opt/gtransfer" - gtransferBasePath="/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" + + gtransferConfigurationFilesPath="/etc/opt/gtransfer" + gtransferBasePath="/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" # For user install in $HOME: elif [[ -e "$HOME/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="$HOME/.gtransfer" - gtransferBasePath="$HOME/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" + + gtransferConfigurationFilesPath="$HOME/.gtransfer" + gtransferBasePath="$HOME/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" # For git deploy, use $BASH_SOURCE elif [[ -e "$( dirname $BASH_SOURCE )/../etc" ]]; then + gtransferConfigurationFilesPath="$( dirname $BASH_SOURCE )/../etc/gtransfer" gtransferBasePath="$( dirname $BASH_SOURCE )/../" gtransferLibPath="$gtransferBasePath/lib" @@ -74,22 +81,25 @@ dpathConfigurationFile="$gtransferConfigurationFilesPath/dpath.conf" __GLOBAL__sourcesIndexFile="sources.index" __GLOBAL__destinationsIndexFile="destinations.index" -__GLOBAL__requiredTools=( cat - sha1sum - cut - mkdir - touch - ln - grep - sed - tar - globus-url-copy +__GLOBAL__requiredTools=( cat + sha1sum + cut + mkdir + touch + ln + grep + sed + tar + globus-url-copy sort ) -#USAGE########################################################################## + +################################################################################ +# FUNCTIONS +################################################################################ usageMsg() { - cat < gsiftpSourceUrl => the host address of the source site - + \$destinationWithoutPath => gsiftpDestinationUrl => the host address of the destination site @@ -197,11 +206,11 @@ The options are as follows: dpaths directory in: "$HOME/.gtransfer/dpaths". - + When used, dpath will create dpaths for all possible connections between the hosts given in the hostsFile and omit connections between the same hosts. - + --hosts|-h hostsFile Set the file name for the file containing the host addresses for which dpaths should be created. The format of each line in this file is as follows: @@ -212,10 +221,10 @@ The options are as follows: When provided, dpath will use the given template for dpath creation. The following variables are expanded during dpath creation: - + \$sourceWithoutPath => gsiftpSourceUrl => the host address of the source site - + \$destinationWithoutPath => gsiftpDestinationUrl => the host address of the destination site @@ -232,7 +241,7 @@ The options are as follows: List all sources from the dpaths in the user-provided path or - if no additional path is given - in the user and system dpaths directories. - + --list-destinations [/path/to/dataPaths] List all destinations from the dpaths in the user- provided path or - if no additional path is given - in @@ -248,7 +257,7 @@ The options are as follows: ...and store them in the user-provided path or - if no additional path is given - in the user dpaths directory. If a "--quiet|-q" is provided, then output is omitted - and success/failure is only reported by the exit value. + and success/failure is only reported by the exit value. -------------------------------------------------------------------------------- @@ -275,27 +284,26 @@ The options are as follows: HELP return } -#END_HELP####################################################################### -#VERSION######################################################################## + versionMsg() { echo "$(basename $0) - The data path helper script v$version" - return + return } -#END_VERSION#################################################################### + hashSourceDestination() { - # hashes the "source;destination" combination + # hashes the "source;destination" combination # - # usage: - #+ hashSourceDestination source destination + # usage: + # hashSourceDestination source destination # - # NOTICE: - #+ "source" and "destination" are URLs without path but with port - #+ number! + # NOTICE: + # "source" and "destination" are URLs without path but with port + # number! local sourceWithoutPath="$1" local destinationWithoutPath="$2" @@ -305,18 +313,19 @@ hashSourceDestination() echo $dataPathName } + createDataPath() { - # creates a data path file and alias link + # creates a data path file and alias link # - # usage: - #+ createDataPath source destination alias path [dpathTemplate] + # usage: + # createDataPath source destination alias path [dpathTemplate] # - # returns: - #+ 0 - success - #+ 2 - data path already existing - #+ everything else - error - + # returns: + # 0 - success + # 2 - data path already existing + # everything else - error + local sourceWithoutPath="$1" local destinationWithoutPath="$2" local dataPathAlias="$3" @@ -325,18 +334,21 @@ createDataPath() local dataPathName="$( hashSourceDestination "$sourceWithoutPath" "$destinationWithoutPath" )" - # check if data path dir is already existing and create it if not + # check if data path dir is already existing and create it if not if [[ ! -e "$pathToDataPaths" ]]; then + mkdir -p "$pathToDataPaths" || return 1 fi - # check if data path is already existing + # check if data path is already existing if [[ -e "$pathToDataPaths/$dataPathName" ]]; then + return 2 fi - # create data path file and link alias to it + # create data path file and link alias to it if [[ "$dpathTemplate" == "" ]]; then + touch "$pathToDataPaths/$dataPathName" && \ ln -s "$dataPathName" "$pathToDataPaths/$dataPathAlias" && \ cat > "$pathToDataPaths/$dataPathAlias" <<-EOF @@ -357,42 +369,48 @@ createDataPath() EOF else dataPath=$( cat "$dpathTemplate" ) - # expand sourceWithoutPath + # expand sourceWithoutPath dataPath=$( echo "${dataPath//\$sourceWithoutPath/${sourceWithoutPath}}" ) - # expand destinationWithoutPath - dataPath=$( echo "${dataPath//\$destinationWithoutPath/${destinationWithoutPath}}" ) - + # expand destinationWithoutPath + dataPath=$( echo "${dataPath//\$destinationWithoutPath/${destinationWithoutPath}}" ) + touch "$pathToDataPaths/$dataPathName" && \ ln -s "$dataPathName" "$pathToDataPaths/$dataPathAlias" && \ echo "$dataPath" > "$pathToDataPaths/$dataPathAlias" fi - + if [[ $? -eq 0 ]]; then - # add source and destination to index files - - # NOTICE: If one provides the input for grep via stdin - #+ (/dev/null; then + echo "$sourceWithoutPath" >>"$pathToDataPaths/$__GLOBAL__sourcesIndexFile" fi + if ! grep "$destinationWithoutPath" "$pathToDataPaths/$__GLOBAL__destinationsIndexFile" &>/dev/null; then + echo "$destinationWithoutPath" >>"$pathToDataPaths/$__GLOBAL__destinationsIndexFile" fi + return else return 1 fi } + listDataPaths() { - # list available data paths + # list available data paths # - # usage: - #+ listDataPaths [-v] dataPathsDir + # usage: + # listDataPaths [-v] dataPathsDir local dataPathsDir="" local verboseExec=1 @@ -402,33 +420,36 @@ listDataPaths() local hashValue="" if [[ "$1" == "-v" ]]; then + verboseExec=0 - #echo "shifting" shift 1 fi - #echo "$1" - if [[ "$1" != "-v" && "$1" != "--verbose" && "$1" != "" ]]; then + dataPathsDir="$1" fi - #echo "$@ - $dataPathsDir" if [[ -e "$dataPathsDir" ]]; then + for dataPath in "$dataPathsDir"/*; do - # don't show links or backups (containing a '~' at the end of - #+ the filename) and just continue if there are no dpaths available. + + # don't show links or backups (containing a '~' at the end of + # the filename) and just continue if there are no dpaths available. if [[ ! -L "$dataPath" && \ "$dataPath" != *~ && \ "$dataPath" != *index && \ "$dataPath" != "${dataPathsDir}/*" \ ]]; then + source=$(xtractXMLAttributeValue "source" "$dataPath") destination=$(xtractXMLAttributeValue "destination" "$dataPath") + if [[ $verboseExec == 0 ]]; then + hashValue="$(hashSourceDestination $source $destination): " fi - + echo "${hashValue}${source} => ${destination}" fi done @@ -443,37 +464,38 @@ listDataPaths() listSources() { - # list all sources + # list all sources # - # usage: - #+ listSources dataPathsDir - + # usage: + # listSources dataPathsDir + local dataPathsDir="$1" local source="" if [[ -e "$dataPathsDir" ]]; then - # if index file is available, just print its contents + + # if index file is available, just print its contents if [[ -e "$dataPathsDir/$__GLOBAL__sourcesIndexFile" ]]; then + cat "$dataPathsDir/$__GLOBAL__sourcesIndexFile" else for dataPath in "$dataPathsDir"/*; do - # don't show links or backups (containing a '~' at the end of - #+ the filename) and just continue if there are no dpaths available. + + # don't show links or backups (containing a '~' at the end of + # the filename) and just continue if there are no dpaths available. + if [[ ! -L "$dataPath" && \ "$dataPath" != *~ && \ "$dataPath" != *index && \ "$dataPath" != "${dataPathsDir}/*" \ ]]; then source=$(xtractXMLAttributeValue "source" "$dataPath") - #sourceAlias=$(xtractXMLAttributeValue "source-alias" $dataPathsDir/$dataPath) echo "${source}" - #[[ ! -z "$sourceAlias" ]] && echo "$sourceAlias" fi done fi else - #echo "ERROR: \"$dataPathsDir\" not existing!" false fi @@ -483,129 +505,124 @@ listSources() listDestinations() { - # list all Destinations + # list all Destinations # - # usage: - #+ listDestinations dataPathsDir - + # usage: + # listDestinations dataPathsDir + local dataPathsDir="$1" local destination="" if [[ -e "$dataPathsDir" ]]; then - # if index file is available, just print its contents + + # if index file is available, just print its contents if [[ -e "$dataPathsDir/$__GLOBAL__destinationsIndexFile" ]]; then + cat "$dataPathsDir/$__GLOBAL__destinationsIndexFile" else for dataPath in "$dataPathsDir"/*; do - # don't show links or backups (containing a '~' at the end of - #+ the filename) and just continue if there are no dpaths available. + + # don't show links or backups (containing a '~' at the end of + # the filename) and just continue if there are no dpaths available. if [[ ! -L "$dataPath" && \ "$dataPath" != *~ && \ "$dataPath" != *index && \ "$dataPath" != "${dataPathsDir}/*" \ ]]; then destination=$(xtractXMLAttributeValue "destination" $dataPathsDir/$dataPath) - #destinationAlias=$(xtractXMLAttributeValue "destination-alias" $dataPathsDir/$dataPath) echo "$destination" - #[[ ! -z "$destinationAlias" ]] && echo "$destinationAlias" fi done fi else - #echo "ERROR: \"$dataPathsDir\" not existing!" false fi return } + xtractXMLAttributeValue() { - # determines the value between XML like tags + # determines the value between XML like tags # - # NOTICE: - #+ This function is limited to XML like files that have there tags in - #+ separate lines. - #+ - #+ Example: - #+ "value" doesn't work - #+ " - #+ value - #+ " works + # NOTICE: + # This function is limited to XML like files that have there tags in + # separate lines. # - # usage: - #+ xtractXMLAttributeValue attribute XMLFile + # Example: + # "value" doesn't work + # " + # value + # " works # - # attribute may contain arguments ('attribute arg="0"') or can be - #+ without + # usage: + # xtractXMLAttributeValue attribute XMLFile + # + # attribute may contain arguments ('attribute arg="0"') or can be + # without local attributeOpen="<$1>" - - #echoDebug "stderr" "DEBUG1" "Open: $attributeOpen" local attributeClose="<\/${1%% *}>" - #echoDebug "stderr" "DEBUG1" "Close: $attributeClose" - local XMLFile="$2" - #echoDebug "stderr" "DEBUG1" "$XMLFile" - - # extract everything between and incl. given attribute tags| remove tags + # extract everything between and incl. given attribute tags | remove tags sed -n -e "/$attributeOpen/,/$attributeClose/p" <"$XMLFile" | sed -e "/^<.*>$/d" } + retrieveDataPaths() { - # retrieves latest data paths available + # retrieves data paths available on [remote] repository # - # usage: - #+ retrieveDataPaths [-q] dataPathsDir + # usage: + # retrieveDataPaths [-q] dataPathsDir local dataPathsDir="" local verboseExec=0 - local wgetVerbose="" + local gucVerbose="" local tarVerbose="" if [[ "$1" == "-q" ]]; then + verboseExec=1 shift 1 fi if [[ verboseExec -eq 1 ]]; then - # make wget quiet - #wgetVerbose="-q" + gucVerbose="" + elif [[ verboseExec -eq 0 ]]; then - # make wget and tar verbose - #wgetVerbose="-v" + + # make guc and tar verbose gucVerbose="-v" tarVerbose="-v" fi if [[ "$1" != "-q" && "$1" != "" ]]; then + dataPathsDir="$1" fi if [[ ! -e "$dataPathsDir" ]]; then + mkdir -p "$dataPathsDir" fi - # retrieve data paths to data paths dir - #cd "$dataPathsDir" && \ - #wget $wgetVerbose "$dataPathsUrl" && \ - #tar $tarVerbose -xzf "$dataPathsUrlPkg" && \ - #rm "$dataPathsUrlPkg" - + # retrieve data paths to data paths dir export GLOBUS_FTP_CLIENT_SOURCE_PASV=1 - + cd "$dataPathsDir" && \ - globus-url-copy "$gucVerbose" "$dataPathsUrl" "file://$PWD/" && \ + globus-url-copy $gucVerbose "$dataPathsUrl" "file://$PWD/" && \ tar $tarVerbose -xzf "$dataPathsUrlPkg" && \ rm "$dataPathsUrlPkg" - + if [[ "$?" == "0" ]]; then + return 0 else return 1 @@ -613,26 +630,29 @@ retrieveDataPaths() } + use() { - # determines if a required tool/binary/etc. is available + # determines if a required tool/binary/etc. is available # - # usage: - #+ use "tool1" "tool2" "tool3" [...] + # usage: + # use "tool1" "tool2" "tool3" [...] local tools=$@ local requiredToolNotAvailable=1 for tool in $tools; do - #echo "$tool" + if ! hash $tool &>/dev/null; then + requiredToolNotAvailable=0 echo "ERROR: Required tool \"$tool\" can not be found!" fi done if [[ $requiredToolNotAvailable == 0 ]]; then + return 1 fi } @@ -640,10 +660,10 @@ use() reindexDataPaths() { - # list available data paths + # reindex (=(re)create index files for dpaths) data paths # - # usage: - #+ reindexDataPaths [-v] dataPathsDir + # usage: + # reindexDataPaths [-v] dataPathsDir local dataPathsDir="" @@ -652,34 +672,36 @@ reindexDataPaths() local hashValue="" if [[ "$1" == "-v" ]]; then + verboseExec=0 - #echo "shifting" shift 1 fi - #echo "$1" - if [[ "$1" != "-v" && "$1" != "--verbose" && "$1" != "" ]]; then + dataPathsDir="$1" fi - #echo "$@ - $dataPathsDir" if [[ -e "$dataPathsDir" ]]; then + for dataPath in "$dataPathsDir"/*; do - # don't show links or backups (containing a '~' at the end of - #+ the filename) + + # don't show links or backups (containing a '~' at the end of + # the filename) if [[ ! -L "$dataPath" && \ "$dataPath" != *~ && \ "$dataPath" != *.index* \ ]]; then source=$(xtractXMLAttributeValue "source" "$dataPath") destination=$(xtractXMLAttributeValue "destination" "$dataPath") + if [[ $verboseExec == 0 ]]; then + hashValue="$(hashSourceDestination $source $destination): " fi - + echo "${hashValue}${source} => ${destination}" - + echo "${source}" >> "$dataPathsDir/${__GLOBAL__sourcesIndexFile}.#dpath#tmp#" echo "${destination}" >> "$dataPathsDir/${__GLOBAL__destinationsIndexFile}.#dpath#tmp#" fi @@ -693,29 +715,32 @@ reindexDataPaths() sort -u "$dataPathsDir/${__GLOBAL__destinationsIndexFile}.#dpath#tmp#" > "$dataPathsDir/${__GLOBAL__destinationsIndexFile}" rm -f "$dataPathsDir/${__GLOBAL__sourcesIndexFile}.#dpath#tmp#" \ "$dataPathsDir/${__GLOBAL__destinationsIndexFile}.#dpath#tmp#" - + return } -#MAIN########################################################################### - -# test if all required tools are available +################################################################################ +# MAIN +################################################################################ +# test if all required tools are available if ! use "${__GLOBAL__requiredTools[@]}"; then + exit 1 fi -# correct number of params? +# correct number of params? if [[ "$#" -lt "1" ]]; then - # no, so output a usage message - usageMsg - exit 1 + + # no, so output a usage message + usageMsg + exit 1 fi # read in all parameters while [[ "$1" != "" ]]; do - # only valid params used? + # only valid params used? # # NOTICE: # This was added to prevent high speed loops @@ -738,256 +763,310 @@ while [[ "$1" != "" ]]; do "$1" != "--reindex" && \ "$1" != "--configfile" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg - exit 1 + exit 1 fi - # "--help" + # "--help" if [[ "$1" == "--help" ]]; then - if [[ "$helpMsgSet" != "0" ]]; then + + if [[ "$helpMsgSet" != "0" ]]; then + helpMsgSet="0" fi - + break - # "--version|-V" + # "--version|-V" elif [[ "$1" == "--version" || "$1" == "-V" ]]; then + versionMsg exit 0 - # "--verbose|-v" + # "--verbose|-v" elif [[ "$1" == "--verbose" || "$1" == "-v" ]]; then + if [[ $verboseExecSet != 0 ]]; then + shift 1 verboseExec=0 verboseExecSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--verbose|-v\" cannot be used multiple times!" exit 1 fi - # "--quiet|-q" + # "--quiet|-q" elif [[ "$1" == "--quiet" || "$1" == "-q" ]]; then + if [[ $quietExecSet != 0 ]]; then + shift 1 quietExec=0 quietExecSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--quiet|-q\" cannot be used multiple times!" exit 1 fi - - # "--reindex [/path/to/dataPaths]" + + # "--reindex [/path/to/dataPaths]" elif [[ "$1" == "--reindex" ]]; then + if [[ $reindexSet != 0 ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + reindexSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--reindex\" cannot be used multiple times!" exit 1 fi - # "--source|-s gsiftpSourceUrl" + # "--source|-s gsiftpSourceUrl" elif [[ "$1" == "--source" || "$1" == "-s" ]]; then + if [[ "$gsiftpSourceUrlSet" != "0" ]]; then + shift 1 gsiftpSourceUrl="$1" gsiftpSourceUrlSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--source|-s\" cannot be used multiple times!" exit 1 fi - # "--destination|-d gsiftpDestinationUrl" + # "--destination|-d gsiftpDestinationUrl" elif [[ "$1" == "--destination" || "$1" == "-d" ]]; then + if [[ "$gsiftpDestinationUrlSet" != "0" ]]; then + shift 1 gsiftpDestinationUrl="$1" gsiftpDestinationUrlSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--destination|-d\" cannot be used multiple times!" exit 1 fi - # "--create|-c [/path/to/files]" + # "--create|-c [/path/to/files]" elif [[ "$1" == "--create" || "$1" == "-c" ]]; then + if [[ "$createDataPathSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" ]]; then - # yes + + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + createDataPathSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--create|-c\" cannot be used multiple times!" exit 1 fi - - # "--batch-create|-b [/path/to/files]" + + # "--batch-create|-b [/path/to/files]" elif [[ "$1" == "--batch-create" || "$1" == "-b" ]]; then + if [[ "$batchCreateDataPathsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" ]]; then - # yes + + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + batchCreateDataPathsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--batch-create|-b\" cannot be used multiple times!" exit 1 fi - # "--list|-l [/path/to/dataPaths]" + # "--list|-l [/path/to/dataPaths]" elif [[ "$1" == "--list" || "$1" == "-l" ]]; then - if [[ "$listDataPathsSet" != "0" ]]; then + + if [[ "$listDataPathsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + listDataPathsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--list|-l\" cannot be used multiple times!" exit 1 fi - - # "--list-sources [/path/to/dataPaths]" + + # "--list-sources [/path/to/dataPaths]" elif [[ "$1" == "--list-sources" ]]; then + if [[ "$listSourcesSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then - # yes + + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + listSourcesSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--list-sources\" cannot be used multiple times!" exit 1 fi - # "--list-destinations [/path/to/dataPaths]" + # "--list-destinations [/path/to/dataPaths]" elif [[ "$1" == "--list-destinations" ]]; then + if [[ "$listDestinationsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then - # yes + + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + listDestinationsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--list-destinations\" cannot be used multiple times!" exit 1 fi - # "--retrieve|-r [/path/to/dataPaths]" + # "--retrieve|-r [/path/to/dataPaths]" elif [[ "$1" == "--retrieve" || "$1" == "-r" ]]; then + if [[ "$retrieveDataPathsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then - # yes + + # yes dataPathsDir="$1" shift 1 else dataPathsDir="" fi + retrieveDataPathsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--retrieve|-r\" cannot be used multiple times!" exit 1 fi - - # "--alias|-a alias" + + # "--alias|-a alias" elif [[ "$1" == "--alias" || "$1" == "-a" ]]; then + if [[ "$aliasSet" != "0" ]]; then + shift 1 alias="$1" aliasSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--alias|-a\" cannot be used multiple times!" exit 1 fi - # "--configfile configurationFile" + # "--configfile configurationFile" elif [[ "$1" == "--configfile" ]]; then + if [[ $dpathConfigurationFileSet != 0 ]]; then + shift 1 dpathConfigurationFile="$1" dpathConfigurationFileSet=0 shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--configfile\" cannot be used multiple times!" exit 1 fi - - # "--hosts|-h hostsFile" + + # "--hosts|-h hostsFile" elif [[ "$1" == "--hosts" || "$1" == "-h" ]]; then + if [[ $hostsFileSet != 0 ]]; then + shift 1 hostsFile="$1" hostsFileSet=0 shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--hosts\" cannot be used multiple times!" exit 1 fi - - # "[--dpath-template|-t dpathTemplate] + + # "[--dpath-template|-t dpathTemplate] elif [[ "$1" == "--dpath-template" || "$1" == "-t" ]]; then + if [[ $dpathTemplateSet != 0 ]]; then + shift 1 dpathTemplate="$1" dpathTemplateSet=0 shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--dpath-template\" cannot be used multiple times!" exit 1 fi @@ -995,104 +1074,124 @@ while [[ "$1" != "" ]]; do fi done -# load configuration file +# load configuration file if [[ -e "$dpathConfigurationFile" ]]; then + . "$dpathConfigurationFile" else echo "ERROR: dpath configuration file missing!" exit 1 fi -# HELP +# HELP if [[ "$helpMsgSet" == "0" ]]; then + helpMsg exit 0 -# REINDEX mode +# REINDEX mode elif [[ "$reindexSet" == "0" ]]; then + if [[ "$dataPathsDir" == "" ]]; then + dataPathsDir="$defaultDataPathsDir" fi - + if [[ "$verboseExecSet" == "0" ]]; then + reindexDataPaths -v "$dataPathsDir" else reindexDataPaths "$dataPathsDir" fi - + exit -# BATCH CREATE mode +# BATCH CREATE mode elif [[ "$batchCreateDataPathsSet" == "0" ]]; then + if [[ "$hostsFileSet" != "0" ]]; then - # no, so output a usage message + + # no, so output a usage message usageMsg exit 1 else if [[ "$dataPathsDir" == "" ]]; then + dataPathsDir="$defaultDataPathsDir" fi declare -a hosts hosts=( $( cat "$hostsFile" ) ) - + maxIndex=${#hosts[@]} - - #echo "($$) DEBUG: maxIndex=\"$maxIndex\"" 1>&2 - + for index in $( seq 0 $maxIndex ); do + for index2 in $( seq 0 $maxIndex ); do + if [[ $index -eq $index2 ]]; then + continue else gsiftpSourceUrl=${hosts[$index]} + if [[ "$gsiftpSourceUrl" == "" ]]; then + continue fi + gsiftpDestinationUrl=${hosts[$index2]} + if [[ "$gsiftpDestinationUrl" == "" ]]; then + continue fi + gsiftpSourceFqdn=$( echo "$gsiftpSourceUrl" | sed -e 's|^.*://||' -e 's/:.*$//' ) gsiftpDestinationFqdn=$( echo "$gsiftpDestinationUrl" | sed -e 's|^.*://||' -e 's/:.*$//' ) alias="${gsiftpSourceFqdn}--to--${gsiftpDestinationFqdn}" - + createDataPath "$gsiftpSourceUrl" "$gsiftpDestinationUrl" "$alias" "$dataPathsDir" "$dpathTemplate" returnValue="$?" if [[ "$returnValue" == "2" ]]; then + echo "ERROR: Data path file already exists. For changes please edit \"$dataPathsDir/$alias\" directly!" exit 1 + elif [[ "$returnValue" != "0" ]]; then + echo "ERROR: Problems during data path creation!" exit 1 else echo "INFO: Data path \"$dataPathsDir/$alias\" was created." fi - + fi done done - - # create index files - exit "$returnValue" - + # create index files + + exit "$returnValue" + fi -# CREATE mode +# CREATE mode elif [[ "$createDataPathSet" == "0" ]]; then + if [[ "$gsiftpSourceUrlSet" != "0" || \ "$gsiftpDestinationUrlSet" != "0" || \ "$aliasSet" != "0" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg exit 1 else if [[ "$dataPathsDir" == "" ]]; then + dataPathsDir="$defaultDataPathsDir" fi @@ -1101,13 +1200,16 @@ elif [[ "$createDataPathSet" == "0" ]]; then returnValue="$?" if [[ "$returnValue" == "2" ]]; then + echo "ERROR: Data path file already exists. For changes please edit \"$dataPathsDir/$alias\" directly!" exit 1 elif [[ "$returnValue" != "0" ]]; then + echo "ERROR: Problems during data path creation!" exit 1 else if [[ "$EDITOR" != "" ]]; then + $EDITOR $dataPathsDir/$alias echo "INFO: Data path \"$dataPathsDir/$alias\" was created." else @@ -1116,18 +1218,20 @@ elif [[ "$createDataPathSet" == "0" ]]; then fi exit "$?" - + fi -# LIST mode +# LIST mode elif [[ "$listDataPathsSet" == "0" ]]; then if [[ "$dataPathsDir" == "" ]]; then - # list both system and user dpaths + + # list both system and user dpaths if [[ "$verboseExecSet" == "0" ]]; then + echo "User dpaths ($defaultDataPathsDir):" listDataPaths -v "$defaultDataPathsDir" - echo "System dpaths ($systemDataPathsDir):" + echo "System dpaths ($systemDataPathsDir):" listDataPaths -v "$systemDataPathsDir" else echo "User dpaths:" @@ -1137,19 +1241,21 @@ elif [[ "$listDataPathsSet" == "0" ]]; then fi else if [[ "$verboseExecSet" == "0" ]]; then + listDataPaths -v "$dataPathsDir" else listDataPaths "$dataPathsDir" - fi - fi + fi + fi exit $? elif [[ "$listSourcesSet" == "0" ]]; then - + if [[ "$dataPathsDir" == "" ]]; then - # list all possible sources (from system and user dpaths) and - #+ eliminate double entries. + + # list all possible sources (from system and user dpaths) and + # eliminate double entries. ( listSources "$defaultDataPathsDir" listSources "$systemDataPathsDir" ) | sort -u else @@ -1161,8 +1267,9 @@ elif [[ "$listSourcesSet" == "0" ]]; then elif [[ "$listDestinationsSet" == "0" ]]; then if [[ "$dataPathsDir" == "" ]]; then - # list all possible destinations (from system and user dpaths) - #+ and eliminate double entries. + + # list all possible destinations (from system and user dpaths) + # and eliminate double entries. ( listDestinations "$defaultDataPathsDir" listDestinations "$systemDataPathsDir" ) | sort -u else @@ -1171,19 +1278,21 @@ elif [[ "$listDestinationsSet" == "0" ]]; then exit $? -# RETRIEVE mode +# RETRIEVE mode elif [[ "$retrieveDataPathsSet" == "0" ]]; then - if ! use wget tar; then - #echo "ERROR: Cannot run without required tools (wget, tar)! Exiting now!" + if ! use globus-url-copy tar; then + exit 1 fi if [[ "$dataPathsDir" == "" ]]; then + dataPathsDir="$defaultDataPathsDir" fi if [[ "$quietExecSet" == "0" ]]; then + retrieveDataPaths -q "$dataPathsDir" returnValue="$?" else @@ -1192,8 +1301,11 @@ elif [[ "$retrieveDataPathsSet" == "0" ]]; then fi if [[ "$returnValue" != "0" && "$quietExecSet" == "0" ]]; then + exit 1 + elif [[ "$returnValue" != "0" ]]; then + echo "ERROR: Problems during dpaths retrieval!" exit 1 else @@ -1203,6 +1315,4 @@ elif [[ "$retrieveDataPathsSet" == "0" ]]; then else usageMsg exit 1 - fi - From 511dcbcabc55106791e91b9e844ab18eb2b9a1ea Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 16 Mar 2017 10:18:16 +0100 Subject: [PATCH 10/21] Added explicit return value for use(). --- bin/datapath.bash | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/datapath.bash b/bin/datapath.bash index c93deb3..0595d27 100755 --- a/bin/datapath.bash +++ b/bin/datapath.bash @@ -654,6 +654,8 @@ use() if [[ $requiredToolNotAvailable == 0 ]]; then return 1 + else + return 0 fi } @@ -741,10 +743,6 @@ fi while [[ "$1" != "" ]]; do # only valid params used? - # - # NOTICE: - # This was added to prevent high speed loops - #+ if parameters are mispositioned. if [[ "$1" != "--help" && \ "$1" != "--version" && "$1" != "-V" && \ "$1" != "--create" && "$1" != "-c" && \ From 3788731344fd9ccb7ab4986fc947621e37712132 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 16 Mar 2017 10:58:22 +0100 Subject: [PATCH 11/21] Updated years in copyright statement. Removed old unneeded commented code and also changed format of comments and tabs --- bin/defaultparam.bash | 434 ++++++++++++++++++++++++------------------ 1 file changed, 246 insertions(+), 188 deletions(-) diff --git a/bin/defaultparam.bash b/bin/defaultparam.bash index e9aebf0..127c0bb 100755 --- a/bin/defaultparam.bash +++ b/bin/defaultparam.bash @@ -1,10 +1,10 @@ #!/bin/bash -# dparam - default params creation, listing, retrieving +# dparam - default params creation, listing, retrieving :</etc" ]]; then +#sed# #sed# gtransferConfigurationFilesPath="/etc" #sed# gtransferBasePath= #sed# gtransferLibPath="$gtransferBasePath/lib" -# According to FHS 2.3, configuration files for packages located in "/opt" have -#+ to be placed here (if you use a provider super dir below "/opt" for the -#+ gtransfer files, please also use the same provider super dir below -#+ "/etc/opt"). +# According to FHS 2.3, configuration files for packages located in "/opt" have +# to be placed here (if you use a provider super dir below "/opt" for the +# gtransfer files, please also use the same provider super dir below +# "/etc/opt"). #elif [[ -e "/etc/opt//gtransfer" ]]; then +# # gtransferConfigurationFilesPath="/etc/opt//gtransfer" # gtransferBasePath="/opt//gtransfer" # gtransferLibPath="$gtransferBasePath/lib" + elif [[ -e "/etc/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="/etc/opt/gtransfer" - gtransferBasePath="/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" -# For user install in $HOME: + gtransferConfigurationFilesPath="/etc/opt/gtransfer" + gtransferBasePath="/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + +# For user install in $HOME: elif [[ -e "$HOME/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="$HOME/.gtransfer" - gtransferBasePath="$HOME/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" -# For git deploy, use $BASH_SOURCE + gtransferConfigurationFilesPath="$HOME/.gtransfer" + gtransferBasePath="$HOME/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + +# For git deploy, use $BASH_SOURCE elif [[ -e "$( dirname $BASH_SOURCE )/../etc" ]]; then + gtransferConfigurationFilesPath="$( dirname $BASH_SOURCE )/../etc/gtransfer" gtransferBasePath="$( dirname $BASH_SOURCE )/../" gtransferLibPath="$gtransferBasePath/lib" @@ -72,10 +79,23 @@ fi dparamConfigurationFile="$gtransferConfigurationFilesPath/dparam.conf" -#USAGE########################################################################## +__GLOBAL__requiredTools=( cat + sha1sum + cut + mkdir + touch + ln + sed + tar + globus-url-copy ) + + +################################################################################ +# FUNCTIONS +################################################################################ usageMsg() { - cat < "$pathToDParams/$dParamAlias" <<-EOF @@ -286,14 +307,14 @@ createDParam() $destinationWithoutPath - + $( tgftp -s $sourceWithoutPath/dev/zero -t $destinationWithoutPath/dev/null -a ) EOF else - - # create data path file and link alias to it + + # create data path file and link alias to it touch "$pathToDParams/$dParamName" && \ ln -s "$dParamName" "$pathToDParams/$dParamAlias" && \ cat > "$pathToDParams/$dParamAlias" <<-EOF @@ -307,18 +328,19 @@ createDParam() # Enter params # EOF - + fi - return + return } + listDParams() { - # list available default params + # list available default params # - # usage: - #+ listDParams [-v] [dParamsDir] + # usage: + # listDParams [-v] [dParamsDir] local dParamsDir="" local verboseExec=1 @@ -328,23 +350,22 @@ listDParams() local hashValue="" if [[ "$1" == "-v" ]]; then + verboseExec=0 - #echo "shifting" shift 1 fi - #echo "$1" - if [[ "$1" != "-v" && "$1" != "--verbose" && "$1" != "" ]]; then + dParamsDir="$1" fi - #echo "$@ - $dataPathsDir" - if [[ -e "$dParamsDir" ]]; then + for dParam in "$dParamsDir"/*; do - # don't show links or backups (containing a '~' at the end of - #+ the filename) and just continue if there are no dprams available. + + # don't show links or backups (containing a '~' at the end of + # the filename) and just continue if there are no dprams available. if [[ ! -L "$dParam" && \ "$dParam" != *~ && \ "$dParam" != "${dParamsDir}/*" \ @@ -352,10 +373,12 @@ listDParams() source=$(xtractXMLAttributeValue "source" "$dParam") destination=$(xtractXMLAttributeValue "destination" "$dParam") dParams=$(xtractXMLAttributeValue "gsiftp_params" "$dParam") + if [[ $verboseExec == 0 ]]; then + hashValue="$(hashSourceDestination $source $destination): " fi - + echo "${hashValue}$source => $destination: \"$dParams\"" fi done @@ -367,113 +390,109 @@ listDParams() return } + xtractXMLAttributeValue() { - # determines the value between XML like tags + # determines the value between XML like tags # - # NOTICE: - #+ This function is limited to XML like files that have there tags in - #+ separate lines. - #+ - #+ Example: - #+ "value" doesn't work - #+ " - #+ value - #+ " works + # NOTICE: + # This function is limited to XML like files that have there tags in + # separate lines. # - # usage: - #+ xtractXMLAttributeValue attribute XMLFile + # Example: + # "value" doesn't work + # " + # value + # " works # - # attribute may contain arguments ('attribute arg="0"') or can be - #+ without + # usage: + # xtractXMLAttributeValue attribute XMLFile + # + # attribute may contain arguments ('attribute arg="0"') or can be + # without local attributeOpen="<$1>" - - #echoDebug "stderr" "DEBUG1" "Open: $attributeOpen" local attributeClose="<\/${1%% *}>" - #echoDebug "stderr" "DEBUG1" "Close: $attributeClose" - local XMLFile="$2" - #echoDebug "stderr" "DEBUG1" "$XMLFile" - - # extract everything between and incl. given attribute tags| remove tags + # extract everything between and incl. given attribute tags | remove tags sed -n -e "/$attributeOpen/,/$attributeClose/p" <"$XMLFile" | sed -e "/^<.*>$/d" } + retrieveDParams() { - # retrieves latest defautlt params available + # retrieves latest defautlt params available # - # usage: - #+ retrieveDataPaths [-q] dParamsDir + # usage: + # retrieveDataPaths [-q] dParamsDir local dParamsDir="" local verboseExec=0 - local wgetVerbose="" + local gucVerbose="" local tarVerbose="" if [[ "$1" == "-q" ]]; then + verboseExec=1 shift 1 fi if [[ verboseExec -eq 1 ]]; then - # make wget quiet - #wgetVerbose="-q" - gucVerbos="" + + gucVerbose="" + elif [[ verboseExec -eq 0 ]]; then - # make wget and tar verbose - #wgetVerbose="-v" + + # make guc and tar verbose gucVerbose="-v" tarVerbose="-v" fi if [[ "$1" != "-q" && "$1" != "" ]]; then + dParamsDir="$1" fi if [[ ! -e "$dParamsDir" ]]; then + mkdir -p "$dParamsDir" fi # retrieve data paths to data paths dir - #cd "$dParamsDir" && \ - #wget $wgetVerbose "$dParamsUrl" && \ - #tar $tarVerbose -xzf "$dParamsUrlPkg" && \ - #rm "$dParamsUrlPkg" - export GLOBUS_FTP_CLIENT_SOURCE_PASV=1 - + cd "$dParamsDir" && \ - globus-url-copy "$gucVerbose" "$dParamsUrl" "file://$PWD/" && \ + globus-url-copy $gucVerbose "$dParamsUrl" "file://$PWD/" && \ tar $tarVerbose -xzf "$dParamsUrlPkg" && \ rm "$dParamsUrlPkg" - + if [[ "$?" == "0" ]]; then + return 0 else return 1 fi - } + use() { - # determines if a required tool/binary/etc. is available + # determines if a required tool/binary/etc. is available # - # usage: - #+ use "tool1" "tool2" "tool3" [...] + # usage: + # use "tool1" "tool2" "tool3" [...] local tools=$@ local requiredToolNotAvailable=1 for tool in $tools; do - #echo "$tool" + if ! which $tool &>/dev/null; then + requiredToolNotAvailable=0 echo "ERROR: Required tool \"$tool\" can not be found!" fi @@ -484,273 +503,313 @@ use() fi } -#MAIN########################################################################### -# correct number of params? +################################################################################ +# MAIN +################################################################################ +# test if all required tools are available +if ! use "${__GLOBAL__requiredTools[@]}"; then + + exit 1 +fi + +# correct number of params? if [[ "$#" -lt "1" ]]; then - # no, so output a usage message - usageMsg - exit 1 + + # no, so output a usage message + usageMsg + exit 1 fi # read in all parameters while [[ "$1" != "" ]]; do # only valid params used? - # - # NOTICE: - # This was added to prevent high speed loops - #+ if parameters are mispositioned. - if [[ "$1" != "--help" && \ - "$1" != "--version" && "$1" != "-V" && \ - "$1" != "--create" && "$1" != "-c" && \ - "$1" != "--alias" && "$1" != "-a" && \ - "$1" != "--source" && "$1" != "-s" && \ - "$1" != "--destination" && "$1" != "-d" && \ - "$1" != "--verbose" && "$1" != "-v" && \ - "$1" != "--quiet" && "$1" != "-q" && \ - "$1" != "--list" && "$1" != "-l" && \ - "$1" != "--retrieve" && "$1" != "-r" && \ - "$1" != "--automatic" && \ - "$1" != "--configfile" \ + if [[ "$1" != "--help" && \ + "$1" != "--version" && "$1" != "-V" && \ + "$1" != "--create" && "$1" != "-c" && \ + "$1" != "--alias" && "$1" != "-a" && \ + "$1" != "--source" && "$1" != "-s" && \ + "$1" != "--destination" && "$1" != "-d" && \ + "$1" != "--verbose" && "$1" != "-v" && \ + "$1" != "--quiet" && "$1" != "-q" && \ + "$1" != "--list" && "$1" != "-l" && \ + "$1" != "--retrieve" && "$1" != "-r" && \ + "$1" != "--automatic" && \ + "$1" != "--configfile" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg - exit 1 + exit 1 fi - # "--help" + # "--help" if [[ "$1" == "--help" ]]; then - if [[ "$helpMsgSet" != "0" ]]; then + + if [[ "$helpMsgSet" != "0" ]]; then + helpMsgSet="0" fi - + break - # "--version|-V" + # "--version|-V" elif [[ "$1" == "--version" || "$1" == "-V" ]]; then + versionMsg exit 0 - # "--verbose|-v" + # "--verbose|-v" elif [[ "$1" == "--verbose" || "$1" == "-v" ]]; then + if [[ $verboseExecSet != 0 ]]; then + shift 1 verboseExec=0 verboseExecSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--verbose|-v\" cannot be used multiple times!" exit 1 fi - # "--quiet|-q" + # "--quiet|-q" elif [[ "$1" == "--quiet" || "$1" == "-q" ]]; then + if [[ $quietExecSet != 0 ]]; then + shift 1 quietExec=0 quietExecSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--quiet|-q\" cannot be used multiple times!" exit 1 fi - # "--automatic" + # "--automatic" elif [[ "$1" == "--automatic" ]]; then + if [[ $automaticExecSet != 0 ]]; then + shift 1 automaticExec=0 automaticExecSet=0 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--automatic\" cannot be used multiple times!" exit 1 fi - # "--source|-s gsiftpSourceUrl" + # "--source|-s gsiftpSourceUrl" elif [[ "$1" == "--source" || "$1" == "-s" ]]; then + if [[ "$gsiftpSourceUrlSet" != "0" ]]; then + shift 1 gsiftpSourceUrl="$1" gsiftpSourceUrlSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--source|-s\" cannot be used multiple times!" exit 1 fi - # "--destination|-d gsiftpDestinationUrl" + # "--destination|-d gsiftpDestinationUrl" elif [[ "$1" == "--destination" || "$1" == "-d" ]]; then + if [[ "$gsiftpDestinationUrlSet" != "0" ]]; then + shift 1 gsiftpDestinationUrl="$1" gsiftpDestinationUrlSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--destination|-d\" cannot be used multiple times!" exit 1 fi - # "--create|-c [/path/to/file]" + # "--create|-c [/path/to/file]" elif [[ "$1" == "--create" || "$1" == "-c" ]]; then + if [[ "$createDParamSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" ]]; then - # yes + + # yes dParamsDir="$1" shift 1 else dParamsDir="" fi - createDParamSet="0" + createDParamSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--create|-c\" cannot be used multiple times!" exit 1 fi - # "--list|-l [/path/to/file]" + # "--list|-l [/path/to/file]" elif [[ "$1" == "--list" || "$1" == "-l" ]]; then + if [[ "$listDParamsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then - # yes + + # yes dParamsDir="$1" shift 1 else dParamsDir="" fi + listDParamsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--list|-l\" cannot be used multiple times!" exit 1 fi - # "--retrieve|-r [/path/to/file]" + # "--retrieve|-r [/path/to/file]" elif [[ "$1" == "--retrieve" || "$1" == "-r" ]]; then + if [[ "$retrieveDParamsSet" != "0" ]]; then + shift 1 - # path provided? + + # path provided? if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then - # yes + + # yes dParamsDir="$1" shift 1 else dParamsDir="" fi + retrieveDParamsSet="0" else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--retrieve|-r\" cannot be used multiple times!" exit 1 fi - # "--alias|-a alias" + # "--alias|-a alias" elif [[ "$1" == "--alias" || "$1" == "-a" ]]; then + if [[ "$aliasSet" != "0" ]]; then + shift 1 alias="$1" aliasSet="0" shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--alias|-a\" cannot be used multiple times!" exit 1 fi - # "--configfile" + # "--configfile" elif [[ "$1" == "--configfile" ]]; then + if [[ $dparamConfigurationFileSet != 0 ]]; then + shift 1 dparamConfigurationFile="$1" dparamConfigurationFileSet=0 shift 1 else - # duplicate usage of this parameter + # duplicate usage of this parameter echo "ERROR: The parameter \"--configfile\" cannot be used multiple times!" exit 1 fi - - fi done - -# load configuration file +# load configuration file if [[ -e "$dparamConfigurationFile" ]]; then + . "$dparamConfigurationFile" else echo "ERROR: dparam configuration file missing!" exit 1 fi - -# HELP +# HELP if [[ "$helpMsgSet" == "0" ]]; then + helpMsg exit 0 -# CREATE mode +# CREATE mode elif [[ "$createDParamSet" == "0" ]]; then + if [[ "$gsiftpSourceUrlSet" != "0" || \ "$gsiftpDestinationUrlSet" != "0" || \ "$aliasSet" != "0" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg exit 1 else if [[ "$dParamsDir" == "" ]]; then + dParamsDir="$defaultDParamsDir" fi if [[ $automaticExecSet == 0 ]]; then - - createDParam "$gsiftpSourceUrl" "$gsiftpDestinationUrl" "$alias" "$dParamsDir" "-a" + createDParam "$gsiftpSourceUrl" "$gsiftpDestinationUrl" "$alias" "$dParamsDir" "-a" else - createDParam "$gsiftpSourceUrl" "$gsiftpDestinationUrl" "$alias" "$dParamsDir" - fi returnValue="$?" if [[ "$returnValue" == "2" ]]; then + echo "ERROR: Default params file already exists. For changes please edit \"$dParamsDir/$alias\" directly!" exit 1 + elif [[ "$returnValue" != "0" ]]; then + echo "ERROR: Problems during default params creation!" exit 1 else if [[ "$EDITOR" != "" && $automaticExecSet != 0 ]]; then + $EDITOR $dParamsDir/$alias echo "INFO: Default params \"$dParamsDir/$alias\" was created." elif [[ $automaticExecSet == 0 ]]; then + echo "INFO: Default params \"$dParamsDir/$alias\" was created" elif [[ $automaticExecSet != 0 ]]; then + echo "INFO: Default params \"$dParamsDir/$alias\" was created. Please use your preferred editor to edit the default params file." fi fi exit "$?" - fi -# LIST mode +# LIST mode elif [[ "$listDParamsSet" == "0" ]]; then if [[ "$dParamsDir" == "" ]]; then + if [[ "$verboseExecSet" == "0" ]]; then + echo "User dparams ($defaultDParamsDir):" listDParams -v "$defaultDParamsDir" echo "System dparams ($systemDParamsDir):" @@ -763,29 +822,31 @@ elif [[ "$listDParamsSet" == "0" ]]; then fi else if [[ "$verboseExecSet" == "0" ]]; then + listDParams -v "$dParamsDir" else listDParams "$dParamsDir" fi fi - - exit $? -# RETRIEVE mode +# RETRIEVE mode elif [[ "$retrieveDParamsSet" == "0" ]]; then if ! use wget tar; then + echo "ERROR: Cannot run without required tools (wget, tar)! Exiting now!" exit 1 fi if [[ "$dParamsDir" == "" ]]; then + dParamsDir="$defaultDParamsDir" fi if [[ "$quietExecSet" == "0" ]]; then + retrieveDParams -q "$dParamsDir" returnValue="$?" else @@ -794,8 +855,11 @@ elif [[ "$retrieveDParamsSet" == "0" ]]; then fi if [[ "$returnValue" != "0" && "$quietExecSet" == "0" ]]; then + exit 1 + elif [[ "$returnValue" != "0" ]]; then + echo "ERROR: Problems during dparams retrieval!" exit 1 else @@ -805,10 +869,4 @@ elif [[ "$retrieveDParamsSet" == "0" ]]; then else usageMsg exit 1 - fi - - - - - From 00bc782e94cb4ab7d8e2b9e979d542f17037ce85 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 16 Mar 2017 10:59:15 +0100 Subject: [PATCH 12/21] Added explicit return value for use(). --- bin/defaultparam.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/defaultparam.bash b/bin/defaultparam.bash index 127c0bb..9ec6055 100755 --- a/bin/defaultparam.bash +++ b/bin/defaultparam.bash @@ -499,7 +499,10 @@ use() done if [[ $requiredToolNotAvailable == 0 ]]; then + return 1 + else + return 0 fi } From b28534cc43b1fc7e577549e6b369fc2d24c6777c Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Wed, 22 Mar 2017 10:13:26 +0100 Subject: [PATCH 13/21] Implemented host alias retrieval functionality for halias tool. Also updated man page. Adapted halias config file for new funtionality. --- bin/halias.bash | 331 ++++++++++++++++++++++++++----------- etc/gtransfer/aliases.conf | 21 ++- share/doc/halias.1.md | 25 ++- share/man/man1/halias.1 | 49 +++--- 4 files changed, 298 insertions(+), 128 deletions(-) diff --git a/bin/halias.bash b/bin/halias.bash index a67aeb8..6c36624 100755 --- a/bin/halias.bash +++ b/bin/halias.bash @@ -3,7 +3,7 @@ :</etc" ]]; then +#sed# #sed# configurationFilesPath="/etc" #sed# basePath= #sed# libPath="$basePath/lib" -# According to FHS 2.3, configuration files for packages located in "/opt" have -#+ to be placed here (if you use a provider super dir below "/opt" for the -#+ gtransfer files, please also use the same provider super dir below -#+ "/etc/opt"). +# According to FHS 2.3, configuration files for packages located in "/opt" have +# to be placed here (if you use a provider super dir below "/opt" for the +# gtransfer files, please also use the same provider super dir below +# "/etc/opt"). elif [[ -e "/etc/opt/gtransfer" ]]; then - configurationFilesPath="/etc/opt/gtransfer" - basePath="/opt/gtransfer" - libPath="$basePath/lib" + + configurationFilesPath="/etc/opt/gtransfer" + basePath="/opt/gtransfer" + libPath="$basePath/lib" + #elif [[ -e "/etc/opt//gtransfer" ]]; then +# # configurationFilesPath="/etc/opt//gtransfer" # basePath="/opt//gtransfer" # libPath="$gtransferBasePath/lib" -# For user install in $HOME: +# For user install in $HOME: elif [[ -e "$HOME/opt/gtransfer" ]]; then - configurationFilesPath="$HOME/.gtransfer" - basePath="$HOME/opt/gtransfer" - libPath="$basePath/lib" -# For git deploy, use $BASH_SOURCE + configurationFilesPath="$HOME/.gtransfer" + basePath="$HOME/opt/gtransfer" + libPath="$basePath/lib" + +# For git deploy, use $BASH_SOURCE elif [[ -e "$( dirname $BASH_SOURCE )/../etc" ]]; then + configurationFilesPath="$( dirname $BASH_SOURCE )/../etc/gtransfer" basePath="$( dirname $BASH_SOURCE )/../" libPath="$basePath/lib" @@ -76,12 +81,13 @@ fi configurationFile="$configurationFilesPath/aliases.conf" -# Set $_LIB so halias and its libraries can find their includes +# Set $_LIB so halias and its libraries can find their includes readonly _LIB="$libPath" readonly _gtransfer_libraryPrefix="gtransfer" + ################################################################################ -# INCLUDE LIBRARY FUNCTIONS +# INCLUDE LIBRARY FUNCTIONS ################################################################################ . "$_LIB"/${_gtransfer_libraryPrefix}/exitCodes.bashlib @@ -91,11 +97,11 @@ readonly _gtransfer_libraryPrefix="gtransfer" ################################################################################ -# FUNCTIONS +# FUNCTIONS ################################################################################ usageMsg() { - cat < + + ...and store them in the user-provided path or - if no + additional path is given - in the user host aliases + directory. If a "-q" is provided, then output is omitted + and success/failure is only reported by the exit value. + --help Display this help and exit. -V, --version Output version information and exit. @@ -152,156 +169,282 @@ versionMsg() cat <<-VERSION $__GLOBAL__programName v${__GLOBAL__version} VERSION - + return } +# Public: List available host aliases. +# +# Returns 0 on success, 1 otherwise. halias/list() { local _userAliases="" local _systemAliases="" - if [[ -e "$__GLOBAL__userAliasesSource" ]]; then - _userAliases=$( alias/list "$__GLOBAL__userAliasesSource" ) + if [[ -e "$__CONFIG__userAliasesSource" ]]; then + + _userAliases=$( alias/list "$__CONFIG__userAliasesSource" ) fi - if [[ -e "$__GLOBAL__systemAliasesSource" ]]; then - _systemAliases=$( alias/list "$__GLOBAL__systemAliasesSource" ) + + if [[ -e "$__CONFIG__systemAliasesSource" ]]; then + + _systemAliases=$( alias/list "$__CONFIG__systemAliasesSource" ) fi if [[ ! -z "$_userAliases" && ! -z "$_systemAliases" ]]; then - + echo -e "$_userAliases\n$_systemAliases" | sort -u - + elif [[ ! -z "$_userAliases" ]]; then - + echo "$_userAliases" - + elif [[ ! -z "$_systemAliases" ]]; then - + echo "$_systemAliases" fi - - return + + return } +# Public: Check if given string is a host alias. +# +# $1 (_string) - Alias string to check +# +# Returns 0 on success (i.e. given string is an alias), 1 otherwise. halias/isAlias() { local _string="$1" - - # The library function checks for existence of aliases source, so not - # needed here. - #if [[ -e "$__GLOBAL__userAliasesSource" ]]; then - if alias/isAlias "$_string" "$__GLOBAL__userAliasesSource"; then - return 0 - fi - #fi - - #if [[ -e "$__GLOBAL__systemAliasesSource" ]]; then - if alias/isAlias "$_string" "$__GLOBAL__systemAliasesSource"; then - return 0 - fi - #fi - + + if alias/isAlias "$_string" "$__CONFIG__userAliasesSource"; then + return 0 + fi + + if alias/isAlias "$_string" "$__CONFIG__systemAliasesSource"; then + return 0 + fi + return 1 } +# Public: Resolve/dealiase the given string. If the given input string is not a +# host alias, it is just printed. If it is a host alias the dealiased +# value is printed. +# +# $1 (_string) - String to resolve/dealias. +# +# Returns 0 on success, 1 otherwise. halias/dealias() { local _string="$1" - + local _dealiasedString="" - - _dealiasedString=$( alias/dealias "$_string" "$__GLOBAL__userAliasesSource" ) - + + _dealiasedString=$( alias/dealias "$_string" "$__CONFIG__userAliasesSource" ) + # If _string is a user alias, _dealiasedString will differ from it and # we can return, as user aliases take precedence. if [[ $? -ne 1 && "$_dealiasedString" != "$_string" ]]; then + echo "$_dealiasedString" return 0 fi - - _dealiasedString=$( alias/dealias "$_string" "$__GLOBAL__systemAliasesSource" ) - + + _dealiasedString=$( alias/dealias "$_string" "$__CONFIG__systemAliasesSource" ) + # If _string is a system alias, _dealiasedString will differ from it... if [[ $? -ne 1 && "$_dealiasedString" != "$_string" ]]; then + echo "$_dealiasedString" return 0 fi - - # ...if not, we just return the input string + + # ...if not, we just print the input string echo "$_string" - + return } -################################################################################ + +# Public: Retrieve host aliases from configured (remote) repository. +# +# $1 (_quiet) - Quiet operation (true (=1)/false (=0)) +# $2 (_hostAliasesDir) - Where to store the host aliaes. +# +# __CONFIG__hostAliasesUrl - The URL of the host aliases packgage. +# __CONFIG__hostAliasesUrlPkg - The file name of the host aliases package (has +# to be a gzipped tar archive) +# +# Returns 0 on success, 1 otherwise. +halias/retrieveHostAliases() +{ + local _quiet="$1" + local _hostAliasesDir="$2" + + # default to quiet operation + local _gucVerbose="" + local _tarVerbose="" + + if [[ $_quiet -eq $_true ]]; then + + : # use defaults + else + # make guc and tar verbose + _gucVerbose="-v" + _tarVerbose="-v" + fi + + if [[ ! -e "$_hostAliasesDir" ]]; then + + mkdir -p "$_hostAliasesDir" + fi + + # retrieve host aliases to host aliases dir + export GLOBUS_FTP_CLIENT_SOURCE_PASV=1 + + cd "$_hostAliasesDir" + + if ! globus-url-copy $_gucVerbose "$__CONFIG__hostAliasesUrl" "file://$PWD/"; then + + # when failing, guc leaves an empty file in $_hostAliasesDir! + rm -f "$__CONFIG__hostAliasesUrlPkg" + return 1 + fi + + if ! ( tar $_tarVerbose -xzf "$__CONFIG__hostAliasesUrlPkg" && \ + rm "$__CONFIG__hostAliasesUrlPkg" ); then + + return 1 + fi + + return 0 +} + ################################################################################ -# MAIN +# MAIN ################################################################################ - -# load configuration file +# load configuration file if [[ -e "$configurationFile" ]]; then + . "$configurationFile" - # The following global vars are defined there: - # __GLOBAL__userAliasesSource - # __GLOBAL__systemAliasesSource + # The following configuration vars are defined there: + # __CONFIG__userAliasesSource + # __CONFIG__systemAliasesSource + # __CONFIG__hostAliasesUrl else echo "$__GLOBAL__programName: configuration file missing!" exit $_gtransfer_exit_software fi -# correct number of params? +# correct number of params? if [[ "$#" -lt "1" ]]; then - # no, so output a usage message - usageMsg - exit $_gtransfer_exit_usage + + # no, so output a usage message + usageMsg + exit $_gtransfer_exit_usage fi # read in all parameters while [[ "$1" != "" ]]; do - # only valid params used? + # only valid params used? if [[ "$1" != "--help" && \ "$1" != "--version" && "$1" != "-V" && \ "$1" != "--list" && "$1" != "-l" && \ "$1" != "--dealias" && "$1" != "-d" && \ - "$1" != "--is-alias" && "$1" != "-i" \ + "$1" != "--is-alias" && "$1" != "-i" && \ + "$1" != "--retrieve" && "$1" != "-r" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg exit $_gtransfer_exit_usage fi - # "--help" + # ""--help" if [[ "$1" == "--help" ]]; then + helpMsg exit $_gtransfer_exit_ok - # "--version|-V" + # "--version|-V" elif [[ "$1" == "--version" || "$1" == "-V" ]]; then + versionMsg exit $_gtransfer_exit_ok - # "--list|-l" + # "--list|-l" elif [[ "$1" == "--list" || "$1" == "-l" ]]; then + halias/list exit - + # "--is-alias|-i" elif [[ "$1" == "--is-alias" || "$1" == "-i" ]]; then + + _option="$1" shift 1 - halias/isAlias "$1" - exit - + + if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + + halias/isAlias "$1" + exit + else + echo "$__GLOBAL__programName: Missing argument for option \"$_option\"!" 1>&2 + usageMsg + exit $_gtransfer_exit_usage + fi + # "--dealias|-d" elif [[ "$1" == "--dealias" || "$1" == "-d" ]]; then + + _option="$1" shift 1 - halias/dealias "$1" - exit + + if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + + halias/dealias "$1" + exit + else + echo "$__GLOBAL__programName: Missing argument for option \"$_option\"!" 1>&2 + usageMsg + exit $_gtransfer_exit_usage + fi + + # "-r, --retrieve" + elif [[ "$1" == "--retrieve" || "$1" == "-r" ]]; then + + _option="$1" + shift 1 + + if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + + _hostAliasesDir="$1" + shift 1 + + if [[ "$1" == "-q" ]]; then + + halias/retrieveHostAliases "$_true" "$_hostAliasesDir" + else + halias/retrieveHostAliases "$_false" "$_hostAliasesDir" + fi + + exit + + elif [[ "$1" == "-q" ]]; then + + _hostAliasesDir="$__CONFIG__userAliasesSource" + halias/retrieveHostAliases "$_true" "$_hostAliasesDir" + + exit + else + _hostAliasesDir="$__CONFIG__userAliasesSource" + halias/retrieveHostAliases "$_false" "$_hostAliasesDir" + + exit + fi fi done - diff --git a/etc/gtransfer/aliases.conf b/etc/gtransfer/aliases.conf index edb4d5e..2301a34 100644 --- a/etc/gtransfer/aliases.conf +++ b/etc/gtransfer/aliases.conf @@ -6,13 +6,26 @@ # System aliases if [[ -e "$configurationFilesPath" ]]; then - __GLOBAL__systemAliasesSource="${configurationFilesPath}/aliases" + __CONFIG__systemAliasesSource="${configurationFilesPath}/aliases" # If the system aliases are located elsewhere locally, please configure the next -# two lines +# two lines #else -# __GLOBAL__systemAliasesSource= +# __CONFIG__systemAliasesSource= fi # User aliases -__GLOBAL__userAliasesSource="$HOME/.gtransfer/aliases" +__CONFIG__userAliasesSource="$HOME/.gtransfer/aliases" +# URL of host aliases package for retrieval +# This can be any valid globus-url-copy URL, e.g.: +# +# For remote repos: +# * "http[s]://host.domain.tld[:{80|443}]/path/to/host-aliases.tar.gz" +# * "gsiftp://host.domain.tld[:2811]/path/to/host-aliases.tar.gz" +# +# For a local repo: +# * "file:///path/to/host-aliases.tar.gz" +__CONFIG__hostAliasesUrl="/host-aliases.tar.gz" + +# Determine package name +__CONFIG__hostAliasesUrlPkg="$( basename "$__CONFIG__hostAliasesUrl" )" diff --git a/share/doc/halias.1.md b/share/doc/halias.1.md index 4769038..6db37d9 100644 --- a/share/doc/halias.1.md +++ b/share/doc/halias.1.md @@ -1,6 +1,6 @@ -% HALIAS(1) gtransfer 0.2.0 | User Commands +% HALIAS(1) halias 0.3.0 | User Commands % Frank Scheiner -% Aug 22, 2013 +% Jan 17, 2017 # NAME # @@ -10,7 +10,7 @@ # SYNOPSIS # -**halias [OPTION] _STRING_** +**halias [OPTION] [_STRING_]** # DESCRIPTION # @@ -30,19 +30,27 @@ The options are as follows: List all available host aliases. If a system alias and a user alias are identical, only one of both is shown. - + ## **-d, --dealias _STRING_** ## Expand a given string. If _STRING_ is not a host alias, then _STRING_ is just printed to STDOUT. - - + + ## **-i, --is-alias _STRING_** ## Check if a given string is a host alias. Returns 0 if yes, 1 otherwise. +## **-r, --retrieve [_/path/to/host-aliases_] [-q]** ## + +Retrieve host aliases from a repository configured in _[...]/halias.conf_ (see +below) and store them in the user-provided path or - if no additional path is +given - in the user host aliases directory. If a "-q" is provided, then output +is omitted and success/failure is only reported by the exit value. + + General options: @@ -57,8 +65,8 @@ Prints out version information. # FILES # - - + + ## _[...]/halias.conf_ ## The halias configuration file. The paths to system and user aliases directories @@ -78,4 +86,3 @@ This file can be a directory or a regular file and contains the user aliases. # SEE ALSO # **gtransfer(1)** - diff --git a/share/man/man1/halias.1 b/share/man/man1/halias.1 index bba6fca..bd904f8 100644 --- a/share/man/man1/halias.1 +++ b/share/man/man1/halias.1 @@ -1,61 +1,68 @@ -.TH HALIAS 1 "Aug 22, 2013" "gtransfer 0.2.0" "User Commands" +.TH "HALIAS" "1" "Jan 17, 2017" "halias 0.3.0" "User Commands" .SH NAME .PP -\f[B]halias\f[] - host alias helper utility +\f[B]halias\f[] \- host alias helper utility .SH SYNOPSIS .PP -\f[B]halias [OPTION] \f[I]STRING\f[]\f[] +\f[B]halias [OPTION] [\f[I]STRING\f[]]\f[] .SH DESCRIPTION .PP \f[B]halias\f[] (host alias) is a small helper utility providing an interface to the alias bashlib. -It can be used to list or expand host aliases and also to check if -a given string is an alias. +It can be used to list or expand host aliases and also to check if a +given string is an alias. You can define system aliases and user aliases. When dealiasing strings, user aliases take precedence over system aliases. .SH OPTIONS .PP The options are as follows: -.SS \f[B]-l, \[em]list\f[] +.SS \f[B]\-l, \-\-list\f[] .PP List all available host aliases. -If a system alias and a user alias are identical, only one of both -is shown. -.SS \f[B]-d, \[em]dealias \f[I]STRING\f[]\f[] +If a system alias and a user alias are identical, only one of both is +shown. +.SS \f[B]\-d, \-\-dealias \f[I]STRING\f[]\f[] .PP Expand a given string. -If \f[I]STRING\f[] is not a host alias, then \f[I]STRING\f[] is -just printed to STDOUT. -.SS \f[B]-i, \[em]is-alias \f[I]STRING\f[]\f[] +If \f[I]STRING\f[] is not a host alias, then \f[I]STRING\f[] is just +printed to STDOUT. +.SS \f[B]\-i, \-\-is\-alias \f[I]STRING\f[]\f[] .PP Check if a given string is a host alias. Returns 0 if yes, 1 otherwise. +.SS \f[B]\-r, \-\-retrieve [\f[I]/path/to/host\-aliases\f[]] [\-q]\f[] +.PP +Retrieve host aliases from a repository configured in +\f[I][...]/halias.conf\f[] (see below) and store them in the +user\-provided path or \- if no additional path is given \- in the user +host aliases directory. +If a "\-q" is provided, then output is omitted and success/failure is +only reported by the exit value. .PP General options: -.SS \f[B][--help]\f[] +.SS \f[B][\-\-help]\f[] .PP Prints out a help message. -.SS \f[B][-V, --version]\f[] +.SS \f[B][\-V, \-\-version]\f[] .PP Prints out version information. .SH FILES -.SS \f[I][\&...]/halias.conf\f[] +.SS \f[I][...]/halias.conf\f[] .PP The halias configuration file. The paths to system and user aliases directories can be configured there. -.SS \f[I][\&...]/aliases[/]\f[] +.SS \f[I][...]/aliases[/]\f[] .PP -This file can be a directory or a regular file and contains the -system aliases. +This file can be a directory or a regular file and contains the system +aliases. .SS \f[I]$HOME/.gtransfer/aliases[/]\f[] .PP -This file can be a directory or a regular file and contains the -user aliases. +This file can be a directory or a regular file and contains the user +aliases. .SH SEE ALSO .PP \f[B]gtransfer(1)\f[] .SH AUTHORS Frank Scheiner. - From 6dea0b056a7a6fc9ecfa0d5c40e411fa30fe33e8 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 23 Mar 2017 09:55:13 +0100 Subject: [PATCH 14/21] The (guc) sync level used by gtransfer for transfers can now be selected by users. Syncing can also be disabled. Adapted gt bash completion to include new options. Adapted gt manpage to include information about new options and sync levels. --- bin/gtransfer.bash | 261 +++++++++++++++++++---------- etc/bash_completion.d/gtransfer.sh | 108 ++++++------ lib/gtransfer/listTransfer.bashlib | 155 +++++++++-------- share/doc/gtransfer.1.md | 79 ++++++--- share/man/man1/gtransfer.1 | 61 +++++-- 5 files changed, 411 insertions(+), 253 deletions(-) diff --git a/bin/gtransfer.bash b/bin/gtransfer.bash index b03f80f..f6f9a84 100755 --- a/bin/gtransfer.bash +++ b/bin/gtransfer.bash @@ -6,7 +6,7 @@ :</etc" ]]; then +#sed# #sed# gtransferConfigurationFilesPath="/etc" #sed# gtransferBasePath= #sed# gtransferLibPath="$gtransferBasePath/lib" @@ -64,25 +66,29 @@ if [[ -e "/etc/gtransfer" ]]; then #+ gtransfer files, please also use the same provider super dir below #+ "/etc/opt"). #elif [[ -e "/etc/opt//gtransfer" ]]; then +# # gtransferConfigurationFilesPath="/etc/opt//gtransfer" # gtransferBasePath="/opt//gtransfer" # gtransferLibPath="$gtransferBasePath/lib" elif [[ -e "/etc/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="/etc/opt/gtransfer" - gtransferBasePath="/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" - gtransferLibexecPath="$gtransferBasePath/libexec" + + gtransferConfigurationFilesPath="/etc/opt/gtransfer" + gtransferBasePath="/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + gtransferLibexecPath="$gtransferBasePath/libexec" # For user install in $HOME: #elif [[ -e "$HOME/.gtransfer" ]]; then elif [[ -e "$HOME/opt/gtransfer" ]]; then - gtransferConfigurationFilesPath="$HOME/.gtransfer" - gtransferBasePath="$HOME/opt/gtransfer" - gtransferLibPath="$gtransferBasePath/lib" - gtransferLibexecPath="$gtransferBasePath/libexec" + + gtransferConfigurationFilesPath="$HOME/.gtransfer" + gtransferBasePath="$HOME/opt/gtransfer" + gtransferLibPath="$gtransferBasePath/lib" + gtransferLibexecPath="$gtransferBasePath/libexec" # For git deploy, use $BASH_SOURCE elif [[ -e "$( dirname $BASH_SOURCE )/../etc" ]]; then + gtransferConfigurationFilesPath="$( dirname $BASH_SOURCE )/../etc/gtransfer" gtransferBasePath="$( dirname $BASH_SOURCE )/../" gtransferLibPath="$gtransferBasePath/lib" @@ -144,7 +150,7 @@ done #USAGE########################################################################## usageMsg() { - cat <&2 else echo "$debugLevel: $debugString" 1>$fd - fi + fi return } @@ -377,8 +409,9 @@ helperFunctions/use cat grep sed cut sleep tgftp telnet uberftp || exit "$_gtran # Defaults ##################################################################### -gtMaxRetries="3" -gucMaxRetries="1" +__GLOBAL__gtMaxRetries="3" +__GLOBAL__gucMaxRetries="1" +__GLOBAL__syncLevel="1" gtProgressIndicator="." gtInstance="$gtProgressIndicator" @@ -389,6 +422,8 @@ tgftpLogfileNameSet="1" recursiveTransferSet=1 _checksumDataChannelSet=1 _encryptDataChannelSet=1 +_syncLevelSet=1 +_noSyncSet=1 ################################################################################ @@ -408,31 +443,29 @@ fi # read in all parameters while [[ "$1" != "" ]]; do - # only valid params used? - # - # NOTICE: - # This was added to prevent high speed loops - #+ if parameters are mispositioned. - if [[ "$1" != "--help" && \ - "$1" != "--version" && "$1" != "-V" && \ - "$1" != "--source" && "$1" != "-s" && \ - "$1" != "--destination" && "$1" != "-d" && \ - "$1" != "--metric" && "$1" != "-m" && \ - "$1" != "--verbose" && "$1" != "-v" && \ - "$1" != "--auto-clean" && "$1" != "-a" && \ - "$1" != "--logfile" && "$1" != "-l" && \ - "$1" != "--configfile" && \ - "$1" != "--guc-max-retries" && \ - "$1" != "--gt-max-retries" && \ - "$1" != "--transfer-list" && "$1" != "-f" && \ - "$1" != "--gt-progress-indicator" && \ - "$1" != "--auto-optimize" && "$1" != "-o" && \ - "$1" != "--recursive" && "$1" != "-r" && \ - "$1" != "--checksum-data-channel" && "$1" != "-c" && \ - "$1" != "--encrypt-data-channel" && "$1" != "-e" && \ - "$1" != "--" \ + # only valid params used? + if [[ "$1" != "--help" && \ + "$1" != "--version" && "$1" != "-V" && \ + "$1" != "--source" && "$1" != "-s" && \ + "$1" != "--destination" && "$1" != "-d" && \ + "$1" != "--metric" && "$1" != "-m" && \ + "$1" != "--verbose" && "$1" != "-v" && \ + "$1" != "--auto-clean" && "$1" != "-a" && \ + "$1" != "--logfile" && "$1" != "-l" && \ + "$1" != "--configfile" && \ + "$1" != "--guc-max-retries" && \ + "$1" != "--gt-max-retries" && \ + "$1" != "--transfer-list" && "$1" != "-f" && \ + "$1" != "--gt-progress-indicator" && \ + "$1" != "--auto-optimize" && "$1" != "-o" && \ + "$1" != "--recursive" && "$1" != "-r" && \ + "$1" != "--checksum-data-channel" && "$1" != "-c" && \ + "$1" != "--encrypt-data-channel" && "$1" != "-e" && \ + "$1" != "--sync-level" && \ + "$1" != "--no-sync" && \ + "$1" != "--" \ ]]; then - # no, so output a usage message + # no, so output a usage message usageMsg exit $_gtransfer_exit_usage fi @@ -446,7 +479,7 @@ while [[ "$1" != "" ]]; do gsiftpUserParams="$@" # exit the loop (this assumes that everything left in "$@" is - #+ a "globus-url-copy" param). + #+ a "globus-url-copy" param). break # "--help" ############################################################ @@ -497,24 +530,26 @@ while [[ "$1" != "" ]]; do autoOptimizeSet="0" shift 1 - # "--guc-max-retries gucMaxRetries" ################################### + # "--guc-max-retries gucMaxRetries" ################################### elif [[ "$1" == "--guc-max-retries" ]]; then shift 1 gucMaxRetries="$1" + __GLOBAL__gucMaxRetries="$gucMaxRetries" gucMaxRetriesSet="0" shift 1 - # "--gt-max-retries gtMaxRetries" ##################################### + # "--gt-max-retries gtMaxRetries" ##################################### elif [[ "$1" == "--gt-max-retries" ]]; then shift 1 gtMaxRetries="$1" + __GLOBAL__gtMaxRetries="$gtMaxRetries" gtMaxRetriesSet="0" shift 1 - # "--gt-progress-indicator indicatorCharacter" ######################## - elif [[ "$1" == "--gt-progress-indicator" ]]; then + # "--gt-progress-indicator indicatorCharacter" ######################## + elif [[ "$1" == "--gt-progress-indicator" ]]; then shift 1 gtProgressIndicator="$1" @@ -534,7 +569,7 @@ while [[ "$1" != "" ]]; do shift 1 verboseExecSet=0 - + # "--recursive|-r" #################################################### elif [[ "$1" == "--recursive" || "$1" == "-r" ]]; then @@ -549,8 +584,8 @@ while [[ "$1" != "" ]]; do shift 1 _checksumDataChannelSet=0 else - echo "${_program}: The parameter \"--checksum-data-channel|-c\" cannot be used in conjunction with the parameter \"--encrypt-data-channel|-e\"!" - echo "Try \`${_program} --help' for more information." + echo "${_program}: The parameter \"--checksum-data-channel|-c\" cannot be used in conjunction with the parameter \"--encrypt-data-channel|-e\"!" 1>&2 + echo "Try \`${_program} --help' for more information." 1>&2 exit $_gtransfer_exit_usage fi @@ -562,8 +597,57 @@ while [[ "$1" != "" ]]; do shift 1 _encryptDataChannelSet=0 else - echo "${_program}: The parameter \"--encrypt-data-channel|-e\" cannot be used in conjunction with the parameter \"--checksum-data-channel|-c\"!" - echo "Try \`${_program} --help' for more information." + echo "${_program}: The parameter \"--encrypt-data-channel|-e\" cannot be used in conjunction with the parameter \"--checksum-data-channel|-c\"!" 1>&2 + echo "Try \`${_program} --help' for more information." 1>&2 + exit $_gtransfer_exit_usage + fi + + # "--sync-level" ####################################################### + elif [[ "$1" == "--sync-level" ]]; then + + _option="$1" + shift 1 + + if [[ $_noSyncSet -ne 0 ]]; then + + if [[ "${1:0:1}" != "-" && "$1" != "" ]]; then + + if [[ "$1" =~ ^[0-3]$ && \ + $1 -ge 0 && \ + $1 -le 4 ]]; then + + _syncLevel="$1" + shift 1 + __GLOBAL__syncLevel="$_syncLevel" + _syncLevelSet=0 + else + echo "${_program}: Sync level has to be either 0, 1, 2 or 3!" 1>&2 + echo "Try \`${_program} --help' for more information." 1>&2 + exit $_gtransfer_exit_usage + fi + else + echo "${_program}: Missing argument for option \"$_option\"!" 1>&2 + usageMsg + exit $_gtransfer_exit_usage + fi + else + echo "${_program}: The option \`$_option' cannot be used in conjunction with the option \`--no-sync'!" 1>&2 + echo "Try \`${_program} --help' for more information." 1>&2 + exit $_gtransfer_exit_usage + fi + + # "--no-sync" ########################################################## + elif [[ "$1" == "--no-sync" ]]; then + + _option="$1" + shift 1 + + if [[ $_syncLevelSet -ne 0 ]]; then + + _noSyncSet=0 + else + echo "${_program}: The option \`$_option' cannot be used in conjunction with the option \`--sync-level'!" 1>&2 + echo "Try \`${_program} --help' for more information." 1>&2 exit $_gtransfer_exit_usage fi @@ -647,7 +731,7 @@ if [[ "$gsiftpSourceUrl" == "" || \ # create directory for temp files mkdir -p "$__GLOBAL__gtTmpDir" - + gsiftpTransferListClean="$__GLOBAL__gtTmpDir/$$_transferList.${__GLOBAL__gtTmpSuffix}" # do not remove commented lines, as guc stores directories in commented lines and if # those are empty, meaning no other uncommented source or destination URL contains a @@ -730,7 +814,7 @@ else # create directory for temp files mkdir -p "$__GLOBAL__gtTmpDir" - # dealias URLs + # dealias URLs if hash halias &>/dev/null; then # remove everything after and including the first "/". As an alias # mustn't contain any forward slashes, if an alias is used in the URLs, @@ -749,7 +833,7 @@ else _originalGsiftpSourceUrl="$gsiftpSourceUrl" _originalGsiftpDestinationUrl="$gsiftpDestinationUrl" - + _tmpSourceAliasValue=$( halias --dealias "$_tmpSourceAlias" ) if [[ $? != 0 ]]; then @@ -768,7 +852,7 @@ else # check if the alias value is different from the alias itself if [[ "$_tmpSourceAlias" != "$_tmpSourceAliasValue" ]]; then - + _tmpSourcePath=${gsiftpSourceUrl#*\/} if [[ "$_tmpSourceUser" != "$_tmpSourceAlias" ]]; then @@ -779,9 +863,9 @@ else gsiftpSourceUrl="${_tmpSourceAliasValue}/${_tmpSourcePath}" fi fi - + if [[ "$_tmpDestinationAlias" != "$_tmpDestinationAliasValue" ]]; then - + _tmpDestinationPath=${gsiftpDestinationUrl#*\/} if [[ "$_tmpDestinationUser" != "$_tmpDestinationAlias" ]]; then @@ -796,49 +880,49 @@ else # Handle persistent identifiers (PIDs) as source if [[ "$gsiftpSourceUrl" =~ 'pid://' ]]; then - + if hash irule &>/dev/null; then _pid="${gsiftpSourceUrl/pid:\/\/}" - + # resolve PID _resolvedUrl=$( pids/irodsMicroService/resolvePid "$_pid" ) - + if [[ $? != 0 ]]; then echo "${_program} [${gtInstance}]: Given PID \"$_pid\" could not be resolved. Exiting." 1>&2 exit $_gtransfer_exit_software fi - + gsiftpSourceUrl="$_resolvedUrl" else echo "${_program} [${gtInstance}]: Cannot resolve PID without \"irule\" tool. Exiting" 1>&2 exit $_gtransfer_exit_software fi - # Handle a list of persistent identifiers (PIDs) as source + # Handle a list of persistent identifiers (PIDs) as source elif [[ "$gsiftpSourceUrl" =~ 'pidfile://' ]]; then - + if hash irule &>/dev/null; then _pidFile="${gsiftpSourceUrl/pidfile:\/\/}" - + if [[ ! -e "$_pidFile" ]]; then echo "${_program} [${gtInstance}]: PID file \"$_pidFile\" cannot be found! Exiting." 1>&2 exit $_gtransfer_exit_usage fi - + helperFunctions/echoIfVerbose "${_program} [${gtInstance}]: Resolving PID file..." _resolvedPids=$( pids/irodsMicroService/resolvePidFile "$_pidFile" ) - + if [[ $? -ne 0 ]]; then echo "${_program} [${gtInstance}]: At least one PID could not be resolved." fi - + if [[ -z "$_resolvedPids" ]]; then echo "${_program} [${gtInstance}]: PIDs in PID file \"$_pidFile\" could not be resolved! Exiting." 1>&2 exit $_gtransfer_exit_software fi - + helperFunctions/echoIfVerbose "${_program} [${gtInstance}]: Building transfer list..." gsiftpTransferList=$( pids/irodsMicroService/buildTransferList "$_resolvedPids" "$gsiftpDestinationUrl" ) - + # exchange source URL specification with transfer list # spec _modifiedGtCommandLine="${__GLOBAL__gtCommandLine/$gsiftpSourceUrl/ -f $gsiftpTransferList}" @@ -851,14 +935,14 @@ else # remove any destination option _modifiedGtCommandLine="${_modifiedGtCommandLine/ -d }" _modifiedGtCommandLine="${_modifiedGtCommandLine/ --destination }" - + # call new gt instance to perform a list transfer $_modifiedGtCommandLine exit $? else echo "${_program} [${gtInstance}]: Cannot resolve PIDs without \"irule\" tool. Exiting." 1>&2 exit $_gtransfer_exit_software - fi + fi fi # automatically strips commend lines! @@ -962,4 +1046,3 @@ if [[ $autoClean == 0 ]]; then fi exit $transferDataReturnValue - diff --git a/etc/bash_completion.d/gtransfer.sh b/etc/bash_completion.d/gtransfer.sh index 36211b9..fa3b8b1 100644 --- a/etc/bash_completion.d/gtransfer.sh +++ b/etc/bash_completion.d/gtransfer.sh @@ -1,6 +1,6 @@ :</dev/null; then echo "/" return fi - + local path=${url#*/} - + # strip any file portion from path path=${path%/*} # strip any file portion from path #path=$(echo $path | grep -o '/.*/') - + if [[ "$path" != "" ]]; then echo "/$path/" else @@ -85,7 +85,7 @@ _gtransfer() fi #if [[ $path == "" ]]; then # path="/" - #fi + #fi # #echo "$path" } @@ -102,12 +102,12 @@ _gtransfer() #+ getURLWithoutPath "URL" local URL="$1" - + # TODO: #+ support URLs not containing any port descriptions: # # done! - + :<<-COMMENT from: " @@ -204,19 +204,19 @@ _gtransfer() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" - # all available gtransfer options/switches/parameters - opts="--source -s --destination -d --transfer-list -f --auto-optimize -o --recursive -r --checksum-data-channel -c --encrypt-data-channel -e --verbose -v --metric -m --logfile -l --auto-clean -a --configfile --guc-max-retries --gt-max-retries --gt-progress-indicator --help --version -V --" + # all available gtransfer options/switches/parameters + opts="--source -s --destination -d --transfer-list -f --auto-optimize -o --recursive -r --checksum-data-channel -c --encrypt-data-channel -e --sync-level --no-sync --verbose -v --metric -m --logfile -l --auto-clean -a --configfile --guc-max-retries --gt-max-retries --gt-progress-indicator --help --version -V --" # parameter completion case "${prev}" in - + --source|-s) # only complete remote paths if globus-url-copy is #+ available if hash globus-url-copy &>/dev/null; then # complete remote paths if echo "$cur" | grep '^gsiftp://.*:.*' &>/dev/null; then - + userhost=$( getURLWithoutPath "${cur}" ) #echo -e "\n$userhost" 1>&2 userpath=$( getPathFromURL "${cur}" ) @@ -225,31 +225,31 @@ _gtransfer() local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 - + elif echo "$cur" | grep '^ftp://.*:.*' &>/dev/null; then - + userhost=$( getURLWithoutPath "${cur}" ) userpath=$( getPathFromURL "${cur}" ) - + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) - + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 fi - + if hash halias &>/dev/null; then - + alias="${cur%%/*}" ## remove path user="${alias%%@*}" alias="${alias#*@}" ## remove "user@" - - if halias --is-alias "$alias" &>/dev/null; then - + + if halias --is-alias "$alias" &>/dev/null; then + userhost=$( halias --dealias "$alias" ) if [[ "$user" != "$alias" ]]; then @@ -259,15 +259,15 @@ _gtransfer() #echo -e "\n$userhost A$alias U$user" 1>&2 userpath=$( getPathFromAliasUrl "$cur" ) #echo "$userpath" 1>&2 - + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) - + if [[ "$user" != "$alias" ]]; then local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${user}@${alias}${userpath}${path}; done ) else local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${alias}${userpath}${path}; done ) fi - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 fi @@ -281,13 +281,13 @@ _gtransfer() else local sites="" fi - + if hash halias &>/dev/null; then local aliases=$( halias --list ) else local aliases="" fi - + if [[ "$sites" != "" && "$aliases" != "" ]]; then COMPREPLY=( $(compgen -W "${sites} ${aliases}" -- ${cur}) ) elif [[ "$sites" != "" ]]; then @@ -295,7 +295,7 @@ _gtransfer() elif [[ "$aliases" != "" ]]; then COMPREPLY=( $(compgen -W "${aliases}" -- ${cur}) ) fi - + return 0 ;; @@ -305,7 +305,7 @@ _gtransfer() if hash globus-url-copy &>/dev/null; then # complete remote paths if echo "$cur" | grep '^gsiftp://.*:.*' &>/dev/null; then - + userhost=$( getURLWithoutPath "${cur}" ) #echo -e "\n$userhost" 1>&2 userpath=$( getPathFromURL "${cur}" ) @@ -314,31 +314,31 @@ _gtransfer() local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 - + elif echo "$cur" | grep '^ftp://.*:.*' &>/dev/null; then - + userhost=$( getURLWithoutPath "${cur}" ) userpath=$( getPathFromURL "${cur}" ) - + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) - + local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${userhost}${userpath}${path}; done ) - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 fi - + if hash halias &>/dev/null; then alias="${cur%%/*}" user="${alias%%@*}" alias="${alias#*@}" ## remove "user@" - + if halias --is-alias "$alias" &>/dev/null; then - + userhost=$( halias --dealias "$alias" ) if [[ "$user" != "$alias" ]]; then @@ -348,20 +348,20 @@ _gtransfer() #echo -e "\n$userhost" 1>&2 userpath=$( getPathFromAliasUrl "$cur" ) #echo "$userpath" 1>&2 - + local remote_paths=( $( globus-url-copy -list "${userhost}${userpath}*" | sed -e 's/^\ *//' -e 1d ) ) - + if [[ "$user" != "$alias" ]]; then local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${user}@${alias}${userpath}${path}; done ) else local remote_urls=$( for path in "${remote_paths[@]}"; do echo ${alias}${userpath}${path}; done ) fi - + COMPREPLY=( $(compgen -W "${remote_urls}" -- ${cur}) ) return 0 fi fi - fi + fi # only complete source URL host parts if dpath is available if hash dpath &>/dev/null; then @@ -370,13 +370,13 @@ _gtransfer() else local sites="" fi - + if hash halias &>/dev/null; then local aliases=$( halias --list ) else local aliases="" fi - + if [[ "$sites" != "" && "$aliases" != "" ]]; then COMPREPLY=( $(compgen -W "${sites} ${aliases}" -- ${cur}) ) elif [[ "$sites" != "" ]]; then @@ -384,14 +384,21 @@ _gtransfer() elif [[ "$aliases" != "" ]]; then COMPREPLY=( $(compgen -W "${aliases}" -- ${cur}) ) fi - + return 0 ;; --metric|-m) # propose metrics 0 to 3 - local metrics=$(for x in 0 1 2 3; do echo $x; done ) - COMPREPLY=( $(compgen -W "${metrics}" -- ${cur}) ) + local metrics=$( for x in 0 1 2 3; do echo $x; done ) + COMPREPLY=( $( compgen -W "${metrics}" -- ${cur} ) ) + return 0 + ;; + + --sync-level) + # propose sync levels 0 to 3 + local syncLevels=$( for x in 0 1 2 3; do echo $x; done ) + COMPREPLY=( $( compgen -W "${syncLevels}" -- ${cur} ) ) return 0 ;; @@ -405,4 +412,3 @@ _gtransfer() } complete -o nospace -F _gtransfer gtransfer gt - diff --git a/lib/gtransfer/listTransfer.bashlib b/lib/gtransfer/listTransfer.bashlib index dcdc478..aec69cc 100644 --- a/lib/gtransfer/listTransfer.bashlib +++ b/lib/gtransfer/listTransfer.bashlib @@ -4,7 +4,7 @@ :< "$_sortedTransferList" @@ -121,10 +121,10 @@ listTransfer/dpathAvailable() { local _source="$1" local _destination="$2" - + local _sourceWithoutPath=$( helperFunctions/getURLWithoutPath "$_source" ) local _destinationWithoutPath=$( helperFunctions/getURLWithoutPath "$_destination" ) - + # get corresponding dpath name (and remove any "username@" portions in #+ the URL before hashing). local _dpathFilename=$( helperFunctions/hashSourceDestination $( echo $_sourceWithoutPath | sed -e 's/:\/\/.*@/:\/\//' ) $( echo $_destinationWithoutPath | sed -e 's/:\/\/.*@/:\/\//' ) ) @@ -208,7 +208,7 @@ listTransfer/getDpathData() { local _source="$1" local _destination="$2" - + local _metric="$3" local _sourceWithoutPath=$( helperFunctions/getURLWithoutPath "$_source" ) @@ -220,29 +220,29 @@ listTransfer/getDpathData() #+ despite the fact that no dpath is available (it should be "1" then!). local _dpathfile _dpathFile=$( listTransfer/dpathAvailable "$_source" "$_destination" ) - + local _functionReturnValue=$? if [[ $_functionReturnValue -eq 0 ]]; then - + #helperFunctions/echoIfVerbose -e "Dpath used:\n$_dPathFile" - + local -a _dpathArray=( $( helperFunctions/xtractXMLAttributeValue "path .*metric=\"$_metric\"" $_dpathFile ) ) - + echo ${_dpathArray[@]} - + return 0 - + elif [[ $_functionReturnValue -eq 1 ]]; then - + #helperFunctions/echoIfVerbose -e "No dpath available. Performing direct transfer." - + local -a _dpathArray[0]="$_sourceWithoutPath;$_destinationWithoutPath" - + echo ${_dpathArray[@]} - + return 0 - else + else return 1 fi } @@ -264,10 +264,10 @@ listTransfer/getTransferParameters() local -a _dpathArray=($1) for _sourceDestination in ${_dpathArray[@]}; do - + local _source="${_sourceDestination%%;*}" local _destination="${_sourceDestination##*;}" - + local _dparamFile _dparamFile=$( listTransfer/dparamAvailable "$_source" "$_destination" ) @@ -309,9 +309,9 @@ listTransfer/getTransferParameters() #+ default parameters _unescapedTransferParameters="$gsiftpDefaultParams" fi - + #local _functionReturnValue=$? - # + # #if [[ $_functionReturnValue -eq 0 ]]; then # _unescapedTransferParameters=$( helperFunctions/xtractXMLAttributeValue "gsiftp_params" $_dparamFile ) # echo "${_unescapedTransferParameters// /#}" @@ -339,14 +339,14 @@ listTransfer/performTransfer() local _tgftpLogFileName="$3" if [[ -s "$_transferList" ]]; then - + # usage: #+ transferData transferList \ #+ metric \ #+ tmpLogfileName \ #+ dpathArray \ #+ dparamsArray - + local _transferListSource=$( listTransfer/getSourceFromTransferList "$_transferList" ) local _transferListDestination=$( listTransfer/getDestinationFromTransferList "$_transferList" ) @@ -377,7 +377,7 @@ listTransfer/performTransfer() helperFunctions/echoIfVerbose "" helperFunctions/echoIfVerbose -e "${_program} [${gtInstance}]: Data path used:\n$_dpathFile" fi - + # get dpath array local -a _dpathArray=( $( listTransfer/getDpathData "$_transferListSource" "$_transferListDestination" "$_metric" ) ) @@ -392,32 +392,32 @@ listTransfer/performTransfer() #]]; then # helperFunctions/echoIfVerbose -e "Default params used:\n$_dparamFile" #fi - + # get dparams array local -a _dparamsArray=( $( listTransfer/getTransferParameters "${_dpathArray[*]}" ) ) - + #echo "DEBUG: ${_dpathArray[0]}" #echo "DEBUG: ${_dpathArray[1]}" - + #echo "DEBUG: ----" - + #echo "DEBUG: ${_dparamsArray[0]}" #echo "DEBUG: ${_dparamsArray[1]}" - + #return listTransfer/transferData "$_transferList" \ "$_metric" \ "$_tgftpLogFileName" \ "${_dpathArray[*]}" \ "${_dparamsArray[*]}" - + _functionReturnValue="$?" - + if [[ $_functionReturnValue -eq $_gtransfer_exit_ok ]]; then # TODO: # Move temp file removal to onExit() function. Exclude #+ files that should survive a restart of gtransfer! - + # if the whole transfer succeeded, # reactivate filename globbing for temp file deletion set +f @@ -434,17 +434,17 @@ listTransfer/performTransfer() helperFunctions/echoIfVerbose "" helperFunctions/echoIfVerbose -e "${_program} [${gtInstance}]: The transfer succeeded!" - + if [[ $gtProgressIndicatorSet != 0 ]]; then echo "" fi - + fi fi return - + } @@ -462,14 +462,14 @@ listTransfer/dparamAvailable() { local _source="$1" local _destination="$2" - + local _sourceWithoutPath=$( helperFunctions/getURLWithoutPath "$_source" ) local _destinationWithoutPath=$( helperFunctions/getURLWithoutPath "$_destination" ) - + # get corresponding dparam name (and remove any "username@" portions in #+ the URL before hashing). local _dparamFilename=$( helperFunctions/hashSourceDestination $( echo $_sourceWithoutPath | sed -e 's/:\/\/.*@/:\/\//' ) $( echo $_destinationWithoutPath | sed -e 's/:\/\/.*@/:\/\//' ) ) - + if [[ -e "$gtransferDefaultParamsDirectory/$_dparamFilename" ]]; then echo "$gtransferDefaultParamsDirectory/$_dparamFilename" return 0 @@ -495,30 +495,30 @@ listTransfer/transferData() #+ tmpLogfileName \ #+ dpathArray \ #+ dparamsArray - + # TODO: # Add argument to enable processing of single specific transfer steps. #+ E.g.: transferStep: all - all steps, 1 - step 1, etc. - + #set -f local _transferList="$1" - + if [[ ! -s "$_transferList" ]]; then return 1 fi - + local dataPathMetric="$2" local tgftpTempLogfileName="$3" - + local -a _dpathArray=($4) local -a _dparamsArray=($5) #echo ${_dpathArray[0]} #echo ${_dpathArray[1]} - + #echo "----" - + #echo ${_dparamsArray[0]} #echo ${_dparamsArray[1]} #return 0 @@ -526,7 +526,7 @@ listTransfer/transferData() local tgftpLogfileName="" # max number of retries gtransfer will do - local maxRetries="${GT_MAX_RETRIES:-$gtMaxRetries}" + local maxRetries="${GT_MAX_RETRIES:-$__GLOBAL__gtMaxRetries}" local retries=0 local _transferRetries="" @@ -587,9 +587,9 @@ listTransfer/transferData() local transferStepSource=${transferStep%%;*} local transferStepDestination=${transferStep##*;} - local _escapedTransferParameters="" + local _escapedTransferParameters="" local _transferParameters="" - + # if the dparamsArray contains only one element (=simple #+ string), then use these parameters for all steps if [[ ${#_dparamsArray[*]} -eq 1 ]]; then @@ -684,7 +684,7 @@ listTransfer/transferData() fi done - + return $_gtransfer_exit_ok } @@ -731,7 +731,7 @@ listTransfer/doTransferStep() if [[ "$sourceUsernamePortion" == "NULL" ]]; then sourceUsernamePortion="" fi - + local destinationUsernamePortion="${10}" if [[ "$destinationUsernamePortion" == "NULL" ]]; then destinationUsernamePortion="" @@ -740,7 +740,7 @@ listTransfer/doTransferStep() local transferId="${11}" local transferList="${12}" - + local transferParameters="${13}" local transferRetries="${14}" @@ -800,9 +800,9 @@ listTransfer/doTransferStep() else local tgftpLogfileName="${tgftpTempLogfileName/%.log/__step_${transferStepNumber}.log}" fi - + transferStepDefaultParams="$transferParameters" - + # TODO: # build temp transfer list by exchanging source and @@ -813,14 +813,14 @@ listTransfer/doTransferStep() cp "$transferList" "$_tempTransferList" || return 3 local _source=$( listTransfer/getSourceFromTransferList "$transferList" ) - local _destination=$( listTransfer/getDestinationFromTransferList "$transferList" ) + local _destination=$( listTransfer/getDestinationFromTransferList "$transferList" ) local _sourceWithoutPath=$( helperFunctions/getURLWithoutPath "$source" ) local _destinationWithoutPath=$( helperFunctions/getURLWithoutPath "$destination" ) local _sourcePath=$( helperFunctions/getPathFromURL "$_source" ) local _destinationPath=$( helperFunctions/getPathFromURL "$_destination" ) - + # (3) transfer data (various steps possible!) ######################################################################## # direct transfer @@ -844,7 +844,7 @@ listTransfer/doTransferStep() # this is a direct transfer, so we can just use the transfer list unchanged local _tmpTransferId=$( listTransfer/getTransferIdForTransferList "$_tempTransferList" ) - + # if this transferStep was already finished, if [[ -e "${__GLOBAL__gtTmpDir}/${_tmpTransferId}.finished" ]]; then # ...just return "4" (SKIP_TRANSFER_STEP), @@ -900,7 +900,7 @@ listTransfer/doTransferStep() sed -e "s|$_destinationWithoutPath|${transferStepDestination}/${transitSiteTempDir}/|g" -i "$_tempTransferList" local _tmpTransferId=$( listTransfer/getTransferIdForTransferList "$_tempTransferList" ) - + # if this transferStep was already finished, if [[ -e "${__GLOBAL__gtTmpDir}/${_tmpTransferId}.finished" ]]; then # ...just return "4" (SKIP_TRANSFER_STEP), @@ -923,7 +923,7 @@ listTransfer/doTransferStep() "1" \ "${__GLOBAL__gtTmpDir}/${_tmpTransferId}.transferList" \ "${transferStepDestination}${transitSiteTempDir}/" ) - + #simulateTransfer #simulateError @@ -957,7 +957,7 @@ listTransfer/doTransferStep() -i "$_tempTransferList" local _tmpTransferId=$( listTransfer/getTransferIdForTransferList "$_tempTransferList" ) - + # if this transferStep was already finished, if [[ -e "${__GLOBAL__gtTmpDir}/${_tmpTransferId}.finished" ]]; then # ...just return "4" (SKIP_TRANSFER_STEP), @@ -1133,7 +1133,8 @@ listTransfer/createTgftpTransferCommand() local tgftpPostCommandParam="" local tgftpPostCommand="" - local gucMaxRetries="${GUC_MAX_RETRIES:-$gucMaxRetries}" + local gucMaxRetries="${GUC_MAX_RETRIES:-$__GLOBAL__gucMaxRetries}" + local _syncLevel="$__GLOBAL__syncLevel" # This should create a unique filename correspondent to this specifc tgftp #+ command. @@ -1166,9 +1167,11 @@ listTransfer/createTgftpTransferCommand() #+ * destination URL #+ * offset - # Deactivate reliability (GT_NO_RELIABILITY=1) if needed. - #local addGsiftpParams local _gucReliabilityOptions + local _gucSyncOptions + + # Deactivate reliability (GT_NO_RELIABILITY=1) if needed. + if [[ $GT_NO_RELIABILITY -eq 1 ]]; then _gucReliabilityOptions="" @@ -1189,7 +1192,12 @@ listTransfer/createTgftpTransferCommand() gsiftpParams=$( echo "$gsiftpParams" | sed -e 's/-pp//' ) fi - local _gucSyncOptions="-sync -sync-level 1" + if [[ $_noSyncSet -eq 0 ]]; then + + _gucSyncOptions="" + else + _gucSyncOptions="-sync -sync-level $_syncLevel" + fi # perform recursive transfer if [[ $recursiveTransferSet -eq 0 ]]; then @@ -1227,10 +1235,10 @@ listTransfer/createTgftpTransferCommand() #+ sleep between dir creation and chmod, as otherwise the chmod fails. #+ #+ $ uberftp -mkdir gsiftp://p6012-deisa.huygens.sara.nl:2812/scratch/shared/prace/gridftp/transitSiteTempDir.rQoM5180 && uberftp -chmod 0700 gsiftp://p6012-deisa.huygens.sara.nl:2812/scratch/shared/prace/gridftp/transitSiteTempDir.rQoM5180 - #+ Failed to connect to 145.100.18.152 port 2812: Cannot assign requested address + #+ Failed to connect to 145.100.18.152 port 2812: Cannot assign requested address #+ #+ $ uberftp -mkdir gsiftp://p6012-deisa.huygens.sara.nl:2812/scratch/shared/prace/gridftp/transitSiteTempDir.rQoM5180 && sleep 0.5 && uberftp -chmod 0700 gsiftp://p6012-deisa.huygens.sara.nl:2812/scratch/shared/prace/gridftp/transitSiteTempDir.rQoM5180 - if [[ $toTransitSite -eq 1 ]]; then + if [[ $toTransitSite -eq 1 ]]; then # get FQDN and port number from destination URL local _fqdn=$( helperFunctions/getFQDNFromURL "$destination" ) local _portNumber=$( helperFunctions/getPortNumberFromURL "$destination" ) @@ -1396,7 +1404,7 @@ listTransfer/createTransferList() { local _source="$1" local _destination="$2" - + local _transferListFile="${__GLOBAL__gtTmpDir}/$$_transferList.$__GLOBAL__gtTmpSuffix" # Check if valid URLs are provided @@ -1413,14 +1421,14 @@ listTransfer/createTransferList() { echo -e "\n${_program} [${gtInstance}]: Target URL cannot be a \"http[s]://\" URL!" 1>&2 return 1 fi - + # perform recursive transfer if [[ $recursiveTransferSet -eq 0 ]]; then local _recursive="-r" else local _recursive="" fi - + # to get the transfer list we use guc with "-do" option globus-url-copy -do "$_transferListFile" $_recursive "$_source" "$_destination" @@ -1571,7 +1579,7 @@ listTransfer/getTransferSizeFromTransferList() { # # usage: # listTransfer/getTransferSizeFromTransferList transferList - + local _transferList="$1" local _sum=0 @@ -1608,11 +1616,11 @@ listTransfer/getNumberOfFilesFromTransferList() { # gets the number if files from the given transfer list (assuming that #+ each line counts for one file) - + local _transferList="$1" - + local _numberOfFiles=$( wc -l <"$_transferList" ) - + if [[ $? -eq 0 ]]; then echo "$_numberOfFiles" return 0 @@ -1620,4 +1628,3 @@ listTransfer/getNumberOfFilesFromTransferList() return 1 fi } - diff --git a/share/doc/gtransfer.1.md b/share/doc/gtransfer.1.md index ecf56d1..fc7edc0 100644 --- a/share/doc/gtransfer.1.md +++ b/share/doc/gtransfer.1.md @@ -1,6 +1,6 @@ -% GTRANSFER(1) gtransfer 0.5.0 | User Commands +% GTRANSFER(1) gtransfer 0.8.0 | User Commands % Frank Scheiner -% Apr 12, 2016 +% Mar 23, 2017 # NAME # @@ -10,21 +10,23 @@ # SYNOPSIS # -**{gtransfer|gt} [\--source|-s _sourceUrl_] -[\--destination|-d _destinationUrl_] -[\--transfer-list|-f _transferList_] -[\--auto-optimize|-o _transferMode_] +**{gtransfer|gt} [\--source|-s _sourceUrl_] +[\--destination|-d _destinationUrl_] +[\--transfer-list|-f _transferList_] +[\--auto-optimize|-o _transferMode_] [\--recursive|-r] [\--checksum-data-channel|-c] [\--encrypt-data-channel|-e] -[\--guc-max-retries _gucMaxRetries_] -[\--gt-max-retries _gtMaxRetries_] -[\--gt-progress-indicator _indicatorCharacter_] -[\--verbose|-v] -[\--metric|-m _dataPathMetric_] -[\--logfile|-l _logfile_] -[\--auto-clean|-a] -[\--configfile _configurationFile_] +[\--sync-level syncLevel] +[\--no-sync] +[\--guc-max-retries _gucMaxRetries_] +[\--gt-max-retries _gtMaxRetries_] +[\--gt-progress-indicator _indicatorCharacter_] +[\--verbose|-v] +[\--metric|-m _dataPathMetric_] +[\--logfile|-l _logfile_] +[\--auto-clean|-a] +[\--configfile _configurationFile_] [\-- _gucParameters_]** @@ -120,6 +122,7 @@ reference: A full table is available on `http://www.w3schools.com/tags/ref_urlencode.asp` + ## **[-f, \--transfer-list _transferList_]** ## As alternative to providing source and destination URLs on the command line, @@ -129,7 +132,7 @@ list. The format of each line of the transfer list file is as follows (including the double quotes!): -"\://\:\/path/to/file" +"\://\:\/path/to/file" "\://\:\/path/to/file[s/]" Throughout all lines the source URL host part (e.g. @@ -144,7 +147,7 @@ size of files to be transferred. If less than 100 files are going to be transferred, gtransfer will fall back to list transfer. The _transferMode_ controls how files of different size classes are transferred. Currently "seq[uential]" (different size classes are transferred sequentially) is -possible. To define different file size classes use the file +possible. To define different file size classes use the file _[...]/chunkConfig_. See **FILES** section below for more details. @@ -152,9 +155,9 @@ _[...]/chunkConfig_. See **FILES** section below for more details. Transfer files recursively. -**NOTICE:** **globus-url-copy(1)** (even with option **-cd**) and therefore also -**gt** will not create directories on the destination side that are empty on the -source side. +**NOTICE:** **globus-url-copy(1)** (even with option **-cd** and sync options) +and therefore also **gt** will not create directories on the destination side +that are empty on the source side. ## **[-c, \--checksum-data-channel]** ## @@ -169,6 +172,36 @@ Enable encryption on the data channel. Cannot be used in conjunction with **-c**! +## **[--sync-level _syncLevel_]** ## + +Set the sync level that should be used for the transfer. This is a +**globus-url-copy(1)** option and the following sync levels are available: + +* Level _0_ will only transfer if the destination does not exist. + +* Level _1_ will transfer if the size of the destination does not match the size + of the source. + +* Level _2_ will transfer if the time stamp of the destination is older than the + time stamp of the source. + +* Level _3_ will perform a checksum of the source and destination and transfer if + the checksums do not match. + +By default gtransfer uses sync level _1_. Cannot be used in conjunction with +**--no-sync**! + +**NOTICE:** **globus-url-copy(1)** (even with option **-cd** and sync options) +and therefore also **gt** will not create directories on the destination side +that are empty on the source side. + + +## **[--no-sync]** ## + +Disable sync(hronization) for the transfer. Cannot be used in conjunction with +**--sync-level**! + + ## **[\--guc-max-retries _gucMaxRetries_]** ## This option sets the maximum number of retries **globus-url-copy(1)** will do @@ -230,8 +263,7 @@ Set the **globus-url-copy(1)** parameters that should be used for all transfer steps. Notice the space between "\--" and the actual parameters. This overwrites any available dparams and is not recommended for regular usage. There exists one exception for the **-len|-partial-length X** option. If this is provided, it -will only be added to the transfer parameters from a dparam for a connection or -- if no dparam is available - to the builtin default transfer parameters. +will only be added to the transfer parameters from a dparam for a connection or, if no dparam is available, to the builtin default transfer parameters. **NOTICE:** If specified, this option must be the last one in a **gt** command line. @@ -275,7 +307,7 @@ the beginning, when retried. # FILES # - + ## _[...]/gtransfer.conf_ ## The **gt** configuration file. @@ -329,6 +361,5 @@ This directory contains the user dparams usable by **gt**. Can be created with # SEE ALSO # -**dparam(1)**, **dparam(5)**, **dpath(1)**, **dpath(5)**, +**dparam(1)**, **dparam(5)**, **dpath(1)**, **dpath(5)**, **halias(1)**, **globus-url-copy(1)**, **tgftp(1)**, **uberftp(1C)** - diff --git a/share/man/man1/gtransfer.1 b/share/man/man1/gtransfer.1 index 481df98..fd91123 100644 --- a/share/man/man1/gtransfer.1 +++ b/share/man/man1/gtransfer.1 @@ -1,20 +1,22 @@ .\"t -.TH "GTRANSFER" "1" "Apr 12, 2016" "gtransfer 0.5.0" "User Commands" +.TH "GTRANSFER" "1" "Mar 23, 2017" "gtransfer 0.8.0" "User Commands" .SH NAME .PP \f[B]gtransfer\f[] \- The GridFTP data transfer script .SH SYNOPSIS .PP -\f[B]{gtransfer|gt} [\-\-source|\-s \f[I]sourceUrl\f[]] -[\-\-destination|\-d \f[I]destinationUrl\f[]] [\-\-transfer\-list|\-f -\f[I]transferList\f[]] [\-\-auto\-optimize|\-o \f[I]transferMode\f[]] +\f[B]{gtransfer|gt} [\-\-source|\-s +\f[I]sourceUrl\f[]][\-\-destination|\-d \f[I]destinationUrl\f[]] +[\-\-transfer\-list|\-f \f[I]transferList\f[]][\-\-auto\-optimize|\-o +\f[I]transferMode\f[]] [\-\-recursive|\-r][\-\-checksum\-data\-channel|\-c] -[\-\-encrypt\-data\-channel|\-e][\-\-guc\-max\-retries -\f[I]gucMaxRetries\f[]] [\-\-gt\-max\-retries \f[I]gtMaxRetries\f[]] -[\-\-gt\-progress\-indicator \f[I]indicatorCharacter\f[]] -[\-\-verbose|\-v] [\-\-metric|\-m \f[I]dataPathMetric\f[]] -[\-\-logfile|\-l \f[I]logfile\f[]] [\-\-auto\-clean|\-a] [\-\-configfile -\f[I]configurationFile\f[]] [\-\- \f[I]gucParameters\f[]]\f[] +[\-\-encrypt\-data\-channel|\-e][\-\-sync\-level syncLevel] +[\-\-no\-sync][\-\-guc\-max\-retries \f[I]gucMaxRetries\f[]] +[\-\-gt\-max\-retries \f[I]gtMaxRetries\f[]][\-\-gt\-progress\-indicator +\f[I]indicatorCharacter\f[]] [\-\-verbose|\-v][\-\-metric|\-m +\f[I]dataPathMetric\f[]] [\-\-logfile|\-l +\f[I]logfile\f[]][\-\-auto\-clean|\-a] [\-\-configfile +\f[I]configurationFile\f[]][\-\- \f[I]gucParameters\f[]]\f[] .SH DESCRIPTION .PP \f[B]gtransfer\f[] or short \f[B]gt\f[] is a wrapper script for the @@ -184,8 +186,9 @@ See \f[B]FILES\f[] section below for more details. Transfer files recursively. .PP \f[B]NOTICE:\f[] \f[B]globus\-url\-copy(1)\f[] (even with option -\f[B]\-cd\f[]) and therefore also \f[B]gt\f[] will not create -directories on the destination side that are empty on the source side. +\f[B]\-cd\f[] and sync options) and therefore also \f[B]gt\f[] will not +create directories on the destination side that are empty on the source +side. .SS \f[B][\-c, \-\-checksum\-data\-channel]\f[] .PP Enable checksumming on the data channel. @@ -194,6 +197,34 @@ Cannot be used in conjunction with \f[B]\-e\f[]! .PP Enable encryption on the data channel. Cannot be used in conjunction with \f[B]\-c\f[]! +.SS \f[B][\-\-sync\-level \f[I]syncLevel\f[]]\f[] +.PP +Set the sync level that should be used for the transfer. +This is a \f[B]globus\-url\-copy(1)\f[] option and the following sync +levels are available: +.IP \[bu] 2 +Level \f[I]0\f[] will only transfer if the destination does not exist. +.IP \[bu] 2 +Level \f[I]1\f[] will transfer if the size of the destination does not +match the size of the source. +.IP \[bu] 2 +Level \f[I]2\f[] will transfer if the time stamp of the destination is +older than the time stamp of the source. +.IP \[bu] 2 +Level \f[I]3\f[] will perform a checksum of the source and destination +and transfer if the checksums do not match. +.PP +By default gtransfer uses sync level \f[I]1\f[]. +Cannot be used in conjunction with \f[B]\-\-no\-sync\f[]! +.PP +\f[B]NOTICE:\f[] \f[B]globus\-url\-copy(1)\f[] (even with option +\f[B]\-cd\f[] and sync options) and therefore also \f[B]gt\f[] will not +create directories on the destination side that are empty on the source +side. +.SS \f[B][\-\-no\-sync]\f[] +.PP +Disable sync(hronization) for the transfer. +Cannot be used in conjunction with \f[B]\-\-sync\-level\f[]! .SS \f[B][\-\-guc\-max\-retries \f[I]gucMaxRetries\f[]]\f[] .PP This option sets the maximum number of retries @@ -260,7 +291,7 @@ usage. There exists one exception for the \f[B]\-len|\-partial\-length X\f[] option. If this is provided, it will only be added to the transfer parameters -from a dparam for a connection or \- if no dparam is available \- to the +from a dparam for a connection or, if no dparam is available, to the builtin default transfer parameters. .PP \f[B]NOTICE:\f[] If specified, this option must be the last one in a @@ -348,7 +379,7 @@ If existing, dparams in this directory have precedence. .SH SEE ALSO .PP \f[B]dparam(1)\f[], \f[B]dparam(5)\f[], \f[B]dpath(1)\f[], -\f[B]dpath(5)\f[], \f[B]globus\-url\-copy(1)\f[], \f[B]tgftp(1)\f[], -\f[B]uberftp(1C)\f[] +\f[B]dpath(5)\f[], \f[B]halias(1)\f[], \f[B]globus\-url\-copy(1)\f[], +\f[B]tgftp(1)\f[], \f[B]uberftp(1C)\f[] .SH AUTHORS Frank Scheiner. From 29c4d9742a521cd01996a4e30445aa83b0be50f3 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 23 Mar 2017 10:47:54 +0100 Subject: [PATCH 15/21] Added manpage "links" for gtools shorthands. Updated spec file. --- gtransfer.spec | 13 ++++++++++++- share/man/man1/gcat.1 | 1 + share/man/man1/gls.1 | 1 + share/man/man1/gmkdir.1 | 1 + share/man/man1/gmv.1 | 1 + share/man/man1/grm.1 | 1 + 6 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 share/man/man1/gcat.1 create mode 100644 share/man/man1/gls.1 create mode 100644 share/man/man1/gmkdir.1 create mode 100644 share/man/man1/gmv.1 create mode 100644 share/man/man1/grm.1 diff --git a/gtransfer.spec b/gtransfer.spec index 4507628..4f8f202 100644 --- a/gtransfer.spec +++ b/gtransfer.spec @@ -113,6 +113,12 @@ cp share/man/man5/dpath.5 %{buildroot}%{_mandir}/man5/ cp share/man/man1/dparam.1 %{buildroot}%{_mandir}/man1/ cp share/man/man5/dparam.5 %{buildroot}%{_mandir}/man5/ cp share/man/man1/halias.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/gtools.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/gcat.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/gls.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/gmkdir.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/gmv.1 %{buildroot}%{_mandir}/man1/ +cp share/man/man1/grm.1 %{buildroot}%{_mandir}/man1/ %clean rm -rf %{buildroot} @@ -188,9 +194,14 @@ rm -rf %{buildroot} %{_mandir}/man5/dparam.5.gz %{_mandir}/man1/halias.1.gz %{_mandir}/man1/gtools.1.gz +%{_mandir}/man1/gcat.1.gz +%{_mandir}/man1/gls.1.gz +%{_mandir}/man1/gmkdir.1.gz +%{_mandir}/man1/gmv.1.gz +%{_mandir}/man1/grm.1.gz %changelog -* Tue Mar 14 2017 Frank Scheiner 0.8.0-1 +* Tue Mar 23 2017 Frank Scheiner 0.8.0-1 - Updated source package and version number to new release. Also added new gtools to the toolkit. * Thu Sep 08 2016 Frank Scheiner 0.7.1-1 diff --git a/share/man/man1/gcat.1 b/share/man/man1/gcat.1 new file mode 100644 index 0000000..97fafe9 --- /dev/null +++ b/share/man/man1/gcat.1 @@ -0,0 +1 @@ +.so man1/gtools.1 diff --git a/share/man/man1/gls.1 b/share/man/man1/gls.1 new file mode 100644 index 0000000..97fafe9 --- /dev/null +++ b/share/man/man1/gls.1 @@ -0,0 +1 @@ +.so man1/gtools.1 diff --git a/share/man/man1/gmkdir.1 b/share/man/man1/gmkdir.1 new file mode 100644 index 0000000..97fafe9 --- /dev/null +++ b/share/man/man1/gmkdir.1 @@ -0,0 +1 @@ +.so man1/gtools.1 diff --git a/share/man/man1/gmv.1 b/share/man/man1/gmv.1 new file mode 100644 index 0000000..97fafe9 --- /dev/null +++ b/share/man/man1/gmv.1 @@ -0,0 +1 @@ +.so man1/gtools.1 diff --git a/share/man/man1/grm.1 b/share/man/man1/grm.1 new file mode 100644 index 0000000..97fafe9 --- /dev/null +++ b/share/man/man1/grm.1 @@ -0,0 +1 @@ +.so man1/gtools.1 From 46ef0d046d73286fc9d6cf3239106bfbf619363e Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 23 Mar 2017 11:16:00 +0100 Subject: [PATCH 16/21] Corrected typo and updated years in copyright statement. --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 40fc793..62f13c9 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ $ gt -s host1:/file/* -d host3:/files/ -m all ![transfer using multipathing](/share/doc/images/multipathing-transfer.png) -The host `host1` has connections to both the Internet and a private network. The +The host `host1` has connections to both the Internet and a private network. The bandwidth of the Internet connection is limited to 1 Gb/s, but the connection to the private network has a bandwidth of 10 Gb/s. The host `host2` has a bandwidth of 10 Gb/s on connections to both the Internet and the private network. In @@ -134,7 +134,7 @@ can use `myGridFTP:` and `gsiftp://host1.domain.tld:2811` synonymically. See ### Persistent identifiers (PIDs) ### -Gtransfer can use persistent identifiers (PIDs) as used by [EUDAT] and provided +Gtransfer can use persistent identifiers (PIDs) as used by [EUDAT] and provided by [EPIC] as source of a data transfer. See [persistent identifiers] for more details. @@ -162,7 +162,7 @@ Additional examples will be made available occasionally. This is a list of HPC centers in Europe that use gtransfer in production: [![HLRS logo](https://raw.github.com/fscheiner/images/master/site_logos/hlrs_logo.png)](http://www.hlrs.de/) - + [Höchstleistungsrechenzentrum Stuttgart (HLRS - Germany)](http://www.hlrs.de/) **** @@ -203,7 +203,7 @@ This is a list of HPC centers in Europe that use gtransfer in production: **** -[![IT4Inoovations logo](https://raw.github.com/fscheiner/images/master/site_logos/it4innovations_logo_h100.png)](http://www.it4i.cz/) +[![IT4Innovations logo](https://raw.github.com/fscheiner/images/master/site_logos/it4innovations_logo_h100.png)](http://www.it4i.cz/) [IT4Innovations national supercomputing center (IT4Innovations - Czech republic)](http://www.it4i.cz/) @@ -212,7 +212,7 @@ This is a list of HPC centers in Europe that use gtransfer in production: (GPLv3) -Copyright (C) 2010, 2011, 2013-2015 Frank Scheiner, HLRS, Universitaet Stuttgart +Copyright (C) 2010, 2011, 2013-2017 Frank Scheiner, HLRS, Universitaet Stuttgart Copyright (C) 2011, 2012, 2013 Frank Scheiner The software is distributed under the terms of the GNU General Public License @@ -231,4 +231,3 @@ You should have received a [copy] of the GNU General Public License along with this program. If not, see . [copy]: /COPYING - From 859b3b6aff175fff5d8348f8d340ae541ed3e916 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 23 Mar 2017 12:33:58 +0100 Subject: [PATCH 17/21] Reworked documentation about sync options in manpage. Also reformatted synopsis. --- share/doc/gtransfer.1.md | 49 ++++++++++++++++---------------------- share/man/man1/gtransfer.1 | 41 ++++++++++++++++--------------- 2 files changed, 42 insertions(+), 48 deletions(-) diff --git a/share/doc/gtransfer.1.md b/share/doc/gtransfer.1.md index fc7edc0..902c0f9 100644 --- a/share/doc/gtransfer.1.md +++ b/share/doc/gtransfer.1.md @@ -10,24 +10,14 @@ # SYNOPSIS # -**{gtransfer|gt} [\--source|-s _sourceUrl_] -[\--destination|-d _destinationUrl_] -[\--transfer-list|-f _transferList_] -[\--auto-optimize|-o _transferMode_] -[\--recursive|-r] -[\--checksum-data-channel|-c] -[\--encrypt-data-channel|-e] -[\--sync-level syncLevel] -[\--no-sync] -[\--guc-max-retries _gucMaxRetries_] -[\--gt-max-retries _gtMaxRetries_] -[\--gt-progress-indicator _indicatorCharacter_] -[\--verbose|-v] -[\--metric|-m _dataPathMetric_] -[\--logfile|-l _logfile_] -[\--auto-clean|-a] -[\--configfile _configurationFile_] -[\-- _gucParameters_]** +**{gtransfer|gt} [\--source|-s _sourceUrl_] [\--destination|-d +_destinationUrl_] [\--transfer-list|-f _transferList_] [\--auto-optimize|-o +_transferMode_] [\--recursive|-r] [\--checksum-data-channel|-c] +[\--encrypt-data-channel|-e] [\--sync-level syncLevel] [\--no-sync] +[\--guc-max-retries _gucMaxRetries_] [\--gt-max-retries _gtMaxRetries_] +[\--gt-progress-indicator _indicatorCharacter_] [\--verbose|-v] [\--metric|-m +_dataPathMetric_] [\--logfile|-l _logfile_] [\--auto-clean|-a] [\--configfile +_configurationFile_] [\-- _gucParameters_]** # DESCRIPTION # @@ -157,19 +147,19 @@ Transfer files recursively. **NOTICE:** **globus-url-copy(1)** (even with option **-cd** and sync options) and therefore also **gt** will not create directories on the destination side -that are empty on the source side. +that are empty on the source side! ## **[-c, \--checksum-data-channel]** ## -Enable checksumming on the data channel. Cannot be used in conjunction with -**-e**! +Enable checksumming on the data channel. The option **-c** cannot be used in +conjunction with **-e**! ## **[-e, \--encrypt-data-channel]** ## -Enable encryption on the data channel. Cannot be used in conjunction with -**-c**! +Enable encryption on the data channel. The option **-e** cannot be used in +conjunction with **-c**! ## **[--sync-level _syncLevel_]** ## @@ -185,20 +175,21 @@ Set the sync level that should be used for the transfer. This is a * Level _2_ will transfer if the time stamp of the destination is older than the time stamp of the source. -* Level _3_ will perform a checksum of the source and destination and transfer if - the checksums do not match. +* Level _3_ will perform a checksum of the source and destination and transfer + if the checksums do not match. -By default gtransfer uses sync level _1_. Cannot be used in conjunction with -**--no-sync**! +By default gt transfers files **conditionally** and uses sync level _1_. The +option **--sync-level** cannot be used in conjunction with **--no-sync**! **NOTICE:** **globus-url-copy(1)** (even with option **-cd** and sync options) and therefore also **gt** will not create directories on the destination side -that are empty on the source side. +that are empty on the source side! ## **[--no-sync]** ## -Disable sync(hronization) for the transfer. Cannot be used in conjunction with +Disable sync(hronization) for the transfer. Gt then transfers files +**unconditionally**. The option **--no-sync** cannot be used in conjunction with **--sync-level**! diff --git a/share/man/man1/gtransfer.1 b/share/man/man1/gtransfer.1 index fd91123..931eccf 100644 --- a/share/man/man1/gtransfer.1 +++ b/share/man/man1/gtransfer.1 @@ -5,18 +5,17 @@ \f[B]gtransfer\f[] \- The GridFTP data transfer script .SH SYNOPSIS .PP -\f[B]{gtransfer|gt} [\-\-source|\-s -\f[I]sourceUrl\f[]][\-\-destination|\-d \f[I]destinationUrl\f[]] -[\-\-transfer\-list|\-f \f[I]transferList\f[]][\-\-auto\-optimize|\-o -\f[I]transferMode\f[]] -[\-\-recursive|\-r][\-\-checksum\-data\-channel|\-c] -[\-\-encrypt\-data\-channel|\-e][\-\-sync\-level syncLevel] -[\-\-no\-sync][\-\-guc\-max\-retries \f[I]gucMaxRetries\f[]] -[\-\-gt\-max\-retries \f[I]gtMaxRetries\f[]][\-\-gt\-progress\-indicator -\f[I]indicatorCharacter\f[]] [\-\-verbose|\-v][\-\-metric|\-m -\f[I]dataPathMetric\f[]] [\-\-logfile|\-l -\f[I]logfile\f[]][\-\-auto\-clean|\-a] [\-\-configfile -\f[I]configurationFile\f[]][\-\- \f[I]gucParameters\f[]]\f[] +\f[B]{gtransfer|gt} [\-\-source|\-s \f[I]sourceUrl\f[]] +[\-\-destination|\-d \f[I]destinationUrl\f[]] [\-\-transfer\-list|\-f +\f[I]transferList\f[]] [\-\-auto\-optimize|\-o \f[I]transferMode\f[]] +[\-\-recursive|\-r] [\-\-checksum\-data\-channel|\-c] +[\-\-encrypt\-data\-channel|\-e] [\-\-sync\-level syncLevel] +[\-\-no\-sync] [\-\-guc\-max\-retries \f[I]gucMaxRetries\f[]] +[\-\-gt\-max\-retries \f[I]gtMaxRetries\f[]] +[\-\-gt\-progress\-indicator \f[I]indicatorCharacter\f[]] +[\-\-verbose|\-v] [\-\-metric|\-m \f[I]dataPathMetric\f[]] +[\-\-logfile|\-l \f[I]logfile\f[]] [\-\-auto\-clean|\-a] [\-\-configfile +\f[I]configurationFile\f[]] [\-\- \f[I]gucParameters\f[]]\f[] .SH DESCRIPTION .PP \f[B]gtransfer\f[] or short \f[B]gt\f[] is a wrapper script for the @@ -188,15 +187,15 @@ Transfer files recursively. \f[B]NOTICE:\f[] \f[B]globus\-url\-copy(1)\f[] (even with option \f[B]\-cd\f[] and sync options) and therefore also \f[B]gt\f[] will not create directories on the destination side that are empty on the source -side. +side! .SS \f[B][\-c, \-\-checksum\-data\-channel]\f[] .PP Enable checksumming on the data channel. -Cannot be used in conjunction with \f[B]\-e\f[]! +The option \f[B]\-c\f[] cannot be used in conjunction with \f[B]\-e\f[]! .SS \f[B][\-e, \-\-encrypt\-data\-channel]\f[] .PP Enable encryption on the data channel. -Cannot be used in conjunction with \f[B]\-c\f[]! +The option \f[B]\-e\f[] cannot be used in conjunction with \f[B]\-c\f[]! .SS \f[B][\-\-sync\-level \f[I]syncLevel\f[]]\f[] .PP Set the sync level that should be used for the transfer. @@ -214,17 +213,21 @@ older than the time stamp of the source. Level \f[I]3\f[] will perform a checksum of the source and destination and transfer if the checksums do not match. .PP -By default gtransfer uses sync level \f[I]1\f[]. -Cannot be used in conjunction with \f[B]\-\-no\-sync\f[]! +By default gt transfers files \f[B]conditionally\f[] and uses sync level +\f[I]1\f[]. +The option \f[B]\-\-sync\-level\f[] cannot be used in conjunction with +\f[B]\-\-no\-sync\f[]! .PP \f[B]NOTICE:\f[] \f[B]globus\-url\-copy(1)\f[] (even with option \f[B]\-cd\f[] and sync options) and therefore also \f[B]gt\f[] will not create directories on the destination side that are empty on the source -side. +side! .SS \f[B][\-\-no\-sync]\f[] .PP Disable sync(hronization) for the transfer. -Cannot be used in conjunction with \f[B]\-\-sync\-level\f[]! +Gt then transfers files \f[B]unconditionally\f[]. +The option \f[B]\-\-no\-sync\f[] cannot be used in conjunction with +\f[B]\-\-sync\-level\f[]! .SS \f[B][\-\-guc\-max\-retries \f[I]gucMaxRetries\f[]]\f[] .PP This option sets the maximum number of retries From 3aad035ef11113ab2fe0419aa9ce7200eb6ae8e0 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Thu, 23 Mar 2017 13:48:39 +0100 Subject: [PATCH 18/21] Corrected typo. --- share/doc/gtools.1.md | 4 ++-- share/man/man1/gtools.1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/share/doc/gtools.1.md b/share/doc/gtools.1.md index c5ace01..96cfd6b 100644 --- a/share/doc/gtools.1.md +++ b/share/doc/gtools.1.md @@ -1,6 +1,6 @@ % GTOOLS(1) gtools 0.3.0 | User Commands % Frank Scheiner -% Mar 14, 2017 +% Mar 23, 2017 # NAME # @@ -36,7 +36,7 @@ Currently defined functions: All functions support host aliases and remote directory browsing via bash completion. Function names are without the **g[...]** prefix when used as -arguments of **gtools**. When used drectly (via symlinks) the **g[...]** prefix +arguments of **gtools**. When used directly (via symlinks) the **g[...]** prefix should be used to differentiate them from the similar OS tools (**cat(1)**, **ls(1)**, **mkdir(1)**, **mv(1)** and **rm(1)**). diff --git a/share/man/man1/gtools.1 b/share/man/man1/gtools.1 index 499fdf4..0802749 100644 --- a/share/man/man1/gtools.1 +++ b/share/man/man1/gtools.1 @@ -1,4 +1,4 @@ -.TH "GTOOLS" "1" "Mar 14, 2017" "gtools 0.3.0" "User Commands" +.TH "GTOOLS" "1" "Mar 23, 2017" "gtools 0.3.0" "User Commands" .SH NAME .PP \f[B]gtools\f[] @@ -28,7 +28,7 @@ All functions support host aliases and remote directory browsing via bash completion. Function names are without the \f[B]g[...]\f[] prefix when used as arguments of \f[B]gtools\f[]. -When used drectly (via symlinks) the \f[B]g[...]\f[] prefix should be +When used directly (via symlinks) the \f[B]g[...]\f[] prefix should be used to differentiate them from the similar OS tools (\f[B]cat(1)\f[], \f[B]ls(1)\f[], \f[B]mkdir(1)\f[], \f[B]mv(1)\f[] and \f[B]rm(1)\f[]). .SH FUNCTION DESCRIPTIONS From ef03664be5e288319b46ff9b6edb6c86d13257a4 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Mon, 27 Mar 2017 09:24:28 +0200 Subject: [PATCH 19/21] Updated URLs of HPC centers (most of them are using HTTPS now and the URL of SURFsara has changed). Also added logo and URL of KIT for usage of gt in bwDataArchiv. --- README.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 62f13c9..11e914d 100644 --- a/README.md +++ b/README.md @@ -167,39 +167,39 @@ This is a list of HPC centers in Europe that use gtransfer in production: **** -[![CSC logo](https://raw.github.com/fscheiner/images/master/site_logos/csc_logo_h100.png)](http://www.csc.fi/) +[![CSC logo](https://raw.github.com/fscheiner/images/master/site_logos/csc_logo_h100.png)](https://www.csc.fi/) -[CSC - IT Center for Science (CSC - Finland)](http://www.csc.fi/) +[CSC - IT Center for Science (CSC - Finland)](https://www.csc.fi/) **** -[![LRZ logo](https://raw.github.com/fscheiner/images/master/site_logos/lrz_logo_new_h100.png)](http://www.lrz.de/) +[![LRZ logo](https://raw.github.com/fscheiner/images/master/site_logos/lrz_logo_new_h100.png)](https://www.lrz.de/) -[Leibniz-Rechenzentrum (LRZ) der Bayerischen Akademie der Wissenschaften (LRZ - Germany)](http://www.lrz.de/) +[Leibniz-Rechenzentrum (LRZ) der Bayerischen Akademie der Wissenschaften (LRZ - Germany)](https://www.lrz.de/) **** -[![ICHEC logo](https://raw.github.com/fscheiner/images/master/site_logos/ichec_logo.png)](http://www.ichec.ie/) +[![ICHEC logo](https://raw.github.com/fscheiner/images/master/site_logos/ichec_logo.png)](https://www.ichec.ie/) -[Irish Centre for High-End Computing (ICHEC - Ireland)](http://www.ichec.ie/) +[Irish Centre for High-End Computing (ICHEC - Ireland)](https://www.ichec.ie/) **** -[![CINECA logo](https://raw.github.com/fscheiner/images/master/site_logos/cineca_logo.png)](http://www.cineca.it/) +[![CINECA logo](https://raw.github.com/fscheiner/images/master/site_logos/cineca_logo.png)](https://www.cineca.it/) -[Centro di supercalcolo, Consorzio di università (CINECA - Italy)](http://www.cineca.it/) +[Centro di supercalcolo, Consorzio di università (CINECA - Italy)](https://www.cineca.it/) **** -[![SURFSARA logo](https://raw.github.com/fscheiner/images/master/site_logos/surfsara_logo.png)](http://www.surfsara.nl/) +[![SURFSARA logo](https://raw.github.com/fscheiner/images/master/site_logos/surfsara_logo.png)](https://www.surf.nl/en/about-surf/subsidiaries/surfsara/) -[SURFsara (SURFsara - The Netherlands)](http://www.surfsara.nl/) +[SURFsara (SURFsara - The Netherlands)](https://www.surf.nl/en/about-surf/subsidiaries/surfsara/) **** -[![CINES logo](https://raw.github.com/fscheiner/images/master/site_logos/cines_logo.png)](http://www.cines.fr/) +[![CINES logo](https://raw.github.com/fscheiner/images/master/site_logos/cines_logo.png)](https://www.cines.fr/) -[Centre Informatique National de l’Enseignement Supérieur (CINES - France)](http://www.cines.fr/) +[Centre Informatique National de l’Enseignement Supérieur (CINES - France)](https://www.cines.fr/) **** @@ -207,6 +207,12 @@ This is a list of HPC centers in Europe that use gtransfer in production: [IT4Innovations national supercomputing center (IT4Innovations - Czech republic)](http://www.it4i.cz/) +**** + +[![KIT logo](https://raw.github.com/fscheiner/images/master/site_logos/kit_logo_h100.png)](https://www.kit.edu/) + +[Karlsruhe Institute of Technology (KIT - Germany)](https://www.kit.edu/) + ## License ## From a8f712301bff8698ea6a5081420bcdf2955579e5 Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Mon, 27 Mar 2017 09:29:41 +0200 Subject: [PATCH 20/21] Added markdown line break (two spaces plus newline) so the copyrigt notice shows up correctly on GitHub. It looks like the GFM line break did not work as expected on GitHub. ?-/ --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11e914d..85bc234 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ This is a list of HPC centers in Europe that use gtransfer in production: (GPLv3) -Copyright (C) 2010, 2011, 2013-2017 Frank Scheiner, HLRS, Universitaet Stuttgart +Copyright (C) 2010, 2011, 2013-2017 Frank Scheiner, HLRS, Universitaet Stuttgart Copyright (C) 2011, 2012, 2013 Frank Scheiner The software is distributed under the terms of the GNU General Public License From 936a42365aea26a611862df1de5b0e536efb968e Mon Sep 17 00:00:00 2001 From: Frank Scheiner Date: Mon, 27 Mar 2017 09:50:52 +0200 Subject: [PATCH 21/21] Updated ChangeLog and date in spec file. --- ChangeLog | 18 +++++++++++++++++- gtransfer.spec | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index e647071..3135394 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,20 @@ +2017-03-27 Frank Scheiner + +* 0.8.0 (gtransfer toolkit): + +- New functionality + +The (guc) sync level used by gtransfer for transfers can now be selected by +users. Syncing can also be disabled. + +The host aliases can now be retrieved and installed from a file provided +locally or from a remote repository, just like dpaths and dparams. + +Wrappers for uberftp (called gtools) were added that provide shorthand +functionality to users for common operations on remote GridFTP servers like +(g)cat, (g)ls, (g)mkdir, (g)mv and (g)rm. All wrappers support host aliases and +remote directory browsing just like gt. + 2016-09-08 Frank Scheiner *0.7.1 (gtransfer toolkit): @@ -413,4 +430,3 @@ configuration files in three different locations: - "/opt/gtransfer/etc/" (default for system install) - "/etc/opt/gtransfer/" - "$HOME/.gtransfer/" (default for user install) - diff --git a/gtransfer.spec b/gtransfer.spec index 4f8f202..2b95d1d 100644 --- a/gtransfer.spec +++ b/gtransfer.spec @@ -201,7 +201,7 @@ rm -rf %{buildroot} %{_mandir}/man1/grm.1.gz %changelog -* Tue Mar 23 2017 Frank Scheiner 0.8.0-1 +* Tue Mar 27 2017 Frank Scheiner 0.8.0-1 - Updated source package and version number to new release. Also added new gtools to the toolkit. * Thu Sep 08 2016 Frank Scheiner 0.7.1-1