Skip to content

Commit

Permalink
make remote username configurable
Browse files Browse the repository at this point in the history
  • Loading branch information
bjarthur committed May 2, 2022
1 parent 7f5c5e7 commit 5cdd627
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 21 deletions.
16 changes: 9 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ panel and configure file sharing to bind "/groups".
your workstation and the server to point to this same image.

* You might need an RSA key pair. If so, you'll need to add `-[v|B]
~/.ssh:/ssh` to `SONGEXPLORER_BIN`.
~/.ssh:/root/.ssh` to `SONGEXPLORER_BIN`.

* You might need to use ssh flags `-i /ssh/id_rsa -o "StrictHostKeyChecking
no"` in "configuration.pysh".
Expand All @@ -433,14 +433,16 @@ If you do not have a shared file system, the SongExplorer image and
configuration file must be separately installed on both computers, and
you'll need to do all of the compute jobs remotely.

Lastly, update "configuration.pysh" with the IP address of the server. As when
doing compute locally, SongExplorer uses a job scheduler on the server to manage
resources. The per-task resources used are the same as specified for the local
machine in `<task>_{gpu,cpu}_{ncpu_cores,ngpu_cards,ngigabytes_memory}`.
Lastly, update "configuration.pysh" with the name of the user and IP
address of the server. As when doing compute locally, SongExplorer
uses a job scheduler on the server to manage resources. The per-task
resources used are the same as specified for the local machine in
`<task>_{gpu,cpu}_{ncpu_cores,ngpu_cards,ngigabytes_memory}`.

$ grep -A1 \'server configuration.pysh
$ grep -A2 \'server configuration.pysh
# URL of the 'server' computer
server_ipaddr="c03u14"
server_username="arthurb"
server_ipaddr="c03u14.int.janelia.org"

### An On-Premise Cluster ###

Expand Down
2 changes: 2 additions & 0 deletions configuration.pysh
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ audio_nchannels=1
video_findfile="same-basename" # given a directory and a WAV file, return the corresponding video file

# URL of the 'server' computer
server_username=""
server_ipaddr=""

# how to dispatch jobs to the 'cluster'
cluster_username=""
cluster_ipaddr=""
cluster_cmd=""
cluster_logfile_flag=""
Expand Down
24 changes: 12 additions & 12 deletions src/gui.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,15 @@ echo INFO: detected $local_ncpu_cores local_ncpu_cores, \
$local_ngigabytes_memory local_ngigabytes_memory

if [[ -n "$server_ipaddr" ]] ; then
server_ncpu_cores=$(ssh $server_ipaddr nproc)
ssh $server_ipaddr which nvidia-smi &> /dev/null
server_ncpu_cores=$(ssh -l $server_username $server_ipaddr nproc)
ssh -l $server_username $server_ipaddr which nvidia-smi &> /dev/null
if [[ "$?" == 0 ]] ; then
nvidia_output=$(ssh $server_ipaddr nvidia-smi -L)
nvidia_output=$(ssh -l $server_username $server_ipaddr nvidia-smi -L)
server_ngpu_cards=$(echo "$nvidia_output" | wc -l)
else
server_ngpu_cards=0
fi
nbytes=$(ssh $server_ipaddr free -b | tail -2 | head -1 | tr -s ' ' | cut -d' ' -f2)
nbytes=$(ssh -l $server_username $server_ipaddr free -b | tail -2 | head -1 | tr -s ' ' | cut -d' ' -f2)
server_ngigabytes_memory=$(dc -e "$nbytes 1024 / 1024 / 512 + 1024 / p")
echo INFO: detected $server_ncpu_cores server_ncpu_cores, \
$server_ngpu_cards server_ngpu_cards, \
Expand Down Expand Up @@ -193,13 +193,13 @@ trap "local_njobs=\`hetero njobs\`; \
hetero stop; \
fi; \
if [[ -n \"$server_ipaddr\" ]] ; then \
server_njobs=\`ssh $server_ipaddr \"export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero njobs\"\`; \
server_njobs=\`ssh -l $server_username $server_ipaddr \"export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero njobs\"\`; \
if [[ \\\$\? && (( \"\$server_njobs\" > 0 )) ]] ; then \
echo WARNING: jobs are still queued on the server; \
echo to kill them execute \\\`ssh $server_ipaddr \\\$SONGEXPLORER_BIN hetero stop force\\\`; \
echo to stop SongExplorer\'s scheduler, wait until they are done and execute \\\`ssh $server_ipaddr \\\$SONGEXPLORER_BIN hetero stop\\\`; \
echo to kill them execute \\\`ssh -l $server_username $server_ipaddr \\\$SONGEXPLORER_BIN hetero stop force\\\`; \
echo to stop SongExplorer\'s scheduler, wait until they are done and execute \\\`ssh -l $server_username $server_ipaddr \\\$SONGEXPLORER_BIN hetero stop\\\`; \
else \
ssh $server_ipaddr \"export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero stop\"; \
ssh -l $server_username $server_ipaddr \"export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero stop\"; \
fi; \
fi" INT TERM KILL STOP HUP

Expand All @@ -217,19 +217,19 @@ elif [[ "$hetero_nslots" != "$local_ncpu_cores $local_ngpu_cards $local_ngigabyt
fi

if [[ -n "$server_ipaddr" ]] ; then
hetero_nslots=`ssh $server_ipaddr "export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero nslots"`
hetero_nslots=`ssh -l $server_username $server_ipaddr "export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero nslots"`
hetero_isrunning=$?
if [[ "$hetero_isrunning" != 0 ]] ; then
ssh $server_ipaddr "export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero start \
ssh -l $server_username $server_ipaddr "export SINGULARITYENV_PREPEND_PATH=$source_path; $SONGEXPLORER_BIN hetero start \
$server_ncpu_cores $server_ngpu_cards $server_ngigabytes_memory" &
elif [[ "$hetero_nslots" != "$server_ncpu_cores $server_ngpu_cards $server_ngigabytes_memory" ]] ; then

echo WARNING: SongExplorer\'s scheduler is already running on
echo $server_ipaddr with server_ncpu_cores, server_ngpu_cards, and
echo server_ngigabytes_memory set to $hetero_nslots, respectively, which
echo is different than specified in the configuration file. To use
echo the latter instead, quit SongExplorer, execute \`ssh $server_ipaddr
echo \$SONGEXPLORER_BIN hetero stop\`, and restart SongExplorer.
echo the latter instead, quit SongExplorer, execute \`ssh -l $server_username
echo $server_ipaddr \$SONGEXPLORER_BIN hetero stop\`, and restart SongExplorer.

fi
fi
Expand Down
4 changes: 2 additions & 2 deletions src/gui/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def generic_actuate(cmd, logfile, where,
jobid = p.stdout.decode('ascii').rstrip()
bokehlog.info(jobid)
elif where == "server":
p = run(["ssh", M.server_ipaddr, "export SINGULARITYENV_PREPEND_PATH="+M.source_path+";",
p = run(["ssh", "-l", M.server_username, M.server_ipaddr, "export SINGULARITYENV_PREPEND_PATH="+M.source_path+";",
"$SONGEXPLORER_BIN", "hetero", "submit",
"\"{ export CUDA_VISIBLE_DEVICES=\$QUEUE1; "+cmd+" "+' '.join(args).replace('"','\\"')+"; } &>> "+logfile+"\"",
str(ncpu_cores), str(ngpu_cards), str(ngigabyes_memory), "'"+localdeps+"'"],
Expand All @@ -59,7 +59,7 @@ def generic_actuate(cmd, logfile, where,
"export SINGULARITYENV_PREPEND_PATH="+M.source_path+";",
os.environ["SONGEXPLORER_BIN"]+" "+cmd+" "+' '.join(args)],
stdout=PIPE)
ps = Popen(["ssh", M.cluster_ipaddr, M.cluster_cmd,
ps = Popen(["ssh", "-l", M.cluster_username, M.cluster_ipaddr, M.cluster_cmd,
#"-J ${logfile//,/}.job",
clusterflags,
M.cluster_logfile_flag+" "+logfile],
Expand Down

0 comments on commit 5cdd627

Please sign in to comment.