Merge pull request #153 from martius-lab/fkloss/autorestart

Add autorestart.sh to example client

Author: luator

comprl-hockey-agent/README.md

@@ -27,5 +27,31 @@ export COMPRL_ACCESS_TOKEN=<YOUR ACCESS TOKEN>
 ```
 Then just call
 ```
-python3 ./run_client.p --args --agent=strong
+python3 ./run_client.py --args --agent=strong
 ```
+
+
+## The Auto-Restart Wrapper Script
+
+
+`autostart.sh` is a wrapper script around `run_client.py`, that is meant to
+automatically restart the client in case of infrequent issues like connection
+hiccups or server restarts.  To use it simply replace `python3 ./run_client.py`
+with `bash ./autorestart.sh` in the examples above.  Example:
+```
+bash ./autorestart.sh --server-url <URL> --server-port <PORT> \
+    --token <YOUR ACCESS TOKEN> \
+    --args --agent=strong
+```
+
+It keeps track of the number of restarts within a certain time window.  It will
+stop if there are too many restarts (e.g. because an issue in the client
+itself).  The length of the time window and the restart limit can be configured
+via variables at the top of the script.
+
+To stop the script, you need to press Ctrl+C twice.  Once to stop the client
+and then again to stop the wrapper script.
+
+You can enable notifications about restarts on ntfy.sh by setting a topic name
+to `NTFY_TOPIC` at the top of the script.  Once set, open https://ntfy.sh/TOPIC
+in your browser (or use the app on your phone) to get notified.

comprl-hockey-agent/autorestart.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# Wrapper script for run_client.py that automatically restarts the client if it
+# terminates.
+# It keeps track of the number of restarts and aborts if there are too many
+# within a certain time window (see variables below).
+# Comprl server information can either be set as environment variables below or
+# be passed as arguments to this script (any arguments will simply be forwarded
+# to run_client.py).
+#
+# NOTE: If you want to stop the script, you need to press Ctrl+C twice.  Once
+# to stop the client and then again to stop the wrapper script.
+
+
+# Keep track of restarts within this time window
+THRESHOLD_TIME_WINDOW=600  # 10 min
+# Threshold for the number of restarts in the time window
+THRESHOLD=10
+
+# If you set a topic name here, restart notifications will be sent to
+# ntfy.sh/$NFTY_TOPIC (so you can more easily monitor if there are problems).
+NTFY_TOPIC=
+
+# You may set the server information here, then you don't need to pass it as
+# argument.
+# export COMPRL_SERVER_URL=<URL>
+# export COMPRL_SERVER_PORT=<PORT>
+# export COMPRL_ACCESS_TOKEN=<YOUR ACCESS TOKEN>
+
+# Array to hold timestamps of terminations
+termination_times=()
+
+while true; do
+    # Run the command foobar
+    python3 ./run_client.py "$@"
+
+    # Get the current timestamp
+    current_time=$(date +%s)
+
+    # Add the current timestamp to the termination times array
+    termination_times+=("$current_time")
+
+    # Remove timestamps outside of the time window
+    termination_times=($(for time in "${termination_times[@]}"; do
+        if (( current_time - time <= ${THRESHOLD_TIME_WINDOW} )); then
+            echo "$time"
+        fi
+    done))
+
+    # Check if the number of terminations exceeds the threshold
+    if (( ${#termination_times[@]} > ${THRESHOLD} )); then
+        echo
+        echo "##############################################"
+        echo "Restarted too many times within the last ${THRESHOLD_TIME_WINDOW} seconds."
+
+        if [ -n "${NTFY_TOPIC}" ]; then
+            curl -H "Priority: urgent" -H "Tags: warning" \
+                -d "Too many restarts within the last ${THRESHOLD_TIME_WINDOW}" \
+                ntfy.sh/${NTFY_TOPIC}
+        fi
+
+        exit 1
+    fi
+
+    # Wait a bit before restarting.  In case the client terminated due to a
+    # server restart, it will take some time before the server is ready again.
+    sleep 20
+
+    echo
+    echo "##############################################"
+    echo "#                Restarting                  #"
+    echo "##############################################"
+    echo
+
+    if [ -n "${NTFY_TOPIC}" ]; then
+        curl -d "Restarting" ntfy.sh/${NTFY_TOPIC}
+    fi
+done
+