S3 mojo for factorbase and sigmalog, in case instance is terminated

Author: 007

install.sh

@@ -15,7 +15,7 @@ fi
 
 apt-get update
 apt-get dist-upgrade
-apt-get install -y build-essential m4
+apt-get install -y build-essential m4 awscli
 
 if [ "$(md5sum gmp-6.0.0a.tar.bz2)" == "b7ff2d88cae7f8085bd5006096eed470  gmp-6.0.0a.tar.bz2" ] ; then
     echo "Using cached gmp"
@@ -53,3 +53,6 @@ cp onboot.sh util.sh /var/primemath/
 (crontab -l >/dev/null 2>&1 && (crontab -l | grep -v 'reboot /var/primemath/onboot.sh';echo "@reboot /var/primemath/onboot.sh") || echo "@reboot /var/primemath/onboot.sh") | crontab -
 (crontab -l | grep -v '/var/primemath/util.sh'; echo "0 * * * * /var/primemath/util.sh") | crontab -
 
+aws configure
+sudo -s aws configure
+

util.sh

@@ -1,5 +1,25 @@
 #!/bin/bash
 
+# part one - get a consolidated local copy of sigmalog and factorbase
 cat /var/primemath/log/factorlog.log | grep sigma | cut -d\  -f5- >> /var/primemath/sigmalog.txt
 sort -nu -o /var/primemath/sigmalog.txt /var/primemath/sigmalog.txt
-cat /var/primemath/factorbase.* | sort -nu > /var/primemath/factorbase.txt
+cat /var/primemath/factorbase.* | sort -nu > /var/primemath/.factorbase.txt
+mv /var/primemath/.factorbase.txt /var/primemath/factorbase.txt
+
+# part two - store instance copy on S3
+INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
+aws s3 cp --storage=class REDUCED_REDUNDANCY /var/primemath/sigmalog.txt s3://primemath/sigmalog_${INSTANCE_ID}.txt
+aws s3 cp --storage=class REDUCED_REDUNDANCY /var/primemath/factorbase.txt s3://primemath/factobase_${INSTANCE_ID}.txt
+
+# part three - consolidate the S3 version
+mkdir -p /tmp/primemath.combine
+aws s3 cp s3://primemath/ --recursive --exclude "*" --include "factorbase_*.txt" --include "sigmalog_*.txt" /tmp/primemath.combine/
+
+cat /tmp/primemath.combine/factorbase_*.txt /var/primemath/factorbase.txt | sort -nu > /var/primemath/.factorbase.txt
+mv /var/primemath/.factorbase.txt /var/primemath/factorbase.txt
+
+cat /tmp/primemath.combine/sigmalog_*.txt /var/primemath/sigmalog.txt | sort -nu > /var/primemath/.sigmalog.txt
+mv /var/primemath/.sigmalog.txt /var/primemath/sigmalog.txt
+
+rm -rf /tmp/primemath.combine.old && mv /tmp/primemath.combine /tmp/primemath.combine.old
+