Merge pull request #7 from mlibrary/sms

Move sms script to this repo

.github/workflows/tests.yml

@@ -8,7 +8,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Create .env file
-        run: cat env.example > .env
+        run: cat env.example env.development > .env
       - name: Load .env file
         uses: xom9ikk/dotenv@v2
       - name: Set up Ruby 3.2

.gitignore

@@ -5,3 +5,19 @@ coverage/
 .bash_history
 .bundle/
 .cache/
+
+.ssh/*
+!.ssh/.keep
+
+sftp/ssh/*
+!/sftp/ssh/.keep
+
+# This is complicated. https://stackoverflow.com/questions/987142/make-gitignore-ignore-everything-except-a-few-files 
+sftp/sms/*
+!sftp/sms/.keep
+!sftp/sms/processed/
+sftp/sms/processed/*
+!sftp/sms/processed/.keep
+
+scratch/*
+!scratch/.keep

Gemfile

@@ -2,9 +2,14 @@ source "https://rubygems.org"
 
 source "https://rubygems.pkg.github.com/mlibrary" do
   gem "alma_rest_client", "~> 2.0"
+  gem "sftp"
 end
 
 gem "thor"
+gem "twilio-ruby"
+gem "telephone_number"
+gem "yabeda"
+gem "yabeda-prometheus"
 
 group :development, :test do
   gem "standard"

Gemfile.lock

@@ -8,6 +8,8 @@ GEM
       tzinfo (~> 2.0)
     addressable (2.8.3)
       public_suffix (>= 2.0.2, < 6.0)
+    anyway_config (2.4.2)
+      ruby-next-core (>= 0.14.0)
     ast (2.4.2)
     byebug (11.1.3)
     coderay (1.1.3)
@@ -16,6 +18,7 @@ GEM
       rexml
     diff-lcs (1.5.0)
     docile (1.4.0)
+    dry-initializer (3.1.1)
     faraday (2.7.4)
       faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
@@ -29,19 +32,25 @@ GEM
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     json (2.6.3)
+    jwt (2.7.1)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
     minitest (5.18.0)
+    nokogiri (1.15.3-x86_64-linux)
+      racc (~> 1.4)
     parallel (1.22.1)
     parser (3.2.2.0)
       ast (~> 2.4.1)
+    prometheus-client (2.1.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.10.1)
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     public_suffix (5.0.1)
+    racc (1.7.1)
+    rack (3.0.8)
     rainbow (3.1.1)
     regexp_parser (2.7.0)
     rexml (3.2.5)
@@ -73,6 +82,7 @@ GEM
     rubocop-performance (1.16.0)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
+    ruby-next-core (0.15.3)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     simplecov (0.22.0)
@@ -85,14 +95,27 @@ GEM
       language_server-protocol (~> 3.17.0.2)
       rubocop (~> 1.48.1)
       rubocop-performance (~> 1.16.0)
+    telephone_number (1.4.20)
     thor (1.2.2)
+    twilio-ruby (6.3.0)
+      faraday (>= 0.9, < 3.0)
+      jwt (>= 1.5, < 3.0)
+      nokogiri (>= 1.6, < 2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
     webmock (3.18.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    yabeda (0.11.0)
+      anyway_config (>= 1.0, < 3)
+      concurrent-ruby
+      dry-initializer
+    yabeda-prometheus (0.8.0)
+      prometheus-client (>= 0.10, < 3.0)
+      rack
+      yabeda (~> 0.10)
 
 GEM
   remote: https://rubygems.pkg.github.com/mlibrary/
@@ -103,6 +126,7 @@ GEM
       faraday-retry
       httpx
       rexml
+    sftp (0.2.1)
 
 PLATFORMS
   x86_64-linux
@@ -112,10 +136,15 @@ DEPENDENCIES
   pry
   pry-byebug
   rspec
+  sftp!
   simplecov
   standard
+  telephone_number
   thor
+  twilio-ruby
   webmock
+  yabeda
+  yabeda-prometheus
 
 BUNDLED WITH
    2.4.10

bin/set_up_development_ssh_keys.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+echo "📋 Check for dependencies"
+if ! command -v ssh-keygen &> /dev/null
+then
+    echo "❌ ssh-keygen could not be found"
+    exit
+fi
+echo "✅ ssh-keygen found"
+
+
+echo "🧹 clear out any existing ssh keys"
+rm -f .ssh/* 
+rm -f sftp/ssh/ssh_host*
+rm -f ssh_host*
+
+
+echo "🔑 generate the host ssh keys for the sftp service"
+ssh-keygen -t ed25519 -f ssh_host_ed25519_key < /dev/null
+ssh-keygen -t rsa -b 4096 -f ssh_host_rsa_key < /dev/null
+
+
+echo "🛻 move the keys into the sftp/ssh directory so they can be picked up"
+echo "by docker-compose bind mounts for the sftp service"
+mv ssh_host_ed25519_key sftp/ssh/
+mv ssh_host_rsa_key sftp/ssh/
+#this seems to be necessary
+mv ssh_host_rsa_key.pub .ssh/known_hosts
+
+echo "🧹 remove the unnecessary files"
+rm ssh_host*
+
+echo "🔑 generate actual host login keys"
+ssh-keygen -t rsa -b 4096 -f ssh_client_rsa_key < /dev/null
+
+echo "🛻 move the keys into the sftp/ssh directory"
+mv ssh_client_rsa_key.pub sftp/ssh/
+mv ssh_client_rsa_key sftp/ssh/

bin/sftp

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sftp -oIdentityFile=${SFTP_SSH_KEY_PATH} -oStrictHostKeyChecking=no ${SFTP_USER}@${SFTP_HOST}

bin/sms/setup.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+#must be run from the project root directory
+cp ./spec/fixtures/sms/sample_message.txt ./sftp/sms/FulSomeFile.txt
+rm -f ./sftp/sms/processed/*.txt

docker-compose.yml

@@ -8,8 +8,30 @@ services:
     volumes:
       - .:/app
       - gem_cache:/gems
+      - ./sftp/ssh/ssh_client_rsa_key:/etc/secret-volume/id_rsa:ro
     env_file:
       - .env
+      - env.development
+    command:
+      - "tail" 
+      - "-f"
+      - "/dev/null"
+
+
+  sftp:
+    image: 'atmoz/sftp'
+    volumes:
+      - ./sftp/sms:/home/alma/sms
+      - ./sftp/ssh/ssh_client_rsa_key.pub:/home/alma/.ssh/keys/id_rsa.pub:ro
+      - ./sftp/ssh/ssh_host_ed25519_key:/etc/ssh/ssh_host_ed25519_key
+      - ./sftp/ssh/ssh_host_rsa_key:/etc/ssh/ssh_host_rsa_key
+
+    command: alma:1001
+
+  pushgateway:
+    image: prom/pushgateway
+    ports:
+      - 9091:9091
 
 volumes:
   gem_cache:

env.development

@@ -0,0 +1,7 @@
+SMS_DIR='sms'
+PROCESSED_SMS_DIR='sms/processed'
+SFTP_HOST='sftp'
+SFTP_USER='alma'
+SFTP_SSH_KEY_PATH="/etc/secret-volume/id_rsa"
+PROMETHEUS_JOB_NAME=send_alma_sms
+PROMETHEUS_PUSH_GATEWAY=http://pushgateway:9091

env.example

@@ -2,6 +2,15 @@ BUNDLE_RUBYGEMS__PKG__GITHUB__COM="your_github_personal_access_token_with_read_p
 
 ALMA_API_KEY='your_alma_api_key'
 ALMA_API_HOST='https://api-na.hosted.exlibrisgroup.com'
+
+# student_users
 STUDENT_USERS_SET_ID="your_users_set_id"
+
+# hathitrust
 DIGIFEEDS_SET_ID="some_set_id"
 CHANGE_PHYSICAL_ITEM_INFORMATION_JOB_ID="some_job_id"
+
+# sms
+MESSAGING_SERVICE_SID="your_messaging_service_id"
+TWILIO_ACCT_SID='twilio_acct_sid'
+TWILIO_AUTH_TOKEN='twilio_auth_token'

init.sh

@@ -1,3 +1,6 @@
+#!/bin/bash
+
+#must be run from the project root directory
 if [ -f ".env" ]; then
   echo "🌎 .env exists. Leaving alone"
 else
@@ -12,6 +15,12 @@ else
   cp .github/pre-commit .git/hooks/pre-commit
 fi
 
+echo "🔑 Set up ssh keys for mock sftp server"
+./bin/set_up_development_ssh_keys.sh
+
+echo "🛠️ Set up sftp/sms directory for sms scripts"
+./bin/sms/setup.sh
+
 echo "🚢 Build docker images"
 docker-compose build
 

lib/aim.rb

@@ -5,6 +5,7 @@
 require "thor"
 require "aim/student_workers"
 require "aim/hathi_trust"
+require "aim/sms"
 require "aim/cli"
 
 module AIM

lib/aim/cli.rb

@@ -23,10 +23,37 @@ def expire_passwords
       end
     end
 
+    class SMS < Thor
+      option :nosend, type: :boolean, desc: "don't actually send with the Twilio API"
+      desc "send", "sends sms messages that were deposited by Alma"
+      long_desc <<~DESC
+        Looks in the sms folder for sms message files deposited by alma. Sends
+        the message via the Twilio API. Moves the message into the processed
+        folder within the sms folder on the sftp server.
+      DESC
+      def send
+        start_time = Time.now.to_i
+
+        AIM::SMS.configure
+        params = {}
+        if options[:nosend]
+          params[:sender] = AIM::SMS::Sender.new(AIM::SMS::FakeTwilioClient.new)
+        end
+
+        results = AIM::SMS::Processor.new(**params).run
+        results[:start_time] = start_time
+
+        AIM::SMS.send_metrics(results)
+      end
+    end
+
     desc "ht SUBCOMMAND", "commands related to the HathiTrust and Google Books project"
     subcommand "ht", HathiTrust
 
     desc "student_workers SUBCOMMAND", "commands related to student workers"
     subcommand "student_workers", StudentWorkers
+
+    desc "sms SUBCOMMAND", "commands related to student workers"
+    subcommand "sms", SMS
   end
 end

lib/aim/sms.rb

@@ -0,0 +1,37 @@
+require "json"
+require "twilio-ruby"
+require "telephone_number"
+require "fileutils"
+require "logger"
+require "yabeda"
+require "yabeda/prometheus"
+
+require "sftp"
+
+require "aim/sms/sender"
+
+module AIM
+  module SMS
+    def self.configure
+      ::Yabeda.configure do
+        gauge :aim_send_sms_last_success, comment: "Time that Alma sms messages were successfully sent"
+        gauge :aim_send_sms_num_messages_sent, comment: "Number of Alma sms messages sent in a job"
+        gauge :aim_send_sms_num_messages_not_sent, comment: "Number of Alma sms messages that caused an error"
+        gauge :aim_send_sms_num_messages_error, comment: "Number of Alma sms messages that were NOT successfully sent"
+      end
+      Yabeda.configure!
+    end
+
+    def self.send_metrics(results)
+      Yabeda.aim_send_sms_num_messages_sent.set({}, results[:num_files_sent])
+      Yabeda.aim_send_sms_num_messages_not_sent.set({}, results[:num_files_not_sent])
+      Yabeda.aim_send_sms_num_messages_error.set({}, results[:num_files_error])
+      Yabeda.aim_send_sms_last_success.set({}, results[:start_time])
+      begin
+        Yabeda::Prometheus.push_gateway.add(Yabeda::Prometheus.registry)
+      rescue
+        Logger.new($stdout).error("Failed to contact the push gateway")
+      end
+    end
+  end
+end