Archival exports (#511, #512, #513, #514)

Gemfile

@@ -29,7 +29,7 @@ gem 'rack-cors', :require => 'rack/cors'
 gem 'pg_search'
 gem 'figaro'
 gem 'open-uri'
-gem 'storyblok-richtext-renderer', github: 'performant-software/storyblok-ruby-richtext-renderer', ref: '0a6c2e8e81560311569d49d06c0e32abd0effcd5'
+gem 'storyblok-richtext-renderer', github: 'performant-software/storyblok-ruby-richtext-renderer', ref: 'bef6903146426e01175887eb92a75bf9bac4c3cb'
 gem 'sidekiq', '~>6.5.1'
 gem 'sidekiq-status', '~>2.1.1'
 
@@ -39,7 +39,7 @@ group :development, :test do
 end
 
 group :development do
-  gem 'listen', '>= 3.0.5', '< 3.2'
+  gem 'listen', '>= 3.3.0', '< 4.0'
   # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
   gem 'spring'
   gem 'spring-watcher-listen', '~> 2.0.0'
@@ -56,3 +56,4 @@ gem 'psych', '< 4'
 gem 'dotenv-rails', '~> 2.8'
 # pinned to < 9.3 until https://github.com/ilyakatz/data-migrate/issues/302 resolved
 gem 'data_migrate', '~> 9.2', '< 9.3'
+gem 'rubyzip', '~> 2.3'

Gemfile.lock

@@ -1,9 +1,9 @@
 GIT
   remote: https://github.com/performant-software/storyblok-ruby-richtext-renderer.git
-  revision: 0a6c2e8e81560311569d49d06c0e32abd0effcd5
-  ref: 0a6c2e8e81560311569d49d06c0e32abd0effcd5
+  revision: bef6903146426e01175887eb92a75bf9bac4c3cb
+  ref: bef6903146426e01175887eb92a75bf9bac4c3cb
   specs:
-    storyblok-richtext-renderer (0.0.6)
+    storyblok-richtext-renderer (0.0.10)
 
 GEM
   remote: https://rubygems.org/
@@ -143,9 +143,9 @@ GEM
       ruby-vips (>= 2.0.17, < 3)
     jmespath (1.6.2)
     jsonapi-renderer (0.2.2)
-    listen (3.0.8)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
+    listen (3.9.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.6.0)
     loofah (2.22.0)
       crass (~> 1.0.2)
@@ -243,6 +243,7 @@ GEM
     ruby-vips (2.2.2)
       ffi (~> 1.12)
       logger
+    rubyzip (2.3.2)
     sidekiq (6.5.12)
       connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
@@ -307,7 +308,7 @@ DEPENDENCIES
   figaro
   foreman
   image_processing (~> 1.12)
-  listen (>= 3.0.5, < 3.2)
+  listen (>= 3.3.0, < 4.0)
   mutex_m (~> 0.2.0)
   open-uri
   pg (>= 0.18, < 2.0)
@@ -317,6 +318,7 @@ DEPENDENCIES
   puma (~> 4.3)
   rack-cors
   rails (~> 6.1)
+  rubyzip (~> 2.3)
   sidekiq (~> 6.5.1)
   sidekiq-status (~> 2.1.1)
   spring

app/controllers/projects_controller.rb

@@ -1,8 +1,8 @@
 class ProjectsController < ApplicationController
-  before_action :set_project, only: [:show, :update, :destroy, :search, :check_in, :move_many]
+  before_action :set_project, only: [:show, :update, :destroy, :search, :check_in, :move_many, :create_export, :exports]
   before_action :validate_user_approved, only: [:create]
 
-  before_action only: [:update, :destroy] do
+  before_action only: [:update, :destroy, :create_export] do
     validate_user_admin(@project)
   end
 
@@ -71,6 +71,74 @@ def check_in
     render json: { checked_in_docs: checked_in_doc_ids }
   end
 
+  # GET /projects/1/exports
+  def exports
+    # completed exports
+    @exports = @project.exports.collect do |exp|
+      {
+        :id => exp.id,
+        :updated_at => exp.created_at,
+        :status => "Complete",
+        :url => Rails.application.routes.url_helpers.rails_blob_url(exp),
+      }
+    end
+    # queued exports
+    queues = Sidekiq::Queue.all
+    queues.each do |queue|
+      queue.each do |job|
+        if job.klass == "ExportProjectWorker" and job.args[0].to_i == @project.id
+          @exports.push({
+            :id => job.jid,
+            :status => "Queued",
+            :updated_at => job.created_at,
+          })
+        end
+      end
+    end
+    # in progress exports
+    worker_set = Sidekiq::WorkSet.new
+    worker_set.each do |_, _, worker|
+      if worker.is_a? Hash and worker["payload"]["class"] == "ExportProjectWorker" and worker["payload"]["args"][0].to_i == @project.id
+        exp = {
+          :id => worker["payload"]["jid"],
+          :updated_at => Time.at(worker["payload"]["created_at"]),
+          :status => "In progress",
+        }
+        status = Sidekiq::Status.get_all worker["payload"]["jid"]
+        if status
+          exp[:status] = "In progress (#{status['pct_complete']}%)"
+          exp[:updated_at] = Time.at(status["update_time"].to_f)
+        end
+        @exports.push(exp)
+      end
+    end
+    # errored exports
+    dead_set = Sidekiq::DeadSet.new
+    dead_set.each do |job|
+      if job.klass == "ExportProjectWorker" and job.args[0].to_i == @project.id
+        @exports.push({
+          :id => job.jid,
+          :status => "Failed",
+          :updated_at => job.created_at,
+          :error_message => job.item["error_message"],
+          :error_class => job.item["error_class"],
+        })
+      end
+    end
+    render json: @exports.sort_by { |hsh| hsh[:updated_at] }.reverse, status: 200
+  end
+
+  # POST /projects/1/create_export
+  def create_export
+    job_id = ExportProjectWorker.perform_async(@project.id)
+    @job = { id: job_id }
+    if @job
+      render json: @job, status: 202
+    else
+      render status: 500
+    end
+  end
+
   private
     # Use callbacks to share common setup or constraints between actions.
     def set_project

app/helpers/download_helper.rb

@@ -0,0 +1,18 @@
+module DownloadHelper
+  def self.download_to_file(uri)
+    begin
+      stream = URI.open(uri, :read_timeout => 10)
+      return stream if stream.respond_to?(:path) # Already file-like
+
+      # Workaround when open(uri) doesn't return File
+      Tempfile.new.tap do |file|
+        file.binmode
+        IO.copy_stream(stream, file)
+        stream.close
+        file.rewind
+      end
+    rescue Net::ReadTimeout
+      return 'failed'
+    end
+  end
+end

app/helpers/export_helper.rb

@@ -0,0 +1,132 @@
+module ExportHelper
+  def self.sanitize_filename(filename)
+    filename.gsub(/[\x00\/\\:\*\?\"<>\|]/, "_").strip
+  end
+
+  def self.get_path(link, current_depth)
+    document_id = link[:document_id]
+    highlight_uid = link[:highlight_uid]
+    # get a relative URL to a document by id, taking into account the current location
+    begin
+      document = Document.find(document_id)
+      filename = self.sanitize_filename(document.title).parameterize
+      path_segments = ["#{filename}.html"]
+      while document[:parent_type] != "Project"
+        # back out from the target document until we hit the project root
+        document = document.parent
+        path_segments.unshift(self.sanitize_filename(document[:title]).parameterize)
+      end
+      to_project_root = current_depth > 0 ? Array.new(current_depth, "..").join("/") + "/" : ""
+      path = to_project_root + path_segments.join("/")
+      if highlight_uid.present?
+        # append #highlight_uid to url if present in order to target the highlight
+        path = "#{path}#highlight-#{highlight_uid}"
+      end
+      return path
+    rescue ActiveRecord::RecordNotFound
+      return "#"
+    end
+  end
+
+  def self.get_svg_styles(obj)
+    # convert fabric object style properties to css
+    styles = [
+      "stroke: #{obj['stroke']};",
+      "fill: #{obj['fill'] || 'transparent'};",
+      "stroke-width: 3px;",
+      "stroke-linecap: #{obj['strokeLineCap'] || 'butt'};",
+      "stroke-dashoffset: #{obj['strokeDashOffset'] || '0'};",
+      "stroke-linejoin: #{obj['strokeLineJoin'] || 'miter'};",
+      "stroke-miterlimit: #{obj['strokeMiterLimit'] || '4'};",
+      "opacity: #{obj['opacity'] || '1'};",
+      "visibility: #{obj['visible'] ? 'visible' : 'hidden'};",
+    ]
+    styles.push("stroke-dasharray: #{obj['strokeDashArray']};") if obj['strokeDashArray']
+    styles.join(" ")
+  end
+
+  def self.get_svg_path(paths)
+    # convert fabric object path property to svg path
+    path = ''
+    paths.each_with_index do |ls, i|
+      path += " " unless i == 0
+      if ls[0] == "C"
+        # C x1 y1, x2 y2, x y
+        path += "#{ls[0]} #{ls[1]} #{ls[2]}, #{ls[3]} #{ls[4]}, #{ls[5]} #{ls[6]}"
+      elsif ls[0] == "S" or ls[0] == "Q"
+        # S x2 y2, x y || Q x1 y1, x y
+        path += "#{ls[0]} #{ls[1]} #{ls[2]}, #{ls[3]} #{ls[4]}"
+      else
+        # M x y || L x y || T x y, etc
+        path += ls.join(" ")
+      end
+    end
+    path
+  end
+
+  def self.fabric_to_svg(highlights)
+    # convert image annotation highlights (fabric objects) to svgs
+    svgs = []
+    self.order_highlights(highlights, "canvas", nil).each do |uid, hl|
+      svg_hash = JSON.parse(hl[:target])
+      elm = "#{svg_hash['type']}"
+      if svg_hash["path"]
+        # path
+        elm += " d=\"#{self.get_svg_path(svg_hash['path'])}\""
+      elsif svg_hash["points"]
+        # polyline
+        elm += ' points="'
+        elm += svg_hash["points"].map { |pt| "#{pt['x']},#{pt['y']}" }.join(" ")
+        elm += '"'
+      elsif svg_hash["type"] == "circle"
+        # circle
+        elm += " r=\"#{svg_hash['radius']}\""
+        cx = svg_hash["left"]
+        cy = svg_hash["top"]
+        if svg_hash["originX"] == "left"
+          cx += svg_hash["radius"]
+          cy += svg_hash["radius"]
+        end
+        elm += " cx=\"#{cx}\" cy=\"#{cy}\""
+      elsif svg_hash["type"] == "rect"
+        # rect
+        elm += " x=\"#{svg_hash['left']}\" y=\"#{svg_hash['top']}\""
+        elm += " width=\"#{svg_hash['width']}\" height=\"#{svg_hash['height']}\""
+      end
+      # common styles
+      elm += " style=\"#{self.get_svg_styles(svg_hash)}\""
+      svg_elm = "<#{elm} vector-effect=\"non-scaling-stroke\" />"
+      # add link to highlight in footer
+      svgs.push("<a id=\"highlight-#{uid}\" href=\"##{uid}\">#{svg_elm}</a>")
+    end
+    return svgs
+  end
+
+  def self.order_highlights(highlights, document_kind, content_html)
+    if !content_html.present? and document_kind == "text"
+      highlights
+    elsif content_html.present?
+      # text type document: order by position in text
+      highlights.sort_by { |uid, hl| content_html.index(uid) || Float::INFINITY }
+    elsif document_kind == "canvas"
+      # canvas type document: order highlights by position on page (top to bottom, LTR)
+      highlights.sort_by { |uid, hl|
+        drawing = JSON.parse(hl[:target])
+        # divide page vertically into 50px blocks; consider all within 50px range to have equal y
+        y = (drawing["top"] / 50).floor()
+        # then sort by x, unmodified
+        x = drawing["left"]
+        [y, x]
+      }
+    end
+  end
+
+  def self.get_link_label(link)
+    label = link[:document_title]
+    if link[:excerpt] and link[:excerpt].length > 0
+      label = "<span>#{link[:title] ? link[:title] : link[:excerpt]}</span>"
+      label += " in <em>#{link[:document_title]}</em>"
+    end
+    label
+  end
+end

app/models/document.rb

@@ -148,32 +148,15 @@ def color
     nil
   end
 
-  def download_to_file(uri)
-    begin
-      stream = URI.open(uri, :read_timeout => 10)
-      return stream if stream.respond_to?(:path) # Already file-like
-
-      # Workaround when open(uri) doesn't return File
-      Tempfile.new.tap do |file|
-        file.binmode
-        IO.copy_stream(stream, file)
-        stream.close
-        file.rewind
-      end
-    rescue Net::ReadTimeout
-      return 'failed'
-    end
-  end
-
   def add_thumbnail( image_url )
     begin
       # Try with PNG
-      opened = download_to_file(image_url)
+      opened = DownloadHelper.download_to_file(image_url)
     rescue OpenURI::HTTPError
       # Only JPG is required for IIIF level 1 compliance,
       # so if we get back a 400 error, use JPG for thumbnail
       with_jpg = image_url.sub('.png', '.jpg')
-      opened = download_to_file(with_jpg)
+      opened = DownloadHelper.download_to_file(with_jpg)
     end
     if opened != 'failed'
       processed = ImageProcessing::MiniMagick.source(opened)

app/models/highlight.rb

@@ -7,7 +7,7 @@ class Highlight < Linkable
 
   def links_to
     all_links = self.highlights_links.sort_by{ |hll| hll.position }.map{ |hll| Link.where(:id => hll.link_id).first }
-    result = all_links.map { |link| self.to_link_obj(link) }.compact
+    result = all_links.map { |link| self.to_link_obj(link) unless link.nil? }.compact
     result.each {|r| 
       if r[:highlight_id]
         hl = Highlight.where(:id => r[:highlight_id]).first
@@ -55,19 +55,6 @@ def add_link_from_duplication(linked, original_id, position)
     end
   end
 
-  def download_to_file(uri)
-    stream = URI.open(uri)
-    return stream if stream.respond_to?(:path) # Already file-like
-
-    # Workaround when open(uri) doesn't return File
-    Tempfile.new.tap do |file|
-      file.binmode
-      IO.copy_stream(stream, file)
-      stream.close
-      file.rewind
-    end
-  end
-
   def set_thumbnail( image_url, thumb_rect )
     if !thumb_rect.nil?
       pad_factor = 0.06
@@ -98,12 +85,12 @@ def set_thumbnail( image_url, thumb_rect )
     else
       begin
         # Try with PNG
-        opened = download_to_file(image_url)
+        opened = DownloadHelper.download_to_file(image_url)
       rescue OpenURI::HTTPError
         # Only JPG is required for IIIF level 1 compliance,
         # so if we get back a 400 error, use JPG for thumbnail
         with_jpg = image_url.sub('.png', '.jpg')
-        opened = download_to_file(with_jpg)
+        opened = DownloadHelper.download_to_file(with_jpg)
       end
       io = ImageProcessing::MiniMagick.source(opened)
         .resize_to_fill(80, 80)

app/models/project.rb

@@ -2,6 +2,7 @@ class Project < ApplicationRecord
   belongs_to :owner, class_name: 'User', optional: true
   has_many :documents, as: :parent
   has_many :document_folders, as: :parent
+  has_many_attached :exports
   has_many :user_project_permissions, dependent: :destroy
   has_many :users, through: :user_project_permissions