Merge pull request #453 from codeforjapan/develop

feat: add support for heic format

Author: ayuki-joto

.github/workflows/deploy.yml

@@ -17,7 +17,7 @@ jobs:
       eb-environment-name: ${{ steps.output-app-env.outputs.eb-environment-name }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set env to staging
         id: set-env-staging
@@ -45,7 +45,7 @@ jobs:
     needs: create-app-env
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -77,7 +77,7 @@ jobs:
       - build
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1

.github/workflows/lint.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 1
     - name: apt-get

.github/workflows/rails-test.yml

@@ -17,6 +17,7 @@ jobs:
       DATABASE_USERNAME: postgres
       DATABASE_PASSWORD: postgres
       RAILS_ENV: test
+      IMAGEMAGICK_SRC: 7.1.0-50.tar.gz
 
     services:
       db:
@@ -37,21 +38,51 @@ jobs:
         - 6379/tcp
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         fetch-depth: 1
     - name: apt-get
       run: |
         sudo apt-get update -y
         sudo apt-get -yqq install libpq-dev postgresql-client
+
+    - name: remove imagemagick 6
+      run: sudo apt remove imagemagick
+    - name: setup imagemagick 7
+      run: |
+          sudo sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
+          sudo apt update
+          sudo apt install build-essential
+          sudo apt build-dep imagemagick
+          sudo apt install libwebp-dev libopenjp2-7-dev librsvg2-dev libde265-dev
+    - name: install libheif
+      run: |
+          git clone https://github.com/strukturag/libheif.git
+          cd libheif
+          ./autogen.sh
+          ./configure --disable-go --disable-examples
+          make
+          sudo make install
+    - name: install imagemagick 7
+      run: |
+          wget https://github.com/ImageMagick/ImageMagick/archive/refs/tags/${IMAGEMAGICK_SRC}
+          tar xf ${IMAGEMAGICK_SRC}
+          cd ImageMagick-7*
+          ./configure
+          make
+          sudo make install
+          sudo ldconfig
+    - name: check imagemagick
+      run: convert -version
+
     - name: Set up Ruby 2.7
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: 2.7
         bundler-cache: true
 
     - name: setup Node
-      uses: actions/setup-node@v2
+      uses: actions/setup-node@v3
       with:
         node-version: 16.9.1
         cache: 'yarn'

Gemfile

@@ -8,20 +8,23 @@ gem "decidim", "0.25.2"
 
 gem "decidim-comments", path: "decidim-comments"
 
-gem "decidim-decidim_awesome", "~> 0.8.1"
+gem "decidim-decidim_awesome", git: "https://github.com/codeforjapan/decidim-module-decidim_awesome.git", branch: "v0.8.3-2022-10-20"
 
 gem "decidim-term_customizer", git: "https://github.com/codeforjapan/decidim-module-term_customizer.git", branch: "025-ja"
 
 gem "bootsnap"
 
 gem "puma", ">= 5.0.0"
+gem "puma_worker_killer"
+
 gem "uglifier", "~> 4.1"
 
 gem "faker", "~> 2.14"
 
 gem "wicked_pdf", "~> 2.1"
 
 gem "deface"
+gem "image_processing"
 gem "newrelic_rpm"
 
 gem "omniauth-line_login", path: "omniauth-line_login"

Gemfile.lock

@@ -1,3 +1,13 @@
+GIT
+  remote: https://github.com/codeforjapan/decidim-module-decidim_awesome.git
+  revision: 658d226e9ed164ffdca2efbde4dfaf79b02c295e
+  branch: v0.8.3-2022-10-20
+  specs:
+    decidim-decidim_awesome (0.8.3)
+      decidim-admin (>= 0.25.0, < 0.27)
+      decidim-core (>= 0.25.0, < 0.27)
+      sassc (~> 2.3)
+
 GIT
   remote: https://github.com/codeforjapan/decidim-module-term_customizer.git
   revision: a0ccb0b32d6d7943af90ff4f8029fd4e7b2afa8f
@@ -293,10 +303,6 @@ GEM
     decidim-debates (0.25.2)
       decidim-comments (= 0.25.2)
       decidim-core (= 0.25.2)
-    decidim-decidim_awesome (0.8.2)
-      decidim-admin (>= 0.25.0, < 0.27)
-      decidim-core (>= 0.25.0, < 0.27)
-      sassc (~> 2.3)
     decidim-dev (0.25.2)
       axe-core-rspec (~> 4.1.0)
       byebug (~> 11.0)
@@ -459,6 +465,8 @@ GEM
       activesupport (>= 4.1, < 7.1)
       railties (>= 4.1, < 7.1)
     geocoder (1.7.5)
+    get_process_mem (0.2.7)
+      ffi (~> 1.0)
     globalid (1.0.0)
       activesupport (>= 5.0)
     graphql (1.13.12)
@@ -610,6 +618,9 @@ GEM
     public_suffix (4.0.7)
     puma (5.6.4)
       nio4r (~> 2.0)
+    puma_worker_killer (0.3.1)
+      get_process_mem (~> 0.2)
+      puma (>= 2.7)
     racc (1.6.0)
     rack (2.2.3)
     rack-attack (6.6.1)
@@ -852,7 +863,7 @@ DEPENDENCIES
   byebug (~> 11.0)
   decidim (= 0.25.2)
   decidim-comments!
-  decidim-decidim_awesome (~> 0.8.1)
+  decidim-decidim_awesome!
   decidim-dev (= 0.25.2)
   decidim-term_customizer!
   decidim-user_extension!
@@ -862,12 +873,14 @@ DEPENDENCIES
   faker (~> 2.14)
   figaro
   fog-aws
+  image_processing
   letter_opener_web (~> 1.3)
   listen (~> 3.1)
   newrelic_rpm
   omniauth-line_login!
   omniauth-rails_csrf_protection
   puma (>= 5.0.0)
+  puma_worker_killer
   rspec-rails
   rubocop-faker
   rubyzip (>= 1.0.0)

app/packs/src/decidim/decidim_awesome/editors/editor.js

@@ -355,7 +355,7 @@ export function createQuillEditor(container) {
       modules: ["Resize", "DisplaySize"]
     }
     modules.imageUpload = {
-      url: $(container).data("uploadImagesPath") || DecidimAwesome.editor_uploader_path,
+      url: DecidimAwesome.editor_uploader_path,
       method: "POST",
       name: "image",
       withCredentials: false,
@@ -366,7 +366,18 @@ export function createQuillEditor(container) {
       },
       callbackKO: (serverError) => {
         $("div.ql-toolbar").last().removeClass("editor-loading")
-        console.error(`Image upload error: ${serverError.message}`);
+        let msg = serverError && serverError.body;
+        try {
+          msg = JSON.parse(msg).message;
+        } catch (e) { console.error("Parsing error", e); }
+        console.error(`Image upload error: ${msg}`);
+        let $p = $(`<p class="text-alert help-text">${msg}</p>`);
+        $(container).after($p)
+        setTimeout(() => {
+          $p.fadeOut(1000, () => {
+            $p.destroy();
+          });
+        }, 3000);
       },
       checkBeforeSend: (file, next) => {
         $("div.ql-toolbar").last().addClass("editor-loading")

app/previewers/heic_previewer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class HeicPreviewer < ActiveStorage::Previewer
+  CONTENT_TYPES = %w(image/heic image/heif image/heic-sequence image/heif-sequence).freeze
+
+  class << self
+    def accept?(blob)
+      CONTENT_TYPES.include?(blob.content_type) && minimagick_exists?
+    end
+
+    def minimagick_exists?
+      return @minimagick_exists if @minimagick_exists.present?
+
+      @minimagick_exists = defined?(ImageProcessing::MiniMagick)
+      Rails.logger.error "#{self.class} :: MiniMagick is not installed" unless @minimagick_exists
+
+      @minimagick_exists
+    end
+  end
+
+  def preview(**_options)
+    download_blob_to_tempfile do |input|
+      begin
+        io = ImageProcessing::MiniMagick.source(input).convert("png").call
+      rescue ImageProcessing::Error
+        io = ImageProcessing::MiniMagick.loader(page: 0).source(input).convert("png").call
+      end
+      yield io: io, filename: "#{blob.filename.base}.png", content_type: "image/png"
+    end
+  end
+end

config/initializers/active_storage.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.active_storage.previewers << HeicPreviewer
+  config.active_storage.variable_content_types << "image/heic"
+  config.active_storage.variable_content_types << "image/heif"
+end

config/initializers/decidim.rb

@@ -242,4 +242,6 @@
 Decidim.register_assets_path File.expand_path("app/packs", Rails.application.root)
 
 # Set max_complexity of GraphQL::Schema
-Decidim::Api::Schema.max_complexity = 100_000
+Rails.application.config.to_prepare do
+  Decidim::Api::Schema.max_complexity = 100_000
+end

config/puma.rb

@@ -20,13 +20,25 @@
 # Specifies the `pidfile` that Puma will use.
 pidfile ENV.fetch("PIDFILE", "tmp/pids/server.pid")
 
+before_fork do
+  PumaWorkerKiller.config do |config|
+    config.ram = 2048
+    config.frequency = 60
+    config.percent_usage = 0.9
+    config.rolling_restart_frequency = 24 * 60 * 60
+    config.reaper_status_logs = true
+    config.pre_term = ->(worker) { puts "Worker #{worker.index}(#{worker.pid}) being killed" }
+  end
+  PumaWorkerKiller.start
+end
+
 # Specifies the number of `workers` to boot in clustered mode.
 # Workers are forked webserver processes. If using threads and workers together
 # the concurrency of the application would be max `threads` * `workers`.
 # Workers do not work on JRuby or Windows (both of which do not support
 # processes).
 #
-# workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+workers ENV.fetch("WEB_CONCURRENCY", 2)
 
 # Use the `preload_app!` method when specifying a `workers` number.
 # This directive tells Puma to first boot the application and load code

docker-compose-etherpad.yml

@@ -11,8 +11,8 @@ services:
   nginx-proxy:
     image: nginx:1.21
     volumes:
-      - ./deployments/etc/nginx/nginx.conf:/etc/nginx/nginx.conf
-      - ./deployments/etc/nginx/conf.d:/etc/nginx/conf.d
+      - ./etc/nginx/nginx.conf:/etc/nginx/nginx.conf
+      - ./etc/nginx/conf.d:/etc/nginx/conf.d
     environment:
       TZ: Asia/Tokyo
     ports: