Merge branch 'release-1.1.0'

CHANGELOG.md

@@ -1,3 +1,14 @@
+### 1.1.0
+
+**Compliance Tool:**
+
+* Internal shipments API useful for the Compliance Tool
+* Logic and mviews for non-compliant shipments
+
+### 1.0.1
+
+* Hotfix that enables nomenclature changes in production and removes exception notifier.
+
 ### 1.0.0
 
 **Species+ Admin:**
@@ -212,7 +223,7 @@ Bug-fix release (trade bulk update issue)
 * adding functionality in the S+ admin panel to manage ranks & change types translations
 * adding functionality in the S+ admin panel to set is_current on CitesCop events
 * adding import scripts to populate translations of: ranks, change_types, Cites regions, hash annotations
-* adding import script to populate CoP start dates 
+* adding import script to populate CoP start dates
 
 ### 0.7.8 (2014-07-17)
 Upgrade to ruby 2.0
@@ -319,7 +330,7 @@ table
 * adds search by common names to Species+
 * adds search by territories to Species+
 * adds a number of features and bugfixes for species management tool
- 
+
 ### 0.3.6 (2013-11-22)
 * fixes some IE issues
 * upgrades newrelic agent

Gemfile

@@ -30,6 +30,8 @@ gem 'sidekiq', '< 5'
 gem 'sidekiq-status'
 gem 'sidekiq-unique-jobs', git: 'git://github.com/mhenrixon/sidekiq-unique-jobs'
 
+gem 'redis-rails'
+
 gem 'whenever', :require => false
 
 gem 'ember-rails'

Gemfile.lock

@@ -336,6 +336,22 @@ GEM
     rdoc (3.12.2)
       json (~> 1.4)
     redis (3.3.0)
+    redis-actionpack (3.2.4)
+      actionpack (~> 3.2.0)
+      redis-rack (~> 1.4.4)
+      redis-store (~> 1.1.4)
+    redis-activesupport (3.2.5)
+      activesupport (~> 3.2.0)
+      redis-store (~> 1.1.0)
+    redis-rack (1.4.4)
+      rack (~> 1.4.0)
+      redis-store (~> 1.1.4)
+    redis-rails (3.2.4)
+      redis-actionpack (~> 3.2.4)
+      redis-activesupport (~> 3.2.4)
+      redis-store (~> 1.1.4)
+    redis-store (1.1.7)
+      redis (>= 2.2)
     referer-parser (0.2.2)
     request_store (1.3.2)
     responders (0.9.3)
@@ -527,6 +543,7 @@ DEPENDENCIES
   rails (= 3.2.22.2)
   rails-secrets
   rake (~> 10.0.3)
+  redis-rails
   request_store (~> 1.1)
   rest-client
   rspec-mocks
@@ -557,4 +574,4 @@ DEPENDENCIES
   yajl-ruby
 
 BUNDLED WITH
-   1.10.6
+   1.16.2

app/controllers/api/v1/shipments_controller.rb

@@ -0,0 +1,118 @@
+class Api::V1::ShipmentsController < ApplicationController
+  respond_to :json
+
+  before_filter :authenticate
+
+  GROUPING_ATTRIBUTES = {
+    category: ['issue_type'],
+    commodity: ['term', 'term_id'],
+    exporting: ['exporter', 'exporter_iso', 'exporter_id'],
+    importing: ['importer', 'importer_iso', 'importer_id'],
+    species: ['taxon_name', 'appendix', 'taxon_concept_id'],
+    taxonomy: [''],
+  }
+
+  def index
+    @search = Trade::Filter.new(search_params)
+    render :json => @search.results,
+      :each_serializer => Trade::ShipmentApiFromViewSerializer,
+      :meta => metadata_for_search(@search)
+  end
+
+  def chart_query
+    @chart_data = Rails.cache.fetch(['chart_data', params], expires_in: 1.week) do
+                    Trade::ComplianceGrouping.new('year', {attributes: ['issue_type']})
+                                             .countries_reported_range(params[:year])
+                  end
+    render :json => @chart_data
+  end
+
+  def grouped_query
+    limit = params[:limit].present? ? params[:limit].to_i : ''
+    years_range = "year >= 2012 AND year <= #{Date.today.year - 1}"
+    query = Trade::ComplianceGrouping.new('year', {attributes: sanitized_attributes, condition: years_range, limit: limit })
+    data = query.run
+    params_hash = {}
+    sanitized_attributes.map { |p| params_hash[p] = p }
+    @grouped_data = Rails.cache.fetch(['grouped_data', params], expires_in: 1.week) do
+                      sanitized_attributes.first.empty? ? query.taxonomic_grouping :
+                                                          query.json_by_year(data, params, params_hash)
+                    end
+    render :json =>  @grouped_data
+  end
+
+  def search_query
+    query = Trade::ComplianceGrouping.new('year', {attributes: sanitized_attributes, condition: "year = #{params[:year]}"})
+    data = query.run
+    @search_data =  Rails.cache.fetch(['search_data', params], expires_in: 1.week) do
+                      query.build_hash(data, params)
+                    end
+    @filtered_data = query.filter(@search_data, params)
+    render :json => Kaminari.paginate_array(@filtered_data).page(params[:page]).per(params[:per_page]),
+           :meta => metadata(@filtered_data, params)
+  end
+
+  def download_data
+    @download_data = Rails.cache.fetch(['download_data', params], expires_in: 1.week) do
+                       Trade::DownloadDataRetriever.dashboard_download(download_params).to_a
+                     end
+    render :json => @download_data
+  end
+
+  def search_download_data
+    @download_data = Rails.cache.fetch(['search_download_data', params], expires_in: 1.week) do
+                       Trade::DownloadDataRetriever.search_download(download_params).to_a
+                     end
+    render :json => @download_data
+  end
+
+  def search_download_all_data
+    query = Trade::ComplianceGrouping.new('year', {attributes: sanitized_attributes, condition: "year = #{params[:year]}"})
+    data = query.run
+    @search_download_all_data = Rails.cache.fetch(['search_download_all_data', params], expires_in: 1.week) do
+                                  search_data = query.build_hash(data, params)
+                                  filtered_data = query.filter(search_data, params)
+                                  data_ids = query.filter_download_data(filtered_data, params)
+                                  hash_params = params_hash_builder(data_ids, download_params)
+                                  Trade::DownloadDataRetriever.search_download(hash_params).to_a
+                                end
+    render :json => @search_download_all_data
+  end
+
+  private
+
+  def params_hash_builder(ids, params)
+    hash_params = {}
+    hash_params[:ids] = ids.join(',')
+    hash_params.merge!(params)
+    hash_params.symbolize_keys
+  end
+
+  def metadata(data, params)
+    {
+      :total => data.count,
+      :page => params[:page] || 1,
+      :per_page => params[:per_page] || 25
+    }
+  end
+
+  def search_params
+    params.permit(:compliance_type, :time_range_start, :time_range_end, :page, :per_page)
+  end
+
+  def download_params
+    params.permit(:year, :ids, :compliance_type, :type, :group_by, :appendix)
+  end
+
+  def sanitized_attributes
+    GROUPING_ATTRIBUTES[params[:group_by].to_sym]
+  end
+
+  def authenticate
+    token = request.headers['X-Authentication-Token']
+    unless token == Rails.application.secrets["compliance_tool_token"]
+      head status: :unauthorized
+      return false
+    end
+  end
+end

app/models/trade/filter.rb

@@ -26,7 +26,7 @@ def initialize_params(options)
   end
 
   def initialize_query
-    @query = Trade::Shipment.from('trade_shipments_with_taxa_view trade_shipments')
+    @query = Trade::Shipment.from("#{@shipments_view} trade_shipments")
 
     unless @taxon_concepts_ids.empty?
       cascading_ranks =

app/models/trade/search_params.rb

@@ -33,6 +33,7 @@ def initialize(params)
       internal: sanitise_boolean(params[:internal]),
       report_type: sanitise_symbol(params[:report_type], :raw),
       taxon_with_descendants: sanitise_boolean(params[:taxon_with_descendants]),
+      shipments_view: sanitise_compliance_type(params[:compliance_type]),
       page: sanitise_positive_integer(params[:page], 1),
       per_page: sanitise_positive_integer(params[:per_page], 100)
     }
@@ -45,4 +46,15 @@ def self.sanitize(params)
     new(params)
   end
 
+  private
+
+  COMPLIANCE_TYPES_VIEWS = {
+    appendix_i: "trade_shipments_appendix_i_mview",
+    trade_suspensions: "trade_shipments_cites_suspensions_mview",
+    mandatory_quotas: "trade_shipments_mandatory_quotas_mview"
+  }
+  def sanitise_compliance_type(compliance_type)
+    compliance_type.present? ? COMPLIANCE_TYPES_VIEWS[compliance_type.to_sym] : "trade_shipments_with_taxa_view"
+  end
+
 end

app/serializers/trade/shipment_api_from_view_serializer.rb

@@ -0,0 +1,48 @@
+class Trade::ShipmentApiFromViewSerializer < ActiveModel::Serializer
+   attributes :id , :year, :appendix, :taxon_name, :class_name, :order_name, :family_name, :genus_name,
+              :term, :importer_reported_quantity, :exporter_reported_quantity,
+              :unit, :importer, :importer_iso, :exporter, :exporter_iso, :origin, :purpose, :source,
+              :import_permit, :export_permit, :origin_permit, :issue_type, :rank_name
+
+
+  def importer
+    object.importer || object.attributes["importer"]
+  end
+
+  def exporter
+    object.exporter || object.attributes["exporter"]
+  end
+
+  def origin
+    object.origin || object.attributes["origin"]
+  end
+
+  def term
+    object.term || object.attributes["term"]
+  end
+
+  def unit
+    object.unit || object.attributes["unit"]
+  end
+
+  def source
+    object.source || object.attributes["source"]
+  end
+
+  def purpose
+    object.purpose || object.attributes["purpose"]
+  end
+
+  def class_name
+    object.attributes["class_name"]
+  end
+
+  def importer_reported_quantity
+    object.attributes['importer_reported_quantity'] || object.attributes['importer_quantity']
+  end
+
+  def exporter_reported_quantity
+    object.attributes['exporter_reported_quantity'] || object.attributes['exporter_quantity']
+  end
+
+end

config/environments/development.rb

@@ -11,7 +11,8 @@
 
   # Show full error reports and disable caching
   config.consider_all_requests_local       = true
-  config.action_controller.perform_caching = false
+  config.action_controller.perform_caching = true
+  config.cache_store = :redis_store
 
   # Don't care if the mailer can't send
   config.action_mailer.raise_delivery_errors = false

config/environments/staging.rb

@@ -7,6 +7,7 @@
   # Full error reports are disabled and caching is turned on
   config.consider_all_requests_local       = false
   config.action_controller.perform_caching = true
+  config.cache_store = :redis_store
 
   # Disable Rails's static asset server (Apache or nginx will already do this)
   config.serve_static_assets = false

config/routes.rb

@@ -34,6 +34,13 @@
       resources :events, only: [:index]
       resources :document_tags, only: [:index]
       match '/dashboard_stats/:iso_code' => 'dashboard_stats#index'
+      resources :shipments, only: [:index]
+      get '/shipments/chart' => 'shipments#chart_query'
+      get '/shipments/grouped' => 'shipments#grouped_query'
+      get '/shipments/search' => 'shipments#search_query'
+      get '/shipments/download' => 'shipments#download_data'
+      get '/shipments/search_download' => 'shipments#search_download_data'
+      get '/shipments/search_download_all' => 'shipments#search_download_all_data'
     end
     resources :languages, :only => [:index]
     resources :users, :only => [:index]

config/schedule.rb

@@ -1,6 +1,6 @@
 set :output, 'log/cron.log'
 
-every :day, :at => '2:42am' do
+every :day, :at => '1:42am' do
   rake "db:migrate:rebuild"
 end
 

db/migrate/20180509135814_create_trade_shipments_appendix_i_view.rb

@@ -0,0 +1,14 @@
+class CreateTradeShipmentsAppendixIView < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_appendix_i_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_appendix_i_view"
+
+    execute "CREATE VIEW trade_shipments_appendix_i_view AS #{view_sql('20180509135814', 'trade_shipments_appendix_i_view')}"
+    execute "CREATE MATERIALIZED VIEW trade_shipments_appendix_i_mview AS SELECT * FROM trade_shipments_appendix_i_view"
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_appendix_i_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_appendix_i_view"
+  end
+end

db/migrate/20180601083440_create_trade_cites_suspension_shipments_view.rb

@@ -0,0 +1,14 @@
+class CreateTradeCitesSuspensionShipmentsView < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_cites_suspensions_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_cites_suspensions_view"
+
+    execute "CREATE VIEW trade_shipments_cites_suspensions_view AS #{view_sql('20180601083440', 'trade_shipments_cites_suspensions_view')}"
+    execute "CREATE MATERIALIZED VIEW trade_shipments_cites_suspensions_mview AS SELECT * FROM trade_shipments_cites_suspensions_view"
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_cites_suspensions_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_cites_suspensions_view"
+  end
+end

db/migrate/20180627135001_create_trade_shipments_mandatory_quotas_view.rb

@@ -0,0 +1,14 @@
+class CreateTradeShipmentsMandatoryQuotasView < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_mandatory_quotas_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_mandatory_quotas_view"
+
+    execute "CREATE VIEW trade_shipments_mandatory_quotas_view AS #{view_sql('20180627135001', 'trade_shipments_mandatory_quotas_view')}"
+    execute "CREATE MATERIALIZED VIEW trade_shipments_mandatory_quotas_mview AS SELECT * FROM trade_shipments_mandatory_quotas_view"
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_mandatory_quotas_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_mandatory_quotas_view"
+  end
+end

db/migrate/20180705094119_update_trade_shipments_appendix_i_view.rb

@@ -0,0 +1,14 @@
+class UpdateTradeShipmentsAppendixIView < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_appendix_i_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_appendix_i_view"
+
+    execute "CREATE VIEW trade_shipments_appendix_i_view AS #{view_sql('20180705094119', 'trade_shipments_appendix_i_view')}"
+    execute "CREATE MATERIALIZED VIEW trade_shipments_appendix_i_mview AS SELECT * FROM trade_shipments_appendix_i_view"
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_appendix_i_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_appendix_i_view"
+  end
+end

db/migrate/20180713123936_update_trade_shipments_cites_suspensions_view.rb

@@ -0,0 +1,14 @@
+class UpdateTradeShipmentsCitesSuspensionsView < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_cites_suspensions_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_cites_suspensions_view"
+
+    execute "CREATE VIEW trade_shipments_cites_suspensions_view AS #{view_sql('20180713123936', 'trade_shipments_cites_suspensions_view')}"
+    execute "CREATE MATERIALIZED VIEW trade_shipments_cites_suspensions_mview AS SELECT * FROM trade_shipments_cites_suspensions_view"
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS trade_shipments_cites_suspensions_mview CASCADE"
+    execute "DROP VIEW IF EXISTS trade_shipments_cites_suspensions_view"
+  end
+end