Merge branch 'searching_by_species_country_combinations'

Levia · Levia · commit 7721c93a6b0e · 2016-06-27T15:06:46.000+01:00
diff --git a/app/models/document_citation.rb b/app/models/document_citation.rb
@@ -30,4 +30,8 @@ def stringy_taxon_concept_ids=(ids)
     self.taxon_concept_ids = ids.split(',')
   end
 
+  after_destroy do |dc|
+    dc.document.touch
+  end
+
 end
diff --git a/app/models/document_citation_geo_entity.rb b/app/models/document_citation_geo_entity.rb
@@ -16,4 +16,8 @@ class DocumentCitationGeoEntity < ActiveRecord::Base
   attr_accessible :created_by_id, :document_citation_id, :geo_entity_id, :updated_by_id
   belongs_to :geo_entity
   belongs_to :document_citation, touch: true
+
+  after_destroy do |dc_ge|
+    dc_ge.document_citation.touch
+  end
 end
diff --git a/app/models/document_citation_taxon_concept.rb b/app/models/document_citation_taxon_concept.rb
@@ -16,4 +16,8 @@ class DocumentCitationTaxonConcept < ActiveRecord::Base
   attr_accessible :created_by_id, :document_citation_id, :taxon_concept_id, :updated_by_id
   belongs_to :taxon_concept
   belongs_to :document_citation, touch: true
+
+  after_destroy do |dc_tc|
+    dc_tc.document_citation.touch
+  end
 end
diff --git a/app/models/document_search.rb b/app/models/document_search.rb
@@ -103,19 +103,56 @@ def add_conditions_for_document
   end
 
   def add_extra_conditions
-    add_taxon_concepts_condition if @taxon_concepts_ids.present?
-    add_geo_entities_condition if @geo_entities_ids.present?
+    if @taxon_concepts_ids.present? && @geo_entities_ids.present?
+      add_citations_condition
+    elsif @taxon_concepts_ids.present?
+      add_taxon_concepts_condition
+    elsif @geo_entities_ids.present?
+      add_geo_entities_condition
+    end
     add_document_tags_condition if @document_tags_ids.present?
   end
 
+  def add_citations_condition
+    combinations = @taxon_concepts_ids.product(@geo_entities_ids)
+    condition_values = []
+    condition_string = combinations.map do |c|
+      condition_values += c
+      'taxon_concept_id = ? AND geo_entity_id = ?'
+    end.join(' OR ')
+    filter_by_citations(
+      condition_string,
+      condition_values
+    )
+  end
+
   def add_taxon_concepts_condition
-    @query = @query.where(
-      "taxon_concept_ids && ARRAY[#{@taxon_concepts_ids.join(',')}]"
+    filter_by_citations(
+      'taxon_concept_id IN (?)',
+      [@taxon_concepts_ids]
     )
   end
 
   def add_geo_entities_condition
-    @query = @query.where("geo_entity_ids && ARRAY[#{@geo_entities_ids.join(',')}]")
+    filter_by_citations(
+      'geo_entity_id IN (?)',
+      [@geo_entities_ids]
+    )
+  end
+
+  def filter_by_citations(condition_string, condition_values)
+    join_sql = ActiveRecord::Base.send(
+      :sanitize_sql_array,
+      [
+        "JOIN (
+        SELECT DISTINCT document_id
+        FROM document_citations_mview
+        WHERE #{condition_string}
+        ) t ON t.document_id = documents.id",
+        *condition_values
+      ]
+    )
+    @query = @query.joins(join_sql)
   end
 
   def add_document_tags_condition
@@ -160,16 +197,27 @@ def select_and_group_query
 
   REFRESH_INTERVAL = 5
 
-  def self.needs_refreshing?
+  def self.documents_need_refreshing?
     Document.where('updated_at > ?', REFRESH_INTERVAL.minutes.ago).limit(1).count > 0 ||
     Document.count < Document.from('api_documents_mview documents').count
   end
 
-  def self.refresh
+  def self.citations_need_refreshing?
+    DocumentCitation.where('updated_at > ?', REFRESH_INTERVAL.minutes.ago).limit(1).count > 0 ||
+    DocumentCitation.count < DocumentCitation.select('DISTINCT id').
+      from('document_citations_mview citations').count
+  end
+
+  def self.refresh_documents
     ActiveRecord::Base.connection.execute('REFRESH MATERIALIZED VIEW api_documents_mview')
     DocumentSearch.increment_cache_iterator
   end
 
+  def self.refresh_citations_and_documents
+    ActiveRecord::Base.connection.execute('REFRESH MATERIALIZED VIEW document_citations_mview')
+    refresh_documents
+  end
+
   def self.clear_cache
     RefreshDocumentsWorker.perform_async
     DownloadsCacheCleanupWorker.perform_async(:documents)
diff --git a/app/workers/refresh_documents_worker.rb b/app/workers/refresh_documents_worker.rb
@@ -3,7 +3,11 @@ class RefreshDocumentsWorker
   sidekiq_options queue: :admin, backtrace: 50, unique: :until_and_while_executing
 
   def perform
-    DocumentSearch.refresh
+    if DocumentSearch.citations_need_refreshing?
+      DocumentSearch.refresh_citations_and_documents
+    else
+      DocumentSearch.refresh_documents
+    end
     DocumentSearch.increment_cache_iterator
   end
 end
diff --git a/db/migrate/20160623105336_create_document_citations_mview.rb b/db/migrate/20160623105336_create_document_citations_mview.rb
@@ -0,0 +1,59 @@
+class CreateDocumentCitationsMview < ActiveRecord::Migration
+  def up
+    execute "DROP MATERIALIZED VIEW IF EXISTS api_documents_mview"
+    execute "DROP VIEW IF EXISTS api_documents_view"
+
+    execute "CREATE VIEW document_citations_view AS #{view_sql('20160623105336', 'document_citations_view')}"
+    execute "CREATE MATERIALIZED VIEW document_citations_mview AS SELECT * FROM document_citations_view"
+
+    add_index :document_citations_mview,
+      [:document_id, :taxon_concept_id, :geo_entity_id, :id],
+      name: :index_combinations_mview_on_document_id_tc_id_ge_id,
+      unique: true
+
+    execute "CREATE VIEW api_documents_view AS #{view_sql('20160623105336', 'api_documents_view')}"
+    execute "CREATE MATERIALIZED VIEW api_documents_mview AS SELECT * FROM api_documents_view"
+
+    add_index :api_documents_mview, [:event_id],
+      name: 'index_documents_mview_on_event_id'
+    add_index :api_documents_mview, [:date_raw],
+      name: 'index_documents_mview_on_date_raw'
+    execute <<-SQL
+      CREATE INDEX index_documents_mview_on_title_to_ts_vector
+      ON api_documents_mview
+      USING gin
+      (to_tsvector('simple'::regconfig, COALESCE(title, ''::text)));
+    SQL
+  end
+
+  def down
+    execute "DROP MATERIALIZED VIEW IF EXISTS document_citations_mview CASCADE"
+    execute "DROP VIEW IF EXISTS document_citations_view"
+
+    execute "DROP MATERIALIZED VIEW IF EXISTS api_documents_mview"
+    execute "DROP VIEW IF EXISTS api_documents_view"
+    execute "CREATE VIEW api_documents_view AS #{view_sql('20160503073026', 'api_documents_view')}"
+    execute "CREATE MATERIALIZED VIEW api_documents_mview AS SELECT * FROM api_documents_view"
+
+    add_index :api_documents_mview, [:event_id],
+      name: 'index_documents_mview_on_event_id'
+    add_index :api_documents_mview, [:date_raw],
+      name: 'index_documents_mview_on_date_raw'
+    execute <<-SQL
+      CREATE INDEX index_documents_mview_on_taxon_concepts_ids
+      ON api_documents_mview
+      USING GIN (taxon_concept_ids)
+    SQL
+    execute <<-SQL
+      CREATE INDEX index_documents_mview_on_geo_entity_ids
+      ON api_documents_mview
+      USING GIN (geo_entity_ids)
+    SQL
+    execute <<-SQL
+      CREATE INDEX index_documents_mview_on_title_to_ts_vector
+      ON api_documents_mview
+      USING gin
+      (to_tsvector('simple'::regconfig, COALESCE(title, ''::text)));
+    SQL
+  end
+end
diff --git a/db/views/api_documents_view/20160623105336.sql b/db/views/api_documents_view/20160623105336.sql
@@ -0,0 +1,52 @@
+SELECT d.id,
+  d.designation_id,
+  designations.name AS designation_name,
+  d.event_id,
+  e.name AS event_name,
+  CASE
+    WHEN e.published_at IS NOT NULL THEN to_char(e.published_at, 'DD/MM/YYYY'::text)
+    ELSE to_char(d.date::timestamp with time zone, 'DD/MM/YYYY'::text)
+  END AS date,
+  CASE
+    WHEN e.published_at IS NOT NULL THEN e.published_at
+    ELSE d.date::timestamp with time zone
+  END AS date_raw,
+  e.type AS event_type,
+  d.title,
+  upper("substring"(d.filename, length(d.filename) - "position"(reverse(d.filename), '.'::text) + 2)) AS extension,
+  d.is_public,
+  d.type AS document_type,
+  d.sort_index,
+  CASE
+    WHEN l.iso_code1 IS NULL THEN 'EN'::character varying(255)
+    ELSE l.iso_code1
+  END AS language,
+  CASE
+    WHEN d.primary_language_document_id IS NULL THEN d.id
+    ELSE d.primary_language_document_id
+  END AS primary_document_id,
+  SQUISH_NULL(pd.proposal_number) AS proposal_number,
+  po.name AS proposal_outcome,
+  rp.name AS review_phase,
+  ARRAY_AGG_NOTNULL(pd.proposal_outcome_id) || ARRAY_AGG_NOTNULL(rd.review_phase_id) || ARRAY_AGG_NOTNULL(rd.process_stage_id) AS document_tags_ids,
+  array_agg_notnull(DISTINCT dc.full_name ORDER BY dc.full_name) AS taxon_names,
+  array_agg_notnull(DISTINCT dc.name_en ORDER BY dc.name_en) AS geo_entity_names,
+  d.created_at,
+  d.updated_at,
+  d.created_by_id,
+  uc.name AS created_by,
+  d.updated_by_id,
+  uu.name AS updated_by
+FROM documents d
+  LEFT JOIN designations ON designations.id = d.designation_id
+  LEFT JOIN events e ON e.id = d.event_id
+  LEFT JOIN document_citations_mview dc ON dc.document_id = d.id
+  LEFT JOIN languages l ON d.language_id = l.id
+  LEFT JOIN proposal_details pd ON pd.document_id = d.id
+  LEFT JOIN document_tags po ON pd.proposal_outcome_id = po.id
+  LEFT JOIN review_details rd ON rd.document_id = d.id
+  LEFT JOIN document_tags rp ON rd.review_phase_id = rp.id
+  LEFT JOIN users as uc ON d.created_by_id = uc.id
+  LEFT JOIN users as uu ON d.updated_by_id = uu.id
+GROUP BY d.id, designations.name, e.name, e.published_at, e.type, d.title, l.iso_code1, pd.proposal_number, po.name, rp.name,
+uc.name, uu.name;
diff --git a/db/views/document_citations_view/20160623105336.sql b/db/views/document_citations_view/20160623105336.sql
@@ -0,0 +1,23 @@
+SELECT
+  dc.id,
+  dc.document_id,
+  dctc.taxon_concept_id,
+  tc.full_name,
+  dcge.geo_entity_id,
+  ge.name_en
+FROM document_citations dc
+LEFT JOIN document_citation_taxon_concepts dctc
+  ON dctc.document_citation_id = dc.id
+LEFT JOIN taxon_concepts tc
+  ON tc.id = dctc.taxon_concept_id
+LEFT JOIN document_citation_geo_entities dcge
+  ON dcge.document_citation_id = dc.id
+LEFT JOIN geo_entities ge
+  ON ge.id = dcge.geo_entity_id
+GROUP BY
+  dc.id,
+  document_id,
+  taxon_concept_id,
+  full_name,
+  geo_entity_id,
+  name_en;
diff --git a/lib/tasks/elibrary/search_refresh_scheduler.rake b/lib/tasks/elibrary/search_refresh_scheduler.rake
@@ -1,10 +1,15 @@
 namespace :elibrary do
   task :refresh_document_search => :environment do
-    if DocumentSearch.needs_refreshing?
+    if DocumentSearch.citations_need_refreshing?
       elapsed_time = Benchmark.realtime do
-        DocumentSearch.refresh
+        DocumentSearch.refresh_citations_and_documents
       end
-      puts "#{Time.now} Document search refreshed in #{elapsed_time}s"
+      puts "#{Time.now} Citations & documents refreshed in #{elapsed_time}s"
+    elsif DocumentSearch.documents_need_refreshing?
+      elapsed_time = Benchmark.realtime do
+        DocumentSearch.refresh_documents
+      end
+      puts "#{Time.now} Documents refreshed in #{elapsed_time}s"
     end
   end
 end
diff --git a/spec/controllers/admin/documents_controller_spec.rb b/spec/controllers/admin/documents_controller_spec.rb
@@ -21,7 +21,7 @@
     describe "GET index" do
       before(:each) do
         @document3 = create(:document, :title => 'CC no event!', date: DateTime.new(2014,01,01))
-        DocumentSearch.refresh
+        DocumentSearch.refresh_citations_and_documents
       end
 
       it "assigns @documents sorted by time of creation" do
@@ -66,7 +66,7 @@
           before(:each) do
             @document3 = create(:proposal, event: create_cites_cop(published_at: DateTime.new(2014,01,01)))
             create(:proposal_details, document_id: @document3.id, proposal_outcome_id: proposal_outcome.id)
-            DocumentSearch.refresh
+            DocumentSearch.refresh_citations_and_documents
           end
           it "retrieves documents for tag" do
             get :index, "document_tags_ids" => [proposal_outcome.id]
@@ -77,7 +77,7 @@
           before(:each) do
             @document3 = create(:review_of_significant_trade, event: create_ec_srg(published_at: DateTime.new(2014,01,01)))
             create(:review_details, document_id: @document3.id, review_phase_id: review_phase.id, process_stage_id: process_stage.id)
-            DocumentSearch.refresh
+            DocumentSearch.refresh_citations_and_documents
           end
           it "retrieves documents for review_phase tag" do
             get :index, "document_tags_ids" => [review_phase.id]
@@ -104,7 +104,7 @@
         end
         it "assigns @documents for event, sorted by title" do
           @document3 = create(:document, title: 'CC hello world', event: event2)
-          DocumentSearch.refresh
+          DocumentSearch.refresh_citations_and_documents
           get :index, events_ids: [event.id, event2.id]
           assigns(:documents).should eq([@document3, @document2, @document1])
         end
diff --git a/spec/controllers/admin/event_documents_controller_spec.rb b/spec/controllers/admin/event_documents_controller_spec.rb
@@ -8,7 +8,7 @@
     before(:each) do
       @document1 = create(:document, event: event, sort_index: 2)
       @document2 = create(:document, event: event, sort_index: 1)
-      DocumentSearch.refresh
+      DocumentSearch.refresh_citations_and_documents
     end
 
     describe "GET show_order" do
diff --git a/spec/controllers/api/documents_controller_spec.rb b/spec/controllers/api/documents_controller_spec.rb
@@ -22,7 +22,7 @@
     citation3 = create(:document_citation, document_id: @document3.id)
     create(:document_citation_taxon_concept, document_citation_id: citation3.id,
       taxon_concept_id: @taxon_concept.id)
-    DocumentSearch.refresh
+    DocumentSearch.refresh_citations_and_documents
   end
 
   context "GET index returns all documents" do
diff --git a/spec/models/document_search_spec.rb b/spec/models/document_search_spec.rb
