Merge pull request #1275 from transitland/feed-rt-models

GTFS-RT Feed models

app/controllers/api/v1/feeds_controller.rb

@@ -41,7 +41,6 @@ def feed_version_update_statistics
   def index_query
     super
     @collection = AllowFiltering.by_attribute_array(@collection, params, :name)
-    @collection = AllowFiltering.by_attribute_array(@collection, params, :url, case_sensitive: true)
     @collection = AllowFiltering.by_attribute_since(@collection, params, :last_imported_since, :last_imported_at)
     if params[:latest_fetch_exception].present?
       @collection = @collection.where_latest_fetch_exception(AllowFiltering.to_boolean(params[:latest_fetch_exception]))
@@ -61,6 +60,10 @@ def index_query
     if params[:latest_feed_version_import_status].present?
       @collection = @collection.where_latest_feed_version_import_status(AllowFiltering.to_boolean(params[:latest_feed_version_import_status]))
     end
+    if params[:url].present?
+      urls = params[:url]
+      @collection = @collection.where("#{Feed.table_name}.urls -> 'static_current' IN (?)", urls)
+    end
   end
 
   def index_includes

app/models/change_payload.rb

@@ -23,6 +23,7 @@ class ChangePayload < ActiveRecord::Base
 
   ENTITY_TYPES = {
     feed: Feed,
+    gtfs_realtime_feed: GTFSRealtimeFeed,
     stop: Stop,
     stop_platform: StopPlatform,
     stop_egress: StopEgress,
@@ -46,6 +47,9 @@ class ChangePayload < ActiveRecord::Base
   JSON::Validator.register_format_validator('feed-onestop-id', -> (onestop_id) {
     onestop_id_format_proc.call(onestop_id, 'feed')
   })
+  JSON::Validator.register_format_validator('gtfs-realtime-feed-onestop-id', -> (onestop_id) {
+    onestop_id_format_proc.call(onestop_id, 'gtfs_realtime_feed')
+  })
   JSON::Validator.register_format_validator('route-onestop-id', -> (onestop_id) {
     onestop_id_format_proc.call(onestop_id, 'route')
   })

app/models/feed.rb

@@ -23,33 +23,26 @@
 #  active_feed_version_id             :integer
 #  edited_attributes                  :string           default([]), is an Array
 #  name                               :string
+#  type                               :string
+#  authorization                      :hstore
+#  urls                               :hstore
 #
 # Indexes
 #
 #  index_current_feeds_on_active_feed_version_id              (active_feed_version_id)
+#  index_current_feeds_on_authorization                       (authorization)
 #  index_current_feeds_on_created_or_updated_in_changeset_id  (created_or_updated_in_changeset_id)
 #  index_current_feeds_on_geometry                            (geometry) USING gist
 #  index_current_feeds_on_onestop_id                          (onestop_id) UNIQUE
+#  index_current_feeds_on_urls                                (urls)
 #
 
 class BaseFeed < ActiveRecord::Base
   self.abstract_class = true
-
-  extend Enumerize
-  enumerize :feed_format, in: [:gtfs]
-  enumerize :license_use_without_attribution, in: [:yes, :no, :unknown]
-  enumerize :license_create_derived_product, in: [:yes, :no, :unknown]
-  enumerize :license_redistribute, in: [:yes, :no, :unknown]
-
-  validates :url, presence: true
-  validates :url, format: { with: URI.regexp }, if: Proc.new { |feed| feed.url.present? }
-  validates :license_url, format: { with: URI.regexp }, if: Proc.new { |feed| feed.license_url.present? }
-
-  attr_accessor :includes_operators, :does_not_include_operators
 end
 
 class Feed < BaseFeed
-  self.table_name_prefix = 'current_'
+  self.table_name = 'current_feeds'
 
   include HasAOnestopId
   include HasTags
@@ -74,6 +67,16 @@ def csv_row_values
     ]
   end
 
+  extend Enumerize
+  enumerize :license_use_without_attribution, in: [:yes, :no, :unknown]
+  enumerize :license_create_derived_product, in: [:yes, :no, :unknown]
+  enumerize :license_redistribute, in: [:yes, :no, :unknown]
+
+  attr_accessor :includes_operators, :does_not_include_operators
+
+  validates :license_url, format: { with: URI.regexp }, if: Proc.new { |feed| feed.license_url.present? }
+  validate :validate_urls
+
   has_many :feed_versions, -> { order 'earliest_calendar_date' }, dependent: :destroy, as: :feed
   has_many :feed_version_imports, -> { order 'created_at DESC' }, through: :feed_versions
   has_many :gtfs_imports, -> { order 'created_at DESC' }, through: :feed_versions
@@ -166,6 +169,34 @@ def csv_row_values
     # WHERE fvi2.id IS NULL GROUP BY (fv.feed_id)
   }
 
+
+  # "static_current": { "type": "string" },
+  # "static_historic": { "type": "string" },
+  # "static_planned": { "type": "string" },
+  # "static_hypothetical": { "type": "string" },
+  # "realtime_vehicle_positions": { "type": "string" },
+  # "realtime_trip_updates": { "type": "string" },
+  # "realtime_alerts": { "type": "string" }
+  def valid_url_types
+    ['static_current', 'static_historic', 'static_planned', 'static_hypothetical']
+  end
+
+  def validate_urls
+    vt = self.valid_url_types
+    self.urls ||= {}    
+    self.urls.each do |k,v|
+      errors.add(:urls, "invalid url type: #{k}") unless vt.include?(k)
+      errors.add(:urls, "invalid url: #{v}") unless v =~ URI.regexp
+    end
+    if self.urls.length == 0 
+      errors.add(:urls, "at least one url is required")
+    end
+  end
+
+  def feed_format
+    'gtfs'
+  end
+
   def self.feed_version_update_statistics(feed)
     fvs = feed.feed_versions.to_a
     fvs_stats = fvs.select { |a| a.url && a.fetched_at && a.earliest_calendar_date && a.latest_calendar_date }.sort_by { |a| a.fetched_at }
@@ -311,6 +342,19 @@ def import_policy
     value
   end
 
+  def url
+    return {} if self.urls.nil?
+    return self.urls["static_current"]
+  end
+
+  def url=(value)
+    if self.urls.nil?
+      self.urls = {}
+    end
+    return if value.nil? # required for changesets to work, assign_attributes is unordered
+    self.urls["static_current"] = value
+  end
+
   def ssl_verify
     if tags['ssl_verify'] == 'false'
       return false
@@ -324,17 +368,48 @@ def ssl_verify
   def set_default_values
     if self.new_record?
       self.tags ||= {}
-      self.feed_format ||= 'gtfs'
       self.license_use_without_attribution ||= 'unknown'
       self.license_create_derived_product ||= 'unknown'
       self.license_redistribute ||= 'unknown'
     end
   end
 end
 
-class OldFeed < BaseFeed
-  include OldTrackedByChangeset
+class GTFSStaticFeed < Feed
+end
+
+class GTFSRealtimeFeed < Feed
+  current_tracked_by_changeset({
+    kind_of_model_tracked: :onestop_entity,
+    virtual_attributes: [
+      :includes_operators,
+      :does_not_include_operators
+    ],
+    protected_attributes: []
+  })
+
+  # "realtime_vehicle_positions": { "type": "string" },
+  # "realtime_trip_updates": { "type": "string" },
+  # "realtime_alerts": { "type": "string" }
+  def valid_url_types
+    ['realtime_vehicle_positions', 'realtime_trip_updates', 'realtime_alerts']
+  end
 
+  def url=(value)
+  end
+
+  def feed_format
+    'gtfs-rt'
+  end
+
+end
+
+class OldFeed < ActiveRecord::Base
+  self.table_name = 'old_feeds'
+  include OldTrackedByChangeset
   has_many :old_operators_in_feed, as: :feed
   has_many :operators, through: :old_operators_in_feed, source_type: 'Feed'
 end
+
+class OldGTFSRealtimeFeed < OldFeed
+end

app/models/json_schemas/changeset.json

@@ -18,6 +18,14 @@
               }
             }
           },
+          {
+            "required": ["gtfsRealtimeFeed"],
+            "properties": {
+              "feed": {
+                "$ref": "./feed.json"
+              }
+            }
+          },
           {
             "required": ["stop"],
             "properties": {

app/serializers/feed_serializer.rb

@@ -23,13 +23,18 @@
 #  active_feed_version_id             :integer
 #  edited_attributes                  :string           default([]), is an Array
 #  name                               :string
+#  type                               :string
+#  authorization                      :hstore
+#  urls                               :hstore
 #
 # Indexes
 #
 #  index_current_feeds_on_active_feed_version_id              (active_feed_version_id)
+#  index_current_feeds_on_authorization                       (authorization)
 #  index_current_feeds_on_created_or_updated_in_changeset_id  (created_or_updated_in_changeset_id)
 #  index_current_feeds_on_geometry                            (geometry) USING gist
 #  index_current_feeds_on_onestop_id                          (onestop_id) UNIQUE
+#  index_current_feeds_on_urls                                (urls)
 #
 
 class FeedSerializer < CurrentEntitySerializer
@@ -50,7 +55,10 @@ class FeedSerializer < CurrentEntitySerializer
              :feed_versions,
              :active_feed_version,
              :import_level_of_active_feed_version,
-             :changesets_imported_from_this_feed
+             :changesets_imported_from_this_feed,
+             :type,
+             :urls,
+             :authorization
 
   has_many :operators_in_feed
 

app/services/feed_fetcher_service.rb

@@ -9,7 +9,7 @@ def self.fetch_these_feeds_async(feeds)
   end
 
   def self.fetch_all_feeds_async
-    feeds = Feed.where('')
+    feeds = Feed.where(type: nil)
     async_enqueue_and_return_workers(feeds)
   end
 
@@ -27,6 +27,8 @@ def self.fetch_some_ready_feeds_async(since: REFETCH_WAIT.ago, split: SPLIT_REFE
   end
 
   def self.fetch_and_return_feed_version(feed)
+    # Only fetch static gtfs
+    return if feed.feed_format != 'gtfs'
     # Check fetch policy
     fetch_policy = feed.status
     if fetch_policy != 'active'

app/services/onestop_id.rb

@@ -93,6 +93,32 @@ class FeedOnestopId < OnestopIdBase
     MODEL = Feed
   end
 
+  class GTFSRealtimeFeedOnestopId < OnestopIdBase
+    NUM_COMPONENTS = 2
+    PREFIX = :f
+    MODEL = GTFSRealtimeFeed
+    def initialize(string: nil, name: nil)
+      if string && string.length > 0
+        name = string.split(COMPONENT_SEPARATOR)[-1]
+      end
+      @name = name
+    end
+
+    def to_s
+      [
+        self.class::PREFIX,
+        @name
+      ].join(COMPONENT_SEPARATOR)[0...self.class::MAX_LENGTH]
+    end
+
+    def validate
+      errors = []
+      errors << 'invalid name' unless @name.present?
+      errors << 'invalid name' unless validate_name(@name)
+      return (errors.size == 0), errors
+    end
+  end
+
   class StopOnestopId < OnestopIdBase
     PREFIX = :s
     MODEL = Stop

circle.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   rails-test-suite:
     docker:
-      - image: ruby:2.3.1
+      - image: ruby:2.3.8
         environment:
           RAILS_ENV: test
           DATABASE_URL: postgis://root@localhost/transitland_datastore_test

db/migrate/20190411214421_add_type_to_feeds.rb

@@ -0,0 +1,12 @@
+class AddTypeToFeeds < ActiveRecord::Migration
+  def change
+    add_column :current_feeds, :type, :string, index: true
+    add_column :old_feeds, :type, :string, index: true
+    add_column :current_feeds, :authorization, :hstore
+    add_column :old_feeds, :authorization, :hstore
+    add_column :current_feeds, :urls, :hstore
+    add_column :old_feeds, :urls, :hstore
+    add_index :current_feeds, :urls
+    add_index :current_feeds, :authorization
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180113021001) do
+ActiveRecord::Schema.define(version: 20190411214421) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -63,12 +63,17 @@
     t.integer   "active_feed_version_id"
     t.string    "edited_attributes",                                                                              default: [], array: true
     t.string    "name"
+    t.string    "type"
+    t.hstore    "authorization"
+    t.hstore    "urls"
   end
 
   add_index "current_feeds", ["active_feed_version_id"], name: "index_current_feeds_on_active_feed_version_id", using: :btree
+  add_index "current_feeds", ["authorization"], name: "index_current_feeds_on_authorization", using: :btree
   add_index "current_feeds", ["created_or_updated_in_changeset_id"], name: "index_current_feeds_on_created_or_updated_in_changeset_id", using: :btree
   add_index "current_feeds", ["geometry"], name: "index_current_feeds_on_geometry", using: :gist
   add_index "current_feeds", ["onestop_id"], name: "index_current_feeds_on_onestop_id", unique: true, using: :btree
+  add_index "current_feeds", ["urls"], name: "index_current_feeds_on_urls", using: :btree
 
   create_table "current_operators", force: :cascade do |t|
     t.string    "name"
@@ -694,6 +699,9 @@
     t.string    "edited_attributes",                                                                              default: [], array: true
     t.string    "action"
     t.string    "name"
+    t.string    "type"
+    t.hstore    "authorization"
+    t.hstore    "urls"
   end
 
   add_index "old_feeds", ["active_feed_version_id"], name: "index_old_feeds_on_active_feed_version_id", using: :btree