Initial import for agencies, stops, routes, trips and stop_times. Refs #6368

7 files changed, 344 insertions, 23 deletions

diff --git a/Gemfile b/Gemfile
index 6fcaf8896..5d5df4b06 100644
--- a/Gemfile
+++ b/Gemfile
@@ -146,6 +146,8 @@ gem 'puma', '~> 3.10.0'
 gem 'newrelic_rpm'
 gem 'letter_opener'
 
+gem 'gtfs'
+
 group :development do
   gem 'capistrano', '2.13.5'
   gem 'capistrano-ext'
diff --git a/Gemfile.lock b/Gemfile.lock
index 4e3c76690..7fd58c713 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -258,6 +258,10 @@ GEM
     google-analytics-rails (1.1.0)
     gretel (3.0.9)
       rails (>= 3.1.0)
+    gtfs (0.2.5)
+      multi_json
+      rake
+      rubyzip (~> 1.1)
     has_scope (0.7.0)
       actionpack (>= 4.1, < 5.1)
       activesupport (>= 4.1, < 5.1)
@@ -321,7 +325,7 @@ GEM
     minitest (5.11.3)
     money (6.10.1)
       i18n (>= 0.6.4, < 1.0)
-    multi_json (1.12.1)
+    multi_json (1.13.1)
     multi_test (0.1.2)
     multi_xml (0.6.0)
     multipart-post (2.0.0)
@@ -424,7 +428,7 @@ GEM
       thor (>= 0.18.1, < 2.0)
     rainbow (2.2.2)
       rake
-    rake (12.3.0)
+    rake (12.3.1)
     ransack (1.8.3)
       actionpack (>= 3.0)
       activerecord (>= 3.0)
@@ -637,6 +641,7 @@ DEPENDENCIES
   georuby-ext (= 0.0.5)
   google-analytics-rails
   gretel
+  gtfs
   has_array_of!
   htmlbeautifier
   i18n-js
diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb
index 03cf49e60..5b332315d 100644
--- a/app/models/import/gtfs.rb
+++ b/app/models/import/gtfs.rb
@@ -1,36 +1,154 @@
-require 'net/http'
 class Import::Gtfs < Import::Base
-  before_destroy :destroy_non_ready_referential
+  after_commit :launch_worker, :on => :create
 
-  after_commit :launch_java_import, on: :create
-  before_save def abort_unless_referential
-    self.status = 'aborted' unless referential
+  def launch_worker
+    GtfsImportWorker.perform_async id
   end
 
-  def launch_java_import
-    return if self.class.finished_statuses.include?(status)
-    threaded_call_boiv_iev
+  def import
+    workbench_import.update(status: 'running', started_at: Time.now)
+
+    import_without_status
+    workbench_import.update(status: 'successful', ended_at: Time.now)
+  rescue Exception => e
+    workbench_import.update(status: 'failed', ended_at: Time.now)
+    raise e
+  end
+
+  attr_accessor :local_file
+
+  # TODO download the imported file
+  # def local_file
+  #   @local_file
+  # end
+
+  # TODO create referential with metadatas
+  # def referential
+  # ...
+  # end
+
+  def source
+    @source ||= ::GTFS::Source.build local_file
+  end
+
+  delegate :line_referential, :stop_area_referential, to: :referential
+
+  def import_without_status
+    referential.switch
+
+    import_agencies
+    import_stops
+    import_routes
+    import_trips
+    import_stop_times
   end
 
-  private
+  def import_agencies
+    source.agencies.each do |agency|
+      company = line_referential.companies.find_or_initialize_by(registration_number: agency.id)
+      company.attributes = { name: agency.name }
 
-  def destroy_non_ready_referential
-    if referential && !referential.ready
-      referential.destroy
+      save company
     end
   end
 
-  def threaded_call_boiv_iev
-    Thread.new(&method(:call_boiv_iev))
+  def import_stops
+    source.stops.each do |stop|
+      stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id)
+
+      stop_area.name = stop.name
+      stop_area.area_type = stop.location_type == "1" ? "zdlp" : "zdep"
+      stop_area.parent = stop_referential.stop_areas.find_by!(registration_number: stop.parent_station) if stop.parent_station.present?
+      stop_area.latitude, stop_area.longitude = stop.lat, stop.lon
+      stop_area.kind = "commercial"
+
+      # TODO correct default timezone
+
+      save stop_area
+    end
   end
 
-  def call_boiv_iev
-    Rails.logger.error("Begin IEV call for import")
-    Net::HTTP.get(URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}"))
-    Rails.logger.error("End IEV call for import")
-  rescue Exception => e
-    logger.error "IEV server error : #{e.message}"
-    logger.error e.backtrace.inspect
+  def import_routes
+    source.routes.each do |route|
+      line = line_referential.lines.find_or_initialize_by(registration_number: route.id)
+      line.name = route.long_name.presence || route.short_name
+      line.number = route.short_name
+      line.published_name = route.long_name
+
+      line.company = line_referential.companies.find_by(registration_number: route.agency_id) if route.agency_id.present?
+
+      # TODO transport mode
+
+      line.comment = route.desc
+
+      # TODO colors
+
+      line.url = route.url
+
+      save line
+    end
+  end
+
+  def vehicle_journey_by_trip_id
+    @vehicle_journey_by_trip_id ||= {}
+  end
+
+  def import_trips
+    source.trips.each do |trip|
+      line = line_referential.lines.find_by registration_number: trip.route_id
+
+      route = referential.routes.build line: line
+      route.wayback = (trip.direction_id == "0" ? :outbound : :inbound)
+      # TODO better name ?
+      name = route.published_name = trip.short_name.presence || trip.headsign.presence || route.wayback.to_s.capitalize
+      route.name = name
+      save route
+
+      journey_pattern = route.journey_patterns.build name: name
+      save journey_pattern
+
+      vehicle_journey = journey_pattern.vehicle_journeys.build route: route
+      vehicle_journey.published_journey_name = trip.headsign.presence || trip.id
+      save vehicle_journey
+
+      vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id
+    end
+  end
+
+  def import_stop_times
+    source.stop_times.group_by(&:trip_id).each do |trip_id, stop_times|
+      vehicle_journey = referential.vehicle_journeys.find vehicle_journey_by_trip_id[trip_id]
+      journey_pattern = vehicle_journey.journey_pattern
+      route = journey_pattern.route
+
+      stop_times.sort_by!(&:stop_sequence)
+
+      stop_times.each do |stop_time|
+        stop_area = stop_area_referential.stop_areas.find_by(registration_number: stop_time.stop_id)
+
+        stop_point = route.stop_points.build stop_area: stop_area
+        save stop_point
+
+        journey_pattern.stop_points << stop_point
+
+        # JourneyPattern#vjas_add creates automaticaly VehicleJourneyAtStop
+        vehicle_journey_at_stop = journey_pattern.vehicle_journey_at_stops.find_by(stop_point_id: stop_point.id)
+        vehicle_journey_at_stop.departure_time = stop_time.departure_time
+        vehicle_journey_at_stop.arrival_time = stop_time.arrival_time
+
+        # TODO offset
+
+        save vehicle_journey_at_stop
+      end
+    end
+  end
+
+  def save(model)
+    unless model.save
+      Rails.logger.info "Can't save #{model.class.name} : #{model.errors.inspect}"
+      raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name}")
+    end
+    Rails.logger.debug "Created #{model.inspect}"
   end
 
 end
diff --git a/app/models/referential.rb b/app/models/referential.rb
index 91a88d02d..65af58873 100644
--- a/app/models/referential.rb
+++ b/app/models/referential.rb
@@ -184,6 +184,10 @@ class Referential < ActiveRecord::Base
     Chouette::VehicleJourneyFrequency.all
   end
 
+  def vehicle_journey_at_stops
+    Chouette::VehicleJourneyAtStop.all
+  end
+
   def routing_constraint_zones
     Chouette::RoutingConstraintZone.all
   end
diff --git a/app/workers/gtfs_import_worker.rb b/app/workers/gtfs_import_worker.rb
new file mode 100644
index 000000000..093869475
--- /dev/null
+++ b/app/workers/gtfs_import_worker.rb
@@ -0,0 +1,7 @@
+class WorkbenchImportWorker
+  include Sidekiq::Worker
+
+  def perform(import_id)
+    Import::Gtfs.find(import_id).import
+  end
+end
diff --git a/spec/fixtures/google-sample-feed.zip b/spec/fixtures/google-sample-feed.zip
new file mode 100644
index 000000000..79819e21a
--- /dev/null
+++ b/spec/fixtures/google-sample-feed.zip
diff --git a/spec/models/import/gtfs_spec.rb b/spec/models/import/gtfs_spec.rb
new file mode 100644
index 000000000..bdfc565d3
--- /dev/null
+++ b/spec/models/import/gtfs_spec.rb
@@ -0,0 +1,185 @@
+require "rails_helper"
+
+RSpec.describe Import::Gtfs do
+
+  let(:referential) do
+    create :referential do |referential|
+      referential.line_referential.objectid_format = "netex"
+      referential.stop_area_referential.objectid_format = "netex"
+    end
+  end
+
+  def create_import(file)
+    Import::Gtfs.new referential: referential, local_file: fixtures_path(file)
+  end
+
+  describe "#import_agencies" do
+    let(:import) { create_import "google-sample-feed.zip" }
+    it "should create a company for each agency" do
+      import.import_agencies
+
+      expect(referential.line_referential.companies.pluck(:registration_number, :name)).to eq([["DTA","Demo Transit Authority"]])
+    end
+  end
+
+  describe "#import_stops" do
+    let(:import) { create_import "google-sample-feed.zip" }
+    it "should create a company for each agency" do
+      import.import_stops
+
+      defined_attributes = [
+        :registration_number, :name, :parent_id, :latitude, :longitude
+      ]
+      expected_attributes = [
+        ["AMV", "Amargosa Valley (Demo)", nil, 36.641496, -116.40094],
+        ["EMSI", "E Main St / S Irving St (Demo)", nil, 36.905697, -116.76218],
+        ["DADAN", "Doing Ave / D Ave N (Demo)", nil, 36.909489, -116.768242],
+        ["NANAA", "North Ave / N A Ave (Demo)", nil, 36.914944, -116.761472],
+        ["NADAV", "North Ave / D Ave N (Demo)", nil, 36.914893, -116.76821],
+        ["STAGECOACH", "Stagecoach Hotel & Casino (Demo)", nil, 36.915682, -116.751677],
+        ["BULLFROG", "Bullfrog (Demo)", nil, 36.88108, -116.81797],
+        ["BEATTY_AIRPORT", "Nye County Airport (Demo)", nil, 36.868446, -116.784582],
+        ["FUR_CREEK_RES", "Furnace Creek Resort (Demo)", nil, 36.425288, -117.133162]
+      ]
+
+      expect(referential.stop_area_referential.stop_areas.pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+  end
+
+  describe "#import_routes" do
+    let(:import) { create_import "google-sample-feed.zip" }
+    it "should create a line for each route" do
+      import.import_routes
+
+      defined_attributes = [
+        :registration_number, :name, :number, :published_name,
+        "companies.registration_number",
+        :comment, :url
+      ]
+      expected_attributes = [
+        ["AAMV", "Airport - Amargosa Valley", "50", "Airport - Amargosa Valley", nil, nil, nil],
+        ["CITY", "City", "40", "City", nil, nil, nil],
+        ["STBA", "Stagecoach - Airport Shuttle", "30", "Stagecoach - Airport Shuttle", nil, nil, nil],
+        ["BFC", "Bullfrog - Furnace Creek Resort", "20", "Bullfrog - Furnace Creek Resort", nil, nil, nil],
+        ["AB", "Airport - Bullfrog", "10", "Airport - Bullfrog", nil, nil, nil]
+      ]
+
+      expect(referential.line_referential.lines.includes(:company).pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+  end
+
+  describe "#import_trips" do
+    let(:import) { create_import "google-sample-feed.zip" }
+    it "should create a Route for each trip" do
+      referential.switch
+
+      import.import_routes
+      import.import_trips
+
+
+      defined_attributes = [
+        "lines.registration_number", :wayback, :name, :published_name
+      ]
+      expected_attributes = [
+        ["AB", "outbound", "to Bullfrog", "to Bullfrog"],
+        ["AB", "inbound", "to Airport", "to Airport"],
+        ["STBA", "inbound", "Shuttle", "Shuttle"],
+        ["CITY", "outbound", "Outbound", "Outbound"],
+        ["CITY", "inbound", "Inbound", "Inbound"],
+        ["BFC", "outbound", "to Furnace Creek Resort", "to Furnace Creek Resort"],
+        ["BFC", "inbound", "to Bullfrog", "to Bullfrog"],
+        ["AAMV", "outbound", "to Amargosa Valley", "to Amargosa Valley"],
+        ["AAMV", "inbound", "to Airport", "to Airport"],
+        ["AAMV", "outbound", "to Amargosa Valley", "to Amargosa Valley"],
+        ["AAMV", "inbound", "to Airport", "to Airport"]
+      ]
+
+      expect(referential.routes.includes(:line).pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+
+    it "should create a JourneyPattern for each trip" do
+      referential.switch
+
+      import.import_routes
+      import.import_trips
+
+      defined_attributes = [
+        :name
+      ]
+      expected_attributes = [
+        "to Bullfrog", "to Airport", "Shuttle", "Outbound", "Inbound", "to Furnace Creek Resort", "to Bullfrog", "to Amargosa Valley", "to Airport", "to Amargosa Valley", "to Airport"
+      ]
+
+      expect(referential.journey_patterns.pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+
+    it "should create a VehicleJourney for each trip" do
+      referential.switch
+
+      import.import_routes
+      import.import_trips
+
+      defined_attributes = [
+        :published_journey_name
+      ]
+      expected_attributes = [
+        "to Bullfrog", "to Airport", "Shuttle", "CITY1", "CITY2", "to Furnace Creek Resort", "to Bullfrog", "to Amargosa Valley", "to Airport", "to Amargosa Valley", "to Airport"
+      ]
+
+      expect(referential.vehicle_journeys.pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+  end
+
+  describe "#import_stop_times" do
+    let(:import) { create_import "google-sample-feed.zip" }
+
+    it "should create a VehicleJourneyAtStop for each stop_time" do
+      referential.switch
+
+      import.import_stops
+      import.import_routes
+      import.import_trips
+      import.import_stop_times
+
+      def t(value)
+        Time.parse(value)
+      end
+
+      defined_attributes = [
+        "stop_areas.registration_number", :position, :departure_time, :arrival_time,
+      ]
+      expected_attributes = [
+        ["STAGECOACH", 0, t("2000-01-01 06:00:00 UTC"), t("2000-01-01 06:00:00 UTC")],
+        ["BEATTY_AIRPORT", 1, t("2000-01-01 06:20:00 UTC"), t("2000-01-01 06:20:00 UTC")],
+        ["STAGECOACH", 0, t("2000-01-01 06:00:00 UTC"), t("2000-01-01 06:00:00 UTC")],
+        ["NANAA", 1, t("2000-01-01 06:07:00 UTC"), t("2000-01-01 06:05:00 UTC")],
+        ["NADAV", 2, t("2000-01-01 06:14:00 UTC"), t("2000-01-01 06:12:00 UTC")],
+        ["DADAN", 3, t("2000-01-01 06:21:00 UTC"), t("2000-01-01 06:19:00 UTC")],
+        ["EMSI", 4, t("2000-01-01 06:28:00 UTC"), t("2000-01-01 06:26:00 UTC")],
+        ["EMSI", 0, t("2000-01-01 06:30:00 UTC"), t("2000-01-01 06:28:00 UTC")],
+        ["DADAN", 1, t("2000-01-01 06:37:00 UTC"), t("2000-01-01 06:35:00 UTC")],
+        ["NADAV", 2, t("2000-01-01 06:44:00 UTC"), t("2000-01-01 06:42:00 UTC")],
+        ["NANAA", 3, t("2000-01-01 06:51:00 UTC"), t("2000-01-01 06:49:00 UTC")],
+        ["STAGECOACH", 4, t("2000-01-01 06:58:00 UTC"), t("2000-01-01 06:56:00 UTC")],
+        ["BEATTY_AIRPORT", 0, t("2000-01-01 08:00:00 UTC"), t("2000-01-01 08:00:00 UTC")],
+        ["BULLFROG", 1, t("2000-01-01 08:15:00 UTC"), t("2000-01-01 08:10:00 UTC")],
+        ["BULLFROG", 0, t("2000-01-01 12:05:00 UTC"), t("2000-01-01 12:05:00 UTC")],
+        ["BEATTY_AIRPORT", 1, t("2000-01-01 12:15:00 UTC"), t("2000-01-01 12:15:00 UTC")],
+        ["BULLFROG", 0, t("2000-01-01 08:20:00 UTC"), t("2000-01-01 08:20:00 UTC")],
+        ["FUR_CREEK_RES", 1, t("2000-01-01 09:20:00 UTC"), t("2000-01-01 09:20:00 UTC")],
+        ["FUR_CREEK_RES", 0, t("2000-01-01 11:00:00 UTC"), t("2000-01-01 11:00:00 UTC")],
+        ["BULLFROG", 1, t("2000-01-01 12:00:00 UTC"), t("2000-01-01 12:00:00 UTC")],
+        ["BEATTY_AIRPORT", 0, t("2000-01-01 08:00:00 UTC"), t("2000-01-01 08:00:00 UTC")],
+        ["AMV", 1, t("2000-01-01 09:00:00 UTC"), t("2000-01-01 09:00:00 UTC")],
+        ["AMV", 0, t("2000-01-01 10:00:00 UTC"), t("2000-01-01 10:00:00 UTC")],
+        ["BEATTY_AIRPORT", 1, t("2000-01-01 11:00:00 UTC"), t("2000-01-01 11:00:00 UTC")],
+        ["BEATTY_AIRPORT", 0, t("2000-01-01 13:00:00 UTC"), t("2000-01-01 13:00:00 UTC")],
+        ["AMV", 1, t("2000-01-01 14:00:00 UTC"), t("2000-01-01 14:00:00 UTC")],
+        ["AMV", 0, t("2000-01-01 15:00:00 UTC"), t("2000-01-01 15:00:00 UTC")],
+        ["BEATTY_AIRPORT", 1, t("2000-01-01 16:00:00 UTC"), t("2000-01-01 16:00:00 UTC")]
+      ]
+      expect(referential.vehicle_journey_at_stops.includes(stop_point: :stop_area).pluck(*defined_attributes)).to eq(expected_attributes)
+    end
+  end
+
+end