Merge pull request #438 from af83/6368-gtfs-import

GTFS import (first step)

8 files changed, 326 insertions, 26 deletions

diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb
index cb2266a3d..9d5737a6c 100644
--- a/app/models/chouette/company.rb
+++ b/app/models/chouette/company.rb
@@ -9,7 +9,7 @@ module Chouette
 
     has_many :lines
 
-    validates_format_of :registration_number, :with => %r{\A[0-9A-Za-z_-]+\Z}, :allow_nil => true, :allow_blank => true
+    # validates_format_of :registration_number, :with => %r{\A[0-9A-Za-z_-]+\Z}, :allow_nil => true, :allow_blank => true
     validates_presence_of :name
     validates_format_of :url, :with => %r{\Ahttps?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?\Z}, :allow_nil => true, :allow_blank => true
 
diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb
index 51851fc2e..4b5d1a68d 100644
--- a/app/models/chouette/line.rb
+++ b/app/models/chouette/line.rb
@@ -29,7 +29,7 @@ module Chouette
     # validates_presence_of :network
     # validates_presence_of :company
 
-    validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true
+    # validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true
     validates_format_of :stable_id, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true
     validates_format_of :url, :with => %r{\Ahttps?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?\Z}, :allow_nil => true, :allow_blank => true
     validates_format_of :color, :with => %r{\A[0-9a-fA-F]{6}\Z}, :allow_nil => true, :allow_blank => true
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index 9f28b7ee6..4ddc7403b 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -33,7 +33,7 @@ module Chouette
     after_update :journey_patterns_control_route_sections,
                 if: Proc.new { |stop_area| ['boarding_position', 'quay'].include? stop_area.stop_area_type }
 
-    validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_blank => true
+    # validates_format_of :registration_number, :with => %r{\A[\d\w_:\-]+\Z}, :allow_blank => true
     validates_presence_of :name
     validates_presence_of :kind
     validates_presence_of :latitude, :if => :longitude
diff --git a/app/models/concerns/application_days_support.rb b/app/models/concerns/application_days_support.rb
index 2d00b5847..6086d9580 100644
--- a/app/models/concerns/application_days_support.rb
+++ b/app/models/concerns/application_days_support.rb
@@ -10,8 +10,10 @@ module ApplicationDaysSupport
   SUNDAY    = 256
   EVERYDAY  = MONDAY | TUESDAY | WEDNESDAY | THURSDAY | FRIDAY | SATURDAY | SUNDAY
 
+  ALL_DAYS = %w(monday tuesday wednesday thursday friday saturday sunday).freeze
+
   def display_day_types
-    %w(monday tuesday wednesday thursday friday saturday sunday).select{ |d| self.send(d) }.map{ |d| self.human_attribute_name(d).first(2)}.join(', ')
+    ALL_DAYS.select{ |d| self.send(d) }.map{ |d| self.human_attribute_name(d).first(2)}.join(', ')
   end
 
   def day_by_mask(flag)
@@ -39,6 +41,10 @@ module ApplicationDaysSupport
     def self.day_by_mask(int_day_types,flag)
       int_day_types & flag == flag
     end
+
+    def self.all_days
+      ALL_DAYS
+    end
   end
 
   def valid_days
diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb
index 03cf49e60..70f448132 100644
--- a/app/models/import/gtfs.rb
+++ b/app/models/import/gtfs.rb
@@ -1,36 +1,296 @@
-require 'net/http'
 class Import::Gtfs < Import::Base
-  before_destroy :destroy_non_ready_referential
+  after_commit :launch_worker, :on => :create
 
-  after_commit :launch_java_import, on: :create
-  before_save def abort_unless_referential
-    self.status = 'aborted' unless referential
+  def launch_worker
+    GtfsImportWorker.perform_async id
   end
 
-  def launch_java_import
-    return if self.class.finished_statuses.include?(status)
-    threaded_call_boiv_iev
+  def import
+    update status: 'running', started_at: Time.now
+
+    import_without_status
+    update status: 'successful', ended_at: Time.now
+  rescue Exception => e
+    update status: 'failed', ended_at: Time.now
+    Rails.logger.error "Error in GTFS import: #{e} #{e.backtrace.join('\n')}"
+  ensure
+    notify_parent
+    referential&.update ready: true
+  end
+
+  def self.accept_file?(file)
+    Zip::File.open(file) do |zip_file|
+      zip_file.glob('agency.txt').size == 1
+    end
+  rescue Exception => e
+    Rails.logger.debug "Error in testing GTFS file: #{e}"
+    return false
+  end
+
+  def create_referential
+    self.referential ||= Referential.create!(
+      name: "GTFS Import",
+      organisation_id: workbench.organisation_id,
+      workbench_id: workbench.id,
+      metadatas: [referential_metadata]
+    )
+  end
+
+  def referential_metadata
+    registration_numbers = source.routes.map(&:id)
+    line_ids = line_referential.lines.where(registration_number: registration_numbers).pluck(:id)
+
+    start_dates, end_dates = source.calendars.map { |c| [c.start_date, c.end_date ] }.transpose
+    excluded_dates = source.calendar_dates.select { |d| d.exception_type == "2" }.map(&:date)
+
+    min_date = Date.parse (start_dates + [excluded_dates.min]).compact.min
+    max_date = Date.parse (end_dates + [excluded_dates.max]).compact.max
+
+    ReferentialMetadata.new line_ids: line_ids, periodes: [min_date..max_date]
+  end
+
+  attr_accessor :local_file
+  def local_file
+    @local_file ||= download_local_file
   end
 
-  private
+  attr_accessor :download_host
+  def download_host
+    @download_host ||= Rails.application.config.rails_host.gsub("http://","")
+  end
 
-  def destroy_non_ready_referential
-    if referential && !referential.ready
-      referential.destroy
+  def local_temp_directory
+    Rails.application.config.try(:import_temporary_directory) ||
+      Rails.root.join('tmp', 'imports')
+  end
+
+  def local_temp_file(&block)
+    Tempfile.open("chouette-import", local_temp_directory) do |file|
+      file.binmode
+      yield file
     end
   end
 
-  def threaded_call_boiv_iev
-    Thread.new(&method(:call_boiv_iev))
+  def download_path
+    Rails.application.routes.url_helpers.download_workbench_import_path(workbench, id, token: token_download)
   end
 
-  def call_boiv_iev
-    Rails.logger.error("Begin IEV call for import")
-    Net::HTTP.get(URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}"))
-    Rails.logger.error("End IEV call for import")
-  rescue Exception => e
-    logger.error "IEV server error : #{e.message}"
-    logger.error e.backtrace.inspect
+  def download_local_file
+    local_temp_file do |file|
+      begin
+        Net::HTTP.start(download_host) do |http|
+          http.request_get(download_path) do |response|
+            response.read_body do |segment|
+              file.write segment
+            end
+          end
+        end
+      ensure
+        file.close
+      end
+
+      file.path
+    end
+  end
+
+  def source
+    @source ||= ::GTFS::Source.build local_file
+  end
+
+  delegate :line_referential, :stop_area_referential, to: :workbench
+
+  def prepare_referential
+    import_agencies
+    import_stops
+    import_routes
+
+    create_referential
+    referential.switch
+  end
+
+  def import_without_status
+    prepare_referential
+
+    import_calendars
+    import_trips
+    import_stop_times
+  end
+
+  def import_agencies
+    Chouette::Company.transaction do
+      source.agencies.each do |agency|
+        company = line_referential.companies.find_or_initialize_by(registration_number: agency.id)
+        company.attributes = { name: agency.name }
+
+        save_model company
+      end
+    end
+  end
+
+  def import_stops
+    Chouette::StopArea.transaction do
+      source.stops.each do |stop|
+        stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id)
+
+        stop_area.name = stop.name
+        stop_area.area_type = stop.location_type == "1" ? "zdlp" : "zdep"
+        stop_area.parent = stop_area_referential.stop_areas.find_by!(registration_number: stop.parent_station) if stop.parent_station.present?
+        stop_area.latitude, stop_area.longitude = stop.lat, stop.lon
+        stop_area.kind = "commercial"
+
+        # TODO correct default timezone
+
+        save_model stop_area
+      end
+    end
+  end
+
+  def import_routes
+    Chouette::Line.transaction do
+      source.routes.each do |route|
+        line = line_referential.lines.find_or_initialize_by(registration_number: route.id)
+        line.name = route.long_name.presence || route.short_name
+        line.number = route.short_name
+        line.published_name = route.long_name
+
+        line.company = line_referential.companies.find_by(registration_number: route.agency_id) if route.agency_id.present?
+
+        # TODO transport mode
+
+        line.comment = route.desc
+
+        # TODO colors
+
+        line.url = route.url
+
+        save_model line
+      end
+    end
+  end
+
+  def vehicle_journey_by_trip_id
+    @vehicle_journey_by_trip_id ||= {}
+  end
+
+  def import_trips
+    source.trips.each_slice(100) do |slice|
+      slice.each do |trip|
+        Chouette::Route.transaction do
+          line = line_referential.lines.find_by registration_number: trip.route_id
+
+          route = referential.routes.build line: line
+          route.wayback = (trip.direction_id == "0" ? :outbound : :inbound)
+          # TODO better name ?
+          name = route.published_name = trip.short_name.presence || trip.headsign.presence || route.wayback.to_s.capitalize
+          route.name = name
+          save_model route
+
+          journey_pattern = route.journey_patterns.build name: name
+          save_model journey_pattern
+
+          vehicle_journey = journey_pattern.vehicle_journeys.build route: route
+          vehicle_journey.published_journey_name = trip.headsign.presence || trip.id
+          save_model vehicle_journey
+
+          time_table = referential.time_tables.find_by(id: time_tables_by_service_id[trip.service_id]) if time_tables_by_service_id[trip.service_id]
+          if time_table
+            vehicle_journey.time_tables << time_table
+          else
+            messages.create! criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id}
+          end
+
+          vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id
+        end
+      end
+    end
+  end
+
+  def import_stop_times
+    source.stop_times.group_by(&:trip_id).each_slice(50) do |slice|
+      slice.each do |trip_id, stop_times|
+        Chouette::VehicleJourneyAtStop.transaction do
+          vehicle_journey = referential.vehicle_journeys.find vehicle_journey_by_trip_id[trip_id]
+          journey_pattern = vehicle_journey.journey_pattern
+          route = journey_pattern.route
+
+          stop_times.sort_by! { |s| s.stop_sequence.to_i }
+
+          stop_times.each do |stop_time|
+            stop_area = stop_area_referential.stop_areas.find_by(registration_number: stop_time.stop_id)
+
+            stop_point = route.stop_points.build stop_area: stop_area
+            save_model stop_point
+
+            journey_pattern.stop_points << stop_point
+
+            # JourneyPattern#vjas_add creates automaticaly VehicleJourneyAtStop
+            vehicle_journey_at_stop = journey_pattern.vehicle_journey_at_stops.find_by(stop_point_id: stop_point.id)
+
+            departure_time = GTFS::Time.parse(stop_time.departure_time)
+            arrival_time = GTFS::Time.parse(stop_time.arrival_time)
+
+            vehicle_journey_at_stop.departure_time = departure_time.time
+            vehicle_journey_at_stop.arrival_time = arrival_time.time
+            vehicle_journey_at_stop.departure_day_offset = departure_time.day_offset
+            vehicle_journey_at_stop.arrival_day_offset = arrival_time.day_offset
+
+            # TODO offset
+
+            save_model vehicle_journey_at_stop
+          end
+        end
+      end
+    end
+  end
+
+  def time_tables_by_service_id
+    @time_tables_by_service_id ||= {}
+  end
+
+  def import_calendars
+    source.calendars.each_slice(500) do |slice|
+      Chouette::TimeTable.transaction do
+        slice.each do |calendar|
+          time_table = referential.time_tables.build comment: "Calendar #{calendar.service_id}"
+          Chouette::TimeTable.all_days.each do |day|
+            time_table.send("#{day}=", calendar.send(day))
+          end
+          time_table.periods.build period_start: calendar.start_date, period_end: calendar.end_date
+
+          save_model time_table
+
+          time_tables_by_service_id[calendar.service_id] = time_table.id
+        end
+      end
+    end
+  end
+
+  def import_calendar_dates
+    source.calendar_dates.each_slice(500) do |slice|
+      Chouette::TimeTable.transaction do
+        slice.each do |calendar_date|
+          time_table = referential.time_tables.find time_tables_by_service_id[calendar_date.service_id]
+          date = time_table.dates.build date: Date.parse(calendar_date.date), in_out: calendar_date.exception_type == "1"
+
+          save_model date
+        end
+      end
+    end
+  end
+
+  def save_model(model)
+    unless model.save
+      Rails.logger.info "Can't save #{model.class.name} : #{model.errors.inspect}"
+      raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name} : #{model.errors.inspect}")
+    end
+    Rails.logger.debug "Created #{model.inspect}"
+  end
+
+  def notify_parent
+    return unless parent.present?
+    return if notified_parent_at
+    parent.child_change
+    update_column :notified_parent_at, Time.now
   end
 
 end
diff --git a/app/models/import/workbench.rb b/app/models/import/workbench.rb
index f6e15cb89..124b9b0d8 100644
--- a/app/models/import/workbench.rb
+++ b/app/models/import/workbench.rb
@@ -2,6 +2,25 @@ class Import::Workbench < Import::Base
   after_commit :launch_worker, :on => :create
 
   def launch_worker
-    WorkbenchImportWorker.perform_async(id)
+    unless Import::Gtfs.accept_file?(file.path)
+      WorkbenchImportWorker.perform_async(id)
+    else
+      import_gtfs
+    end
+  end
+
+  def import_gtfs
+    update_column :status, 'running'
+    update_column :started_at, Time.now
+
+    Import::Gtfs.create! parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service"
+
+    update_column :status, 'successful'
+    update_column :ended_at, Time.now
+  rescue Exception => e
+    Rails.logger.error "Error while processing GTFS file: #{e}"
+
+    update_column :status, 'failed'
+    update_column :ended_at, Time.now
   end
 end
diff --git a/app/models/referential.rb b/app/models/referential.rb
index 3304108d0..1794126a2 100644
--- a/app/models/referential.rb
+++ b/app/models/referential.rb
@@ -168,6 +168,10 @@ class Referential < ApplicationModel
     Chouette::TimeTable.all
   end
 
+  def time_table_dates
+    Chouette::TimeTableDate.all
+  end
+
   def timebands
     Chouette::Timeband.all
   end
@@ -184,6 +188,10 @@ class Referential < ApplicationModel
     Chouette::VehicleJourneyFrequency.all
   end
 
+  def vehicle_journey_at_stops
+    Chouette::VehicleJourneyAtStop.all
+  end
+
   def routing_constraint_zones
     Chouette::RoutingConstraintZone.all
   end
diff --git a/app/workers/gtfs_import_worker.rb b/app/workers/gtfs_import_worker.rb
new file mode 100644
index 000000000..02f5053b0
--- /dev/null
+++ b/app/workers/gtfs_import_worker.rb
@@ -0,0 +1,7 @@
+class GtfsImportWorker
+  include Sidekiq::Worker
+
+  def perform(import_id)
+    Import::Gtfs.find(import_id).import
+  end
+end