Create Referential in Import::Gtfs. Test file with Import::Gtfs.accept_file?. Refs #6368

3 files changed, 91 insertions, 41 deletions

diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb
index ea88ca9fe..fb81423bb 100644
--- a/app/models/import/gtfs.rb
+++ b/app/models/import/gtfs.rb
@@ -6,13 +6,47 @@ class Import::Gtfs < Import::Base
   end
 
   def import
-    workbench_import.update(status: 'running', started_at: Time.now)
+    update status: 'running', started_at: Time.now
 
     import_without_status
-    workbench_import.update(status: 'successful', ended_at: Time.now)
+    update status: 'successful', ended_at: Time.now
   rescue Exception => e
-    workbench_import.update(status: 'failed', ended_at: Time.now)
-    raise e
+    update status: 'failed', ended_at: Time.now
+    Rails.logger.error "Error in GTFS import: #{e} #{e.backtrace.join('\n')}"
+  ensure
+    notify_parent
+    referential&.update ready: true
+  end
+
+  def self.accept_file?(file)
+    Zip::File.open(file) do |zip_file|
+      zip_file.glob('agency.txt').size == 1
+    end
+  rescue Exception => e
+    Rails.logger.debug "Error in testing GTFS file: #{e}"
+    return false
+  end
+
+  def create_referential
+    self.referential ||= Referential.create!(
+      name: "GTFS Import",
+      organisation_id: workbench.organisation_id,
+      workbench_id: workbench.id,
+      metadatas: [referential_metadata]
+    )
+  end
+
+  def referential_metadata
+    registration_numbers = source.routes.map(&:id)
+    line_ids = line_referential.lines.where(registration_number: registration_numbers).pluck(:id)
+
+    start_dates, end_dates = source.calendars.map { |c| [c.start_date, c.end_date ] }.transpose
+    excluded_dates = source.calendar_dates.select { |d| d.exception_type == "2" }.map(&:date)
+
+    min_date = Date.parse (start_dates + [excluded_dates.min]).compact.min
+    max_date = Date.parse (end_dates + [excluded_dates.max]).compact.max
+
+    ReferentialMetadata.new line_ids: line_ids, periodes: [min_date..max_date]
   end
 
   attr_accessor :local_file
@@ -31,7 +65,10 @@ class Import::Gtfs < Import::Base
   end
 
   def local_temp_file(&block)
-    Tempfile.open "chouette-import", local_temp_directory, &block
+    Tempfile.open("chouette-import", local_temp_directory) do |file|
+      file.binmode
+      yield file
+    end
   end
 
   def download_path
@@ -60,16 +97,21 @@ class Import::Gtfs < Import::Base
     @source ||= ::GTFS::Source.build local_file
   end
 
-  delegate :line_referential, :stop_area_referential, to: :referential
-
-  def import_without_status
-    referential.switch
+  delegate :line_referential, :stop_area_referential, to: :workbench
 
+  def prepare_referential
     import_agencies
     import_stops
-    import_calandars
-
     import_routes
+
+    create_referential
+    referential.switch
+  end
+
+  def import_without_status
+    prepare_referential
+
+    import_calendars
     import_trips
     import_stop_times
   end
@@ -197,9 +239,16 @@ class Import::Gtfs < Import::Base
   def save_model(model)
     unless model.save
       Rails.logger.info "Can't save #{model.class.name} : #{model.errors.inspect}"
-      raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name}")
+      raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name} : #{model.errors.inspect}")
     end
     Rails.logger.debug "Created #{model.inspect}"
   end
 
+  def notify_parent
+    return unless parent.present?
+    return if notified_parent_at
+    parent.child_change
+    update_column :notified_parent_at, Time.now
+  end
+
 end
diff --git a/app/workers/gtfs_import_worker.rb b/app/workers/gtfs_import_worker.rb
index 093869475..02f5053b0 100644
--- a/app/workers/gtfs_import_worker.rb
+++ b/app/workers/gtfs_import_worker.rb
@@ -1,4 +1,4 @@
-class WorkbenchImportWorker
+class GtfsImportWorker
   include Sidekiq::Worker
 
   def perform(import_id)
diff --git a/spec/models/import/gtfs_spec.rb b/spec/models/import/gtfs_spec.rb
index c2ba4e909..934f13c1d 100644
--- a/spec/models/import/gtfs_spec.rb
+++ b/spec/models/import/gtfs_spec.rb
@@ -9,8 +9,15 @@ RSpec.describe Import::Gtfs do
     end
   end
 
+  let(:workbench) do
+    create :workbench do |workbench|
+      workbench.line_referential.objectid_format = "netex"
+      workbench.stop_area_referential.objectid_format = "netex"
+    end
+  end
+
   def create_import(file)
-    Import::Gtfs.new referential: referential, local_file: fixtures_path(file)
+    Import::Gtfs.new workbench: workbench, local_file: fixtures_path(file)
   end
 
   describe "#import_agencies" do
@@ -18,7 +25,7 @@ RSpec.describe Import::Gtfs do
     it "should create a company for each agency" do
       import.import_agencies
 
-      expect(referential.line_referential.companies.pluck(:registration_number, :name)).to eq([["DTA","Demo Transit Authority"]])
+      expect(workbench.line_referential.companies.pluck(:registration_number, :name)).to eq([["DTA","Demo Transit Authority"]])
     end
   end
 
@@ -42,7 +49,7 @@ RSpec.describe Import::Gtfs do
         ["FUR_CREEK_RES", "Furnace Creek Resort (Demo)", nil, 36.425288, -117.133162]
       ]
 
-      expect(referential.stop_area_referential.stop_areas.pluck(*defined_attributes)).to eq(expected_attributes)
+      expect(workbench.stop_area_referential.stop_areas.pluck(*defined_attributes)).to eq(expected_attributes)
     end
   end
 
@@ -64,17 +71,18 @@ RSpec.describe Import::Gtfs do
         ["AB", "Airport - Bullfrog", "10", "Airport - Bullfrog", nil, nil, nil]
       ]
 
-      expect(referential.line_referential.lines.includes(:company).pluck(*defined_attributes)).to eq(expected_attributes)
+      expect(workbench.line_referential.lines.includes(:company).pluck(*defined_attributes)).to eq(expected_attributes)
     end
   end
 
   describe "#import_trips" do
     let(:import) { create_import "google-sample-feed.zip" }
-    it "should create a Route for each trip" do
-      referential.switch
-
+    before do
+      import.prepare_referential
       import.import_calendars
-      import.import_routes
+    end
+
+    it "should create a Route for each trip" do
       import.import_trips
 
       defined_attributes = [
@@ -94,14 +102,10 @@ RSpec.describe Import::Gtfs do
         ["AAMV", "inbound", "to Airport", "to Airport"]
       ]
 
-      expect(referential.routes.includes(:line).pluck(*defined_attributes)).to eq(expected_attributes)
+      expect(import.referential.routes.includes(:line).pluck(*defined_attributes)).to eq(expected_attributes)
     end
 
     it "should create a JourneyPattern for each trip" do
-      referential.switch
-
-      import.import_calendars
-      import.import_routes
       import.import_trips
 
       defined_attributes = [
@@ -111,14 +115,10 @@ RSpec.describe Import::Gtfs do
         "to Bullfrog", "to Airport", "Shuttle", "Outbound", "Inbound", "to Furnace Creek Resort", "to Bullfrog", "to Amargosa Valley", "to Airport", "to Amargosa Valley", "to Airport"
       ]
 
-      expect(referential.journey_patterns.pluck(*defined_attributes)).to eq(expected_attributes)
+      expect(import.referential.journey_patterns.pluck(*defined_attributes)).to eq(expected_attributes)
     end
 
     it "should create a VehicleJourney for each trip" do
-      referential.switch
-
-      import.import_calendars
-      import.import_routes
       import.import_trips
 
       defined_attributes = ->(v) {
@@ -138,20 +138,20 @@ RSpec.describe Import::Gtfs do
         ["to Airport", "Calendar WE"]
       ]
 
-      expect(referential.vehicle_journeys.map(&defined_attributes)).to eq(expected_attributes)
+      expect(import.referential.vehicle_journeys.map(&defined_attributes)).to eq(expected_attributes)
     end
   end
 
   describe "#import_stop_times" do
     let(:import) { create_import "google-sample-feed.zip" }
 
-    it "should create a VehicleJourneyAtStop for each stop_time" do
-      referential.switch
-
-      import.import_stops
-      import.import_routes
+    before do
+      import.prepare_referential
       import.import_calendars
       import.import_trips
+    end
+
+    it "should create a VehicleJourneyAtStop for each stop_time" do
       import.import_stop_times
 
       def t(value)
@@ -198,9 +198,11 @@ RSpec.describe Import::Gtfs do
   describe "#import_calendars" do
     let(:import) { create_import "google-sample-feed.zip" }
 
-    it "should create a Timetable for each calendar" do
-      referential.switch
+    before do
+      import.prepare_referential
+    end
 
+    it "should create a Timetable for each calendar" do
       import.import_calendars
 
       def d(value)
@@ -221,9 +223,8 @@ RSpec.describe Import::Gtfs do
   describe "#download_local_file" do
 
     let(:file) { "google-sample-feed.zip" }
-    let(:referential) { create :workbench_referential }
     let(:import) do
-      Import::Gtfs.create! name: "GTFS test", creator: "Test", workbench: referential.workbench, referential: referential, file: open_fixture(file), download_host: "rails_host"
+      Import::Gtfs.create! name: "GTFS test", creator: "Test", workbench: workbench, file: open_fixture(file), download_host: "rails_host"
     end
 
     let(:download_url) { "#{import.download_host}/workbenches/#{import.workbench_id}/imports/#{import.id}/download?token=#{import.token_download}" }