Refs #5924; Fix some bugs revealed during imports

- The checksum computing for VehicleJourneyAtStops
- Offsets calculation in VehicleJourneys

5 files changed, 73 insertions, 34 deletions

diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb
index 028cd18dd..1a79db823 100644
--- a/app/models/chouette/vehicle_journey.rb
+++ b/app/models/chouette/vehicle_journey.rb
@@ -105,7 +105,7 @@ module Chouette
         attrs << self.try(:company).try(:get_objectid).try(:local_id)
         attrs << self.footnotes.map(&:checksum).sort
         vjas =  self.vehicle_journey_at_stops
-        vjas += VehicleJourneyAtStop.where(vehicle_journey_id: self.id)
+        vjas += VehicleJourneyAtStop.where(vehicle_journey_id: self.id) unless self.new_record?
         attrs << vjas.uniq.sort_by { |s| s.stop_point&.position }.map(&:checksum).sort
       end
     end
diff --git a/app/models/chouette/vehicle_journey_at_stop.rb b/app/models/chouette/vehicle_journey_at_stop.rb
index eda711ade..3b4f35f13 100644
--- a/app/models/chouette/vehicle_journey_at_stop.rb
+++ b/app/models/chouette/vehicle_journey_at_stop.rb
@@ -41,7 +41,7 @@ module Chouette
           :arrival_day_offset,
           I18n.t(
             'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max',
-            short_id: vehicle_journey.get_objectid.short_id,
+            short_id: vehicle_journey&.get_objectid&.short_id,
             max: DAY_OFFSET_MAX + 1
           )
         )
@@ -52,7 +52,7 @@ module Chouette
           :departure_day_offset,
           I18n.t(
             'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max',
-            short_id: vehicle_journey.get_objectid.short_id,
+            short_id: vehicle_journey&.get_objectid&.short_id,
             max: DAY_OFFSET_MAX + 1
           )
         )
@@ -69,8 +69,8 @@ module Chouette
 
     def checksum_attributes
       [].tap do |attrs|
-        attrs << self.departure_time.try(:to_s, :time)
-        attrs << self.arrival_time.try(:to_s, :time)
+        attrs << self.departure_time&.utc.try(:to_s, :time)
+        attrs << self.arrival_time&.utc.try(:to_s, :time)
         attrs << self.departure_day_offset.to_s
         attrs << self.arrival_day_offset.to_s
       end
diff --git a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb
index b2cb90d11..7497cd72c 100644
--- a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb
+++ b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb
@@ -11,13 +11,19 @@ module Chouette
       @at_stops.inject(nil) do |prior_stop, stop|
         next stop if prior_stop.nil?
 
-        if stop.arrival_time < prior_stop.departure_time ||
-            stop.arrival_time < prior_stop.arrival_time
+        # we only compare time of the day, not actual times
+        stop_arrival_time = stop.arrival_time - stop.arrival_time.to_date.to_time
+        stop_departure_time = stop.departure_time - stop.departure_time.to_date.to_time
+        prior_stop_arrival_time = prior_stop.arrival_time - prior_stop.arrival_time.to_date.to_time
+        prior_stop_departure_time = prior_stop.departure_time - prior_stop.departure_time.to_date.to_time
+
+        if stop_arrival_time < prior_stop_departure_time ||
+            stop_arrival_time < prior_stop_arrival_time
           arrival_offset += 1
         end
 
-        if stop.departure_time < stop.arrival_time ||
-            stop.departure_time < prior_stop.departure_time
+        if stop_departure_time < stop_arrival_time ||
+            stop_departure_time < prior_stop_departure_time
           departure_offset += 1
         end
 
@@ -39,4 +45,4 @@ module Chouette
       save
     end
   end
-end
\ No newline at end of file
+end
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index dea8f85ad..d9538a074 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -130,7 +130,7 @@ class SimpleImporter < ActiveRecord::Base
       begin
         handle_row row
         fail_with_error ->(){ @current_record.errors.messages } do
-          new_record = @current_record.new_record?
+          new_record = @current_record&.new_record?
           @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange)
           @event = new_record ? :creation : :update
           self.configuration.before_actions(:each_save).each do |action|
@@ -139,13 +139,15 @@ class SimpleImporter < ActiveRecord::Base
           ### This could fail if the record has a mandatory relation which is not yet resolved
           ### TODO: do not attempt to save if the current record if waiting for resolution
           ###       and fail at the end if there remains unresolved relations
-          if self.configuration.ignore_failures
-            unless @current_record.save
-              @new_status = colorize("x", :red)
-              push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
+          if @current_record
+            if self.configuration.ignore_failures
+              unless @current_record.save
+                @new_status = colorize("x", :red)
+                push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
+              end
+            else
+              @current_record.save!
             end
-          else
-            @current_record.save!
           end
           self.configuration.after_actions(:each_save).each do |action|
             action.call self, @current_record
@@ -157,7 +159,7 @@ class SimpleImporter < ActiveRecord::Base
       push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
       @statuses += @new_status
       self.configuration.columns.each do |col|
-      if col.name && @resolution_queue.any?
+      if @current_record && col.name && @resolution_queue.any?
         val = @current_record.send col[:attribute]
         (@resolution_queue.delete([col.name, val]) || []).each do |res|
           record = res[:record]
@@ -239,6 +241,14 @@ class SimpleImporter < ActiveRecord::Base
     rescue
       100
     end
+
+    @status_height ||= begin
+      term_height = %x(tput lines).to_i
+      term_height - 3
+    rescue
+      50
+    end
+
     full_status = @statuses || ""
     full_status = full_status.last(@status_width*10) || ""
     padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
@@ -246,18 +256,21 @@ class SimpleImporter < ActiveRecord::Base
 
     msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
 
+    lines_count = (@status_height / 2) - 3
+
     if @messages.any?
       msg += "\n\n"
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
-      msg += "[...]\n" if @messages.count > 10
-      msg += @messages.last(10).join("\n")
+      msg += "[...]\n" if @messages.count > lines_count
+      msg += @messages.last(lines_count).join("\n")
+      msg += "\n"*[lines_count-@messages.count, 0].max
     end
 
     if @errors.any?
       msg += "\n\n"
       msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
-      msg += "[...]\n" if @errors.count > 10
-      msg += @errors.last(10).map do |j|
+      msg += "[...]\n" if @errors.count > lines_count
+      msg += @errors.last(lines_count).map do |j|
         kind = j[:kind]
         kind = colorize(kind, kind == :error ? :red : :orange)
         encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake
index eca7e6849..6791bd877 100644
--- a/lib/tasks/imports.rake
+++ b/lib/tasks/imports.rake
@@ -37,9 +37,14 @@ namespace :import do
       end
     end
     puts "\e[33m***\e[0m Start importing"
-    importer.import(verbose: true)
-    puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
-    importer_output_to_csv importer
+    begin
+      importer.import(verbose: true)
+    rescue Interrupt
+      raise
+    ensure
+      puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m"
+      importer_output_to_csv importer
+    end
   end
 
   desc "import the given file with the corresponding importer in the given StopAreaReferential"
@@ -51,9 +56,14 @@ namespace :import do
       config.context = {stop_area_referential: referential}
     end
     puts "\e[33m***\e[0m Start importing"
-    importer.import(verbose: true)
-    puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
-    importer_output_to_csv importer
+    begin
+      importer.import(verbose: true)
+    rescue Interrupt
+      raise
+    ensure
+      puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m"
+      importer_output_to_csv importer
+    end
   end
 
   desc "import the given routes files"
@@ -67,9 +77,14 @@ namespace :import do
       config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath]}
     end
     puts "\e[33m***\e[0m Start importing"
-    importer.import(verbose: true)
-    puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
-    importer_output_to_csv importer
+    begin
+      importer.import(verbose: true)
+    rescue Interrupt
+      raise
+    ensure
+      puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m"
+      importer_output_to_csv importer
+    end
   end
 
   desc "import the given file with the corresponding importer in the given LineReferential"
@@ -81,8 +96,13 @@ namespace :import do
       config.context = {line_referential: referential}
     end
     puts "\e[33m***\e[0m Start importing"
-    importer.import(verbose: true)
-    puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
-    importer_output_to_csv importer
+    begin
+      importer.import(verbose: true)
+    rescue Interrupt
+      raise
+    ensure
+      puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m"
+      importer_output_to_csv importer
+    end
   end
 end