From eb428adf7d5fedf947c00f30bf1f04462f2291e3 Mon Sep 17 00:00:00 2001 From: Zog Date: Tue, 20 Feb 2018 08:31:43 +0100 Subject: Refs #5924; Fix some bugs revealed during imports - The checksum computing for VehicleJourneyAtStops - Offsets calculation in VehicleJourneys --- app/models/chouette/vehicle_journey.rb | 2 +- app/models/chouette/vehicle_journey_at_stop.rb | 8 ++-- .../vehicle_journey_at_stops_day_offset.rb | 16 +++++--- app/models/simple_importer.rb | 37 ++++++++++++------ lib/tasks/imports.rake | 44 ++++++++++++++++------ 5 files changed, 73 insertions(+), 34 deletions(-) diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb index 028cd18dd..1a79db823 100644 --- a/app/models/chouette/vehicle_journey.rb +++ b/app/models/chouette/vehicle_journey.rb @@ -105,7 +105,7 @@ module Chouette attrs << self.try(:company).try(:get_objectid).try(:local_id) attrs << self.footnotes.map(&:checksum).sort vjas = self.vehicle_journey_at_stops - vjas += VehicleJourneyAtStop.where(vehicle_journey_id: self.id) + vjas += VehicleJourneyAtStop.where(vehicle_journey_id: self.id) unless self.new_record? attrs << vjas.uniq.sort_by { |s| s.stop_point&.position }.map(&:checksum).sort end end diff --git a/app/models/chouette/vehicle_journey_at_stop.rb b/app/models/chouette/vehicle_journey_at_stop.rb index eda711ade..3b4f35f13 100644 --- a/app/models/chouette/vehicle_journey_at_stop.rb +++ b/app/models/chouette/vehicle_journey_at_stop.rb @@ -41,7 +41,7 @@ module Chouette :arrival_day_offset, I18n.t( 'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max', - short_id: vehicle_journey.get_objectid.short_id, + short_id: vehicle_journey&.get_objectid&.short_id, max: DAY_OFFSET_MAX + 1 ) ) @@ -52,7 +52,7 @@ module Chouette :departure_day_offset, I18n.t( 'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max', - short_id: vehicle_journey.get_objectid.short_id, + short_id: vehicle_journey&.get_objectid&.short_id, max: DAY_OFFSET_MAX + 1 ) ) @@ -69,8 +69,8 @@ module Chouette def checksum_attributes [].tap do |attrs| - attrs << self.departure_time.try(:to_s, :time) - attrs << self.arrival_time.try(:to_s, :time) + attrs << self.departure_time&.utc.try(:to_s, :time) + attrs << self.arrival_time&.utc.try(:to_s, :time) attrs << self.departure_day_offset.to_s attrs << self.arrival_day_offset.to_s end diff --git a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb index b2cb90d11..7497cd72c 100644 --- a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb +++ b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb @@ -11,13 +11,19 @@ module Chouette @at_stops.inject(nil) do |prior_stop, stop| next stop if prior_stop.nil? - if stop.arrival_time < prior_stop.departure_time || - stop.arrival_time < prior_stop.arrival_time + # we only compare time of the day, not actual times + stop_arrival_time = stop.arrival_time - stop.arrival_time.to_date.to_time + stop_departure_time = stop.departure_time - stop.departure_time.to_date.to_time + prior_stop_arrival_time = prior_stop.arrival_time - prior_stop.arrival_time.to_date.to_time + prior_stop_departure_time = prior_stop.departure_time - prior_stop.departure_time.to_date.to_time + + if stop_arrival_time < prior_stop_departure_time || + stop_arrival_time < prior_stop_arrival_time arrival_offset += 1 end - if stop.departure_time < stop.arrival_time || - stop.departure_time < prior_stop.departure_time + if stop_departure_time < stop_arrival_time || + stop_departure_time < prior_stop_departure_time departure_offset += 1 end @@ -39,4 +45,4 @@ module Chouette save end end -end \ No newline at end of file +end diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb index dea8f85ad..d9538a074 100644 --- a/app/models/simple_importer.rb +++ b/app/models/simple_importer.rb @@ -130,7 +130,7 @@ class SimpleImporter < ActiveRecord::Base begin handle_row row fail_with_error ->(){ @current_record.errors.messages } do - new_record = @current_record.new_record? + new_record = @current_record&.new_record? @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange) @event = new_record ? :creation : :update self.configuration.before_actions(:each_save).each do |action| @@ -139,13 +139,15 @@ class SimpleImporter < ActiveRecord::Base ### This could fail if the record has a mandatory relation which is not yet resolved ### TODO: do not attempt to save if the current record if waiting for resolution ### and fail at the end if there remains unresolved relations - if self.configuration.ignore_failures - unless @current_record.save - @new_status = colorize("x", :red) - push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error}) + if @current_record + if self.configuration.ignore_failures + unless @current_record.save + @new_status = colorize("x", :red) + push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error}) + end + else + @current_record.save! end - else - @current_record.save! end self.configuration.after_actions(:each_save).each do |action| action.call self, @current_record @@ -157,7 +159,7 @@ class SimpleImporter < ActiveRecord::Base push_in_journal({event: @event, kind: :log}) if @current_record&.valid? @statuses += @new_status self.configuration.columns.each do |col| - if col.name && @resolution_queue.any? + if @current_record && col.name && @resolution_queue.any? val = @current_record.send col[:attribute] (@resolution_queue.delete([col.name, val]) || []).each do |res| record = res[:record] @@ -239,6 +241,14 @@ class SimpleImporter < ActiveRecord::Base rescue 100 end + + @status_height ||= begin + term_height = %x(tput lines).to_i + term_height - 3 + rescue + 50 + end + full_status = @statuses || "" full_status = full_status.last(@status_width*10) || "" padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min @@ -246,18 +256,21 @@ class SimpleImporter < ActiveRecord::Base msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}" + lines_count = (@status_height / 2) - 3 + if @messages.any? msg += "\n\n" msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green - msg += "[...]\n" if @messages.count > 10 - msg += @messages.last(10).join("\n") + msg += "[...]\n" if @messages.count > lines_count + msg += @messages.last(lines_count).join("\n") + msg += "\n"*[lines_count-@messages.count, 0].max end if @errors.any? msg += "\n\n" msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red - msg += "[...]\n" if @errors.count > 10 - msg += @errors.last(10).map do |j| + msg += "[...]\n" if @errors.count > lines_count + msg += @errors.last(lines_count).map do |j| kind = j[:kind] kind = colorize(kind, kind == :error ? :red : :orange) encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}" diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake index eca7e6849..6791bd877 100644 --- a/lib/tasks/imports.rake +++ b/lib/tasks/imports.rake @@ -37,9 +37,14 @@ namespace :import do end end puts "\e[33m***\e[0m Start importing" - importer.import(verbose: true) - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" - importer_output_to_csv importer + begin + importer.import(verbose: true) + rescue Interrupt + raise + ensure + puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" + importer_output_to_csv importer + end end desc "import the given file with the corresponding importer in the given StopAreaReferential" @@ -51,9 +56,14 @@ namespace :import do config.context = {stop_area_referential: referential} end puts "\e[33m***\e[0m Start importing" - importer.import(verbose: true) - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" - importer_output_to_csv importer + begin + importer.import(verbose: true) + rescue Interrupt + raise + ensure + puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" + importer_output_to_csv importer + end end desc "import the given routes files" @@ -67,9 +77,14 @@ namespace :import do config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath]} end puts "\e[33m***\e[0m Start importing" - importer.import(verbose: true) - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" - importer_output_to_csv importer + begin + importer.import(verbose: true) + rescue Interrupt + raise + ensure + puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" + importer_output_to_csv importer + end end desc "import the given file with the corresponding importer in the given LineReferential" @@ -81,8 +96,13 @@ namespace :import do config.context = {line_referential: referential} end puts "\e[33m***\e[0m Start importing" - importer.import(verbose: true) - puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" - importer_output_to_csv importer + begin + importer.import(verbose: true) + rescue Interrupt + raise + ensure + puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + (importer.status || "") + "\e[0m" + importer_output_to_csv importer + end end end -- cgit v1.2.3