aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--app/models/chouette/company.rb1
-rw-r--r--app/models/chouette/line.rb10
-rw-r--r--app/models/chouette/purchase_window.rb1
-rw-r--r--app/models/chouette/route.rb12
-rw-r--r--app/models/chouette/stop_area.rb1
-rw-r--r--app/models/chouette/time_table.rb4
-rw-r--r--app/models/simple_importer.rb276
-rw-r--r--lib/tasks/imports.rake38
-rw-r--r--spec/fixtures/simple_importer/lines_mapping.csv11
-rw-r--r--spec/models/simple_importer_spec.rb83
10 files changed, 335 insertions, 102 deletions
diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb
index b3d40ab96..2e942d520 100644
--- a/app/models/chouette/company.rb
+++ b/app/models/chouette/company.rb
@@ -15,6 +15,7 @@ module Chouette
[:organizational_unit, :operating_department_name, :code, :phone, :fax, :email, :url, :time_zone]
end
+ def local_id; id end
end
end
diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb
index 874353752..5df45efbd 100644
--- a/app/models/chouette/line.rb
+++ b/app/models/chouette/line.rb
@@ -49,7 +49,7 @@ module Chouette
[:published_name, :number, :comment, :url, :color, :text_color, :stable_id]
end
- def local_id; registration_number end
+ def local_id; registration_number || id end
def geometry_presenter
Chouette::Geometry::LinePresenter.new self
@@ -83,6 +83,14 @@ module Chouette
line_referential.companies.where(id: ([company_id] + Array(secondary_company_ids)).compact)
end
+ def deactivate
+ self.deactivated = true
+ end
+
+ def activate
+ self.deactivated = false
+ end
+
def deactivate!
update_attribute :deactivated, true
end
diff --git a/app/models/chouette/purchase_window.rb b/app/models/chouette/purchase_window.rb
index 334493015..157390a21 100644
--- a/app/models/chouette/purchase_window.rb
+++ b/app/models/chouette/purchase_window.rb
@@ -19,6 +19,7 @@ module Chouette
scope :contains_date, ->(date) { where('date ? <@ any (date_ranges)', date) }
scope :overlap_dates, ->(date_range) { where('daterange(?, ?) && any (date_ranges)', date_range.first, date_range.last + 1.day) }
+ scope :matching_dates, ->(date_range) { where('ARRAY[daterange(?, ?)] = date_ranges', date_range.first, date_range.last + 1.day) }
def self.ransackable_scopes(auth_object = nil)
[:contains_date]
diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb
index e418134de..3729deb7d 100644
--- a/app/models/chouette/route.rb
+++ b/app/models/chouette/route.rb
@@ -186,15 +186,9 @@ module Chouette
end
def full_journey_pattern
- out = journey_patterns.find{|jp| jp.stop_points.count == self.stop_points.count }
- unless out
- out = journey_patterns.build name: self.name
- self.stop_points.each do |sp|
- out.stop_points.build stop_area: sp.stop_area, position: sp.position
- end
- out.save!
- end
- out
+ journey_pattern = journey_patterns.find_or_create_by registration_number: self.number, name: self.name
+ journey_pattern.stop_points = self.stop_points
+ journey_pattern
end
protected
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index c85a8b37d..699616863 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -112,7 +112,6 @@ module Chouette
id.to_s
end
end
- alias_method :local_id, :user_objectid
alias_method :local_id, :user_objectid
diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb
index 15b22b671..b76de852a 100644
--- a/app/models/chouette/time_table.rb
+++ b/app/models/chouette/time_table.rb
@@ -44,10 +44,10 @@ module Chouette
attrs << self.int_day_types
dates = self.dates
dates += TimeTableDate.where(time_table_id: self.id)
- attrs << dates.map(&:checksum).map(&:to_s).sort
+ attrs << dates.map(&:checksum).map(&:to_s).uniq.sort
periods = self.periods
periods += TimeTablePeriod.where(time_table_id: self.id)
- attrs << periods.map(&:checksum).map(&:to_s).sort
+ attrs << periods.map(&:checksum).map(&:to_s).uniq.sort
end
end
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index bf6f3b406..dea8f85ad 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -30,7 +30,7 @@ class SimpleImporter < ActiveRecord::Base
end
def context
- self.configuration.context || {}
+ self.configuration.context
end
def resolve col_name, value, &block
@@ -42,99 +42,183 @@ class SimpleImporter < ActiveRecord::Base
def import opts={}
@verbose = opts.delete :verbose
+
+
@resolution_queue = Hash.new{|h,k| h[k] = []}
- number_of_lines = 0
- padding = 1
+ @errors = []
+ @messages = []
+ @number_of_lines = 0
+ @padding = 1
+ @current_line = 0
fail_with_error "File not found: #{self.filepath}" do
- number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
- padding = [1, Math.log(number_of_lines, 10).ceil()].max
+ @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
+ @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+ end
+
+
+ self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+ @statuses = ""
+
+ if ENV["NO_TRANSACTION"]
+ process_csv_file
+ else
+ ActiveRecord::Base.transaction do
+ process_csv_file
+ end
+ end
+ self.status ||= :success
+ rescue FailedImport
+ self.status = :failed
+ ensure
+ self.save!
+ end
+
+ def fail_with_error msg=nil, opts={}
+ begin
+ yield
+ rescue => e
+ msg = msg.call if msg.is_a?(Proc)
+ custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
+ push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
+ @new_status = colorize("x", :red)
+ if self.configuration.ignore_failures
+ raise FailedRow if opts[:abort_row]
+ else
+ raise FailedImport
+ end
end
+ end
+
+ def encode_string s
+ s.encode("utf-8").force_encoding("utf-8")
+ end
+
+ def dump_csv_from_context
+ filepath = "./#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
+ # for some reason, context[:csv].to_csv does not work
+ CSV.open(filepath, 'w') do |csv|
+ header = true
+ context[:csv].each do |row|
+ csv << row.headers if header
+ csv << row.fields
+ header = false
+ end
+ end
+ log "CSV file dumped in #{filepath}"
+ end
+
+ def log msg, opts={}
+ msg = colorize msg, opts[:color] if opts[:color]
+ if opts[:append]
+ @messages[-1] = (@messages[-1] || "") + msg
+ else
+ @messages << msg
+ end
+ print_state
+ end
- current_line = 0
- status = :success
- statuses = ""
- log "#{"%#{padding}d" % 0}/#{number_of_lines}", clear: true
- ActiveRecord::Base.transaction do
- self.configuration.before_actions(:all).each do |action| action.call self end
+ protected
- CSV.foreach(filepath, self.configuration.csv_options) do |row|
- status = handle_row row, status
+ def process_csv_file
+ self.configuration.before_actions(:all).each do |action| action.call self end
+ log "Starting import ...", color: :green
+ (context[:csv] || CSV.read(filepath, self.configuration.csv_options)).each do |row|
+ @current_row = row
+ @new_status = nil
+ begin
+ handle_row row
fail_with_error ->(){ @current_record.errors.messages } do
new_record = @current_record.new_record?
+ @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange)
+ @event = new_record ? :creation : :update
self.configuration.before_actions(:each_save).each do |action|
action.call self, @current_record
end
### This could fail if the record has a mandatory relation which is not yet resolved
### TODO: do not attempt to save if the current record if waiting for resolution
### and fail at the end if there remains unresolved relations
- @current_record.save!
- self.journal.push({event: (new_record ? :creation : :update), kind: :log})
- statuses += new_record ? colorize("✓", :green) : colorize("-", :orange)
- end
- self.configuration.columns.each do |col|
- if col.name && @resolution_queue.any?
- val = @current_record.send col[:attribute]
- (@resolution_queue.delete([col.name, val]) || []).each do |res|
- record = res[:record]
- attribute = res[:attribute]
- value = res[:block].call(val, record)
- record.send "#{attribute}=", value
- record.save!
+ if self.configuration.ignore_failures
+ unless @current_record.save
+ @new_status = colorize("x", :red)
+ push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
end
+ else
+ @current_record.save!
+ end
+ self.configuration.after_actions(:each_save).each do |action|
+ action.call self, @current_record
end
end
- current_line += 1
- log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
+ rescue FailedRow
+ @new_status = colorize("x", :red)
end
- self.configuration.after_actions(:all).each do |action|
- action.call self
+ push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
+ @statuses += @new_status
+ self.configuration.columns.each do |col|
+ if col.name && @resolution_queue.any?
+ val = @current_record.send col[:attribute]
+ (@resolution_queue.delete([col.name, val]) || []).each do |res|
+ record = res[:record]
+ attribute = res[:attribute]
+ value = res[:block].call(val, record)
+ record.send "#{attribute}=", value
+ record.save!
+ end
end
end
- self.update_attribute :status, status
- rescue FailedImport
- self.update_attribute :status, :failed
- ensure
- self.save!
- end
+ print_state
+ @current_line += 1
+ end
- def fail_with_error msg
begin
- yield
- rescue => e
- msg = msg.call if msg.is_a?(Proc)
- log "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red
- self.journal.push({message: msg, error: e.message, event: :error, kind: :error})
- raise FailedImport
+ self.configuration.after_actions(:all).each do |action|
+ action.call self
+ end
+ rescue FailedRow
end
end
- protected
-
- def handle_row row, status
+ def handle_row row
if self.configuration.get_custom_handler
instance_exec(row, &self.configuration.get_custom_handler)
else
- @current_record = self.configuration.find_record row
- self.configuration.columns.each do |col|
- @current_attribute = col[:attribute]
- val = col[:value]
- if val.nil? || val.is_a?(Proc)
- if row.has_key? col.name
- if val.is_a?(Proc)
- val = instance_exec(row[col.name], &val)
+ fail_with_error "", abort_row: true do
+ @current_record = self.configuration.find_record row
+ self.configuration.columns.each do |col|
+ @current_attribute = col[:attribute]
+ val = col[:value]
+ if val.nil? || val.is_a?(Proc)
+ if row.has_key? col.name
+ if val.is_a?(Proc)
+ val = instance_exec(row[col.name], &val)
+ else
+ val = row[col.name]
+ end
else
- val = row[col.name]
+ push_in_journal({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+ self.status ||= :success_with_warnings
end
- else
- self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
- status = :success_with_warnings
end
+
+ if val.nil? && col.required?
+ raise "MISSING VALUE FOR COLUMN #{col.name}"
+ end
+ val = encode_string(val) if val.is_a?(String)
+ @current_record.send "#{@current_attribute}=", val if val
end
- @current_record.send "#{@current_attribute}=", val if val
end
end
- status
+ end
+
+ def push_in_journal data
+ line = @current_line + 1
+ line += 1 if configuration.headers
+ self.journal.push data.update(line: line, row: @current_row)
+ if data[:kind] == :error || data[:kind] == :warning
+ @errors.push data
+ end
end
def colorize txt, color
@@ -146,23 +230,59 @@ class SimpleImporter < ActiveRecord::Base
"\e[#{color}m#{txt}\e[0m"
end
- def log msg, opts={}
+ def print_state
+ return unless @verbose
+
+ @status_width ||= begin
+ term_width = %x(tput cols).to_i
+ term_width - @padding - 10
+ rescue
+ 100
+ end
+ full_status = @statuses || ""
+ full_status = full_status.last(@status_width*10) || ""
+ padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
+ full_status = "#{full_status}#{"."*[padding_size, 0].max}"
+
+ msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+
+ if @messages.any?
+ msg += "\n\n"
+ msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
+ msg += "[...]\n" if @messages.count > 10
+ msg += @messages.last(10).join("\n")
+ end
+
+ if @errors.any?
+ msg += "\n\n"
+ msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
+ msg += "[...]\n" if @errors.count > 10
+ msg += @errors.last(10).map do |j|
+ kind = j[:kind]
+ kind = colorize(kind, kind == :error ? :red : :orange)
+ encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
+ end.join("\n")
+ end
+ custom_print msg, clear: true
+ end
+
+ def custom_print msg, opts={}
return unless @verbose
out = ""
msg = colorize(msg, opts[:color]) if opts[:color]
- if opts[:clear] && @prev_msg_size
- out += "\b"*@prev_msg_size
- end
+ puts "\e[H\e[2J" if opts[:clear]
out += msg
print out
- @prev_msg_size = msg.size
end
class FailedImport < RuntimeError
end
+ class FailedRow < RuntimeError
+ end
+
class Configuration
- attr_accessor :model, :headers, :separator, :key, :context, :encoding
+ attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
attr_reader :columns
def initialize import_name, opts={}
@@ -174,6 +294,11 @@ class SimpleImporter < ActiveRecord::Base
@columns = opts[:columns] || []
@model = opts[:model]
@custom_handler = opts[:custom_handler]
+ @before = opts[:before]
+ @after = opts[:after]
+ @ignore_failures = opts[:ignore_failures]
+ @context = opts[:context] || {}
+ @scope = opts[:scope] || {}
end
def duplicate
@@ -188,7 +313,12 @@ class SimpleImporter < ActiveRecord::Base
encoding: @encoding,
columns: @columns.map(&:duplicate),
model: model,
- custom_handler: @custom_handler
+ custom_handler: @custom_handler,
+ before: @before,
+ after: @after,
+ ignore_failures: @ignore_failures,
+ context: @context,
+ scope: @scope
}
end
@@ -201,8 +331,14 @@ class SimpleImporter < ActiveRecord::Base
column && column[:attribute] || col_name
end
+ def record_scope
+ _scope = @scope
+ _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
+ _scope || model
+ end
+
def find_record attrs
- model.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+ record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
end
def csv_options
@@ -261,6 +397,10 @@ class SimpleImporter < ActiveRecord::Base
Column.new @options.dup
end
+ def required?
+ !!@options[:required]
+ end
+
def [](key)
@options[key]
end
diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake
index 9c6b1fbcd..eca7e6849 100644
--- a/lib/tasks/imports.rake
+++ b/lib/tasks/imports.rake
@@ -1,3 +1,5 @@
+require 'csv'
+
namespace :import do
desc "Notify parent imports when children finish"
task notify_parent: :environment do
@@ -9,12 +11,35 @@ namespace :import do
NetexImport.abort_old
end
+ def importer_output_to_csv importer
+ filepath = "./#{importer.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+ cols = %w(line kind event message error)
+ if importer.reload.journal.size > 0
+ keys = importer.journal.first["row"].map(&:first)
+ CSV.open(filepath, "w") do |csv|
+ csv << cols + keys
+ importer.journal.each do |j|
+ csv << cols.map{|c| j[c]} + j["row"].map(&:last)
+ end
+ end
+ puts "Import Output written in #{filepath}"
+ end
+ end
+
desc "import the given file with the corresponding importer"
- task :import, [:configuration_name, :filepath] => :environment do |t, args|
+ task :import, [:configuration_name, :filepath, :referential_id] => :environment do |t, args|
importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath]
+ if args[:referential_id].present?
+ referential = Referential.find args[:referential_id]
+ importer.configure do |config|
+ config.add_value :referential, referential
+ config.context = {referential: referential}
+ end
+ end
puts "\e[33m***\e[0m Start importing"
importer.import(verbose: true)
puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+ importer_output_to_csv importer
end
desc "import the given file with the corresponding importer in the given StopAreaReferential"
@@ -28,21 +53,23 @@ namespace :import do
puts "\e[33m***\e[0m Start importing"
importer.import(verbose: true)
puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+ importer_output_to_csv importer
end
- desc "import the given file with the corresponding importer in the given Referential and StopAreaReferential"
- task :import_in_referential_and_stop_area_referential, [:referential_id, :stop_area_referential_id, :configuration_name, :filepath] => :environment do |t, args|
+ desc "import the given routes files"
+ task :import_routes, [:referential_id, :configuration_name, :mapping_filepath, :filepath] => :environment do |t, args|
referential = Referential.find args[:referential_id]
referential.switch
- stop_area_referential = StopAreaReferential.find args[:stop_area_referential_id]
+ stop_area_referential = referential.stop_area_referential
importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath]
importer.configure do |config|
config.add_value :stop_area_referential, referential
- config.context = {stop_area_referential: stop_area_referential}
+ config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath]}
end
puts "\e[33m***\e[0m Start importing"
importer.import(verbose: true)
puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+ importer_output_to_csv importer
end
desc "import the given file with the corresponding importer in the given LineReferential"
@@ -56,5 +83,6 @@ namespace :import do
puts "\e[33m***\e[0m Start importing"
importer.import(verbose: true)
puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+ importer_output_to_csv importer
end
end
diff --git a/spec/fixtures/simple_importer/lines_mapping.csv b/spec/fixtures/simple_importer/lines_mapping.csv
new file mode 100644
index 000000000..b26d0ab59
--- /dev/null
+++ b/spec/fixtures/simple_importer/lines_mapping.csv
@@ -0,0 +1,11 @@
+id;timetable_route_id;route_name;stop_sequence;stop_distance;station_code;station_name;border;Ligne Chouette;Transporteur
+3354;1136;Paris centre - Bercy > Lille > Londres;1;0;XPB;Paris City Center - Bercy;f;Paris <> Londres - OUIBUS;OUIBUS
+3355;1136;Paris centre - Bercy > Lille > Londres;2;232;XDB;Lille;f;Paris <> Londres - OUIBUS;OUIBUS
+3749;1136;Paris centre - Bercy > Lille > Londres;3;350;COF;Coquelles - France;t;Paris <> Londres - OUIBUS;OUIBUS
+4772;1136;Paris centre - Bercy > Lille > Londres;4;350;COU;Coquelles - UK;t;Paris <> Londres - OUIBUS;OUIBUS
+3357;1136;Paris centre - Bercy > Lille > Londres;5;527;ZEP;London;f;Paris <> Londres - OUIBUS;OUIBUS
+3358;1137;Londres > Lille > Paris centre - Bercy;1;0;ZEP;London;f;Paris <> Londres - OUIBUS;OUIBUS
+3559;1137;Londres > Lille > Paris centre - Bercy;2;177;COU;Coquelles - UK;t;Paris <> Londres - OUIBUS;OUIBUS
+3743;1137;Londres > Lille > Paris centre - Bercy;3;177;COF;Coquelles - France;t;Paris <> Londres - OUIBUS;OUIBUS
+3360;1137;Londres > Lille > Paris centre - Bercy;4;295;XDB;Lille;f;Paris <> Londres - OUIBUS;OUIBUS
+3361;1137;Londres > Lille > Paris centre - Bercy;5;527;XPB;Paris City Center - Bercy;f;Paris <> Londres - OUIBUS;OUIBUS
diff --git a/spec/models/simple_importer_spec.rb b/spec/models/simple_importer_spec.rb
index e324399c7..2f520a3f3 100644
--- a/spec/models/simple_importer_spec.rb
+++ b/spec/models/simple_importer_spec.rb
@@ -27,7 +27,7 @@ RSpec.describe SimpleImporter do
describe "#import" do
let(:importer){ importer = SimpleImporter.new(configuration_name: :test, filepath: filepath) }
- let(:filepath){ Rails.root + "spec/fixtures/simple_importer/#{filename}" }
+ let(:filepath){ fixtures_path 'simple_importer', filename }
let(:filename){ "stop_area.csv" }
let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) }
@@ -47,7 +47,7 @@ RSpec.describe SimpleImporter do
end
it "should import the given file" do
- expect{importer.import}.to change{Chouette::StopArea.count}.by 1
+ expect{importer.import verbose: false}.to change{Chouette::StopArea.count}.by 1
expect(importer.status).to eq "success"
stop = Chouette::StopArea.last
expect(stop.name).to eq "Nom du Stop"
@@ -90,6 +90,21 @@ RSpec.describe SimpleImporter do
expect(stop.area_type).to eq "zdep"
expect(importer.reload.journal.last["event"]).to eq("update")
end
+
+ context "in another scope" do
+ before(:each) do
+ ref = create(:stop_area_referential)
+ importer.configure do |config|
+ config.context = { stop_area_referential: ref }
+ config.scope = ->{ context[:stop_area_referential].stop_areas }
+ end
+ end
+
+ it "should create the record" do
+ expect{importer.import verbose: true}.to change{Chouette::StopArea.count}.by 1
+ expect(importer.status).to eq "success"
+ end
+ end
end
context "with a missing column" do
@@ -101,7 +116,7 @@ RSpec.describe SimpleImporter do
end
end
- context "with a incomplete dataset" do
+ context "with an incomplete dataset" do
let(:filename){ "stop_area_incomplete.csv" }
it "should fail" do
expect{importer.import}.to_not raise_error
@@ -116,7 +131,7 @@ RSpec.describe SimpleImporter do
context "with a wrong filepath" do
let(:filename){ "not_found.csv" }
- it "should create a StopArea" do
+ it "should fail" do
expect{importer.import}.to_not raise_error
expect(importer.status).to eq "failed"
expect(importer.reload.journal.first["message"]).to eq "File not found: #{importer.filepath}"
@@ -221,27 +236,42 @@ RSpec.describe SimpleImporter do
let(:filename){ "stop_points_full.csv" }
before(:each) do
- create :line, name: "Paris centre - Bercy > Lille > Londres"
- create :line, name: "Londres > Lille > Paris centre - Bercy"
+ create :line, name: "Paris <> Londres - OUIBUS"
SimpleImporter.define :test do |config|
config.model = Chouette::Route
config.separator = ";"
config.context = {stop_area_referential: stop_area_referential}
+
+ config.before do |importer|
+ mapping = {}
+ path = Rails.root + "spec/fixtures/simple_importer/lines_mapping.csv"
+ CSV.foreach(path, importer.configuration.csv_options) do |row|
+ if row["Ligne Chouette"].present?
+ mapping[row["timetable_route_id"]] ||= Chouette::Line.find_by(name: importer.encode_string(row["Ligne Chouette"]))
+ end
+ end
+ importer.context[:mapping] = mapping
+ end
+
config.custom_handler do |row|
line = nil
- fail_with_error "MISSING LINE: #{row["route_name"]}" do
- line = Chouette::Line.find_by! name: row["route_name"]
+ fail_with_error "MISSING LINE FOR ROUTE: #{encode_string row["route_name"]}" do
+ line = context[:mapping][row["timetable_route_id"]]
+ raise unless line
end
@current_record = Chouette::Route.find_or_initialize_by number: row["timetable_route_id"]
- @current_record.name = row["route_name"]
- @current_record.published_name = row["route_name"]
+ @current_record.name = encode_string row["route_name"]
+ @current_record.published_name = encode_string row["route_name"]
@current_record.line = line
if @prev_route != @current_record
- if @prev_route
+ if @prev_route && @prev_route.valid?
journey_pattern = @prev_route.full_journey_pattern
- journey_pattern.set_distances @distances
+ fail_with_error "WRONG DISTANCES FOR ROUTE #{@prev_route.name} (#{@prev_route.number}): #{@distances.count} distances for #{@prev_route.stop_points.count} stops" do
+ journey_pattern.stop_points = @prev_route.stop_points
+ journey_pattern.set_distances @distances
+ end
fail_with_error ->(){ journey_pattern.errors.messages } do
journey_pattern.save!
end
@@ -250,6 +280,7 @@ RSpec.describe SimpleImporter do
end
@distances.push row["stop_distance"]
position = row["stop_sequence"].to_i - 1
+
stop_area = context[:stop_area_referential].stop_areas.where(registration_number: row["station_code"]).last
unless stop_area
stop_area = Chouette::StopArea.new registration_number: row["station_code"]
@@ -257,7 +288,7 @@ RSpec.describe SimpleImporter do
stop_area.kind = row["border"] == "f" ? :commercial : :non_commercial
stop_area.area_type = row["border"] == "f" ? :zdep : :border
stop_area.stop_area_referential = context[:stop_area_referential]
- fail_with_error ->{ stop_area.errors.messages } do
+ fail_with_error ->{p stop_area; "UNABLE TO CREATE STOP_AREA: #{stop_area.errors.messages}" }, abort_row: true do
stop_area.save!
end
end
@@ -266,15 +297,32 @@ RSpec.describe SimpleImporter do
stop_point.set_list_position position
else
stop_point = @current_record.stop_points.build(stop_area_id: stop_area.id, position: position)
+ stop_point.for_boarding = :normal
+ stop_point.for_alighting = :normal
end
+
@prev_route = @current_record
end
+ config.after(:each_save) do |importer, route|
+ opposite_route_name = route.name.split(" > ").reverse.join(' > ')
+ opposite_route = Chouette::Route.where(name: opposite_route_name).where('id < ?', route.id).last
+ if opposite_route && opposite_route.line == route.line
+ route.update_attribute :wayback, :inbound
+ opposite_route.update_attribute :wayback, :outbound
+ route.update_attribute :opposite_route_id, opposite_route.id
+ opposite_route.update_attribute :opposite_route_id, route.id
+ end
+ end
+
config.after do |importer|
prev_route = importer.instance_variable_get "@prev_route"
- if prev_route
+ if prev_route && prev_route.valid?
journey_pattern = prev_route.full_journey_pattern
- journey_pattern.set_distances importer.instance_variable_get("@distances")
+ importer.fail_with_error "WRONG DISTANCES FOR ROUTE #{prev_route.name}: #{importer.instance_variable_get("@distances").count} distances for #{prev_route.stop_points.count} stops" do
+ journey_pattern.set_distances importer.instance_variable_get("@distances")
+ journey_pattern.stop_points = prev_route.stop_points
+ end
importer.fail_with_error ->(){ journey_pattern.errors.messages } do
journey_pattern.save!
end
@@ -287,13 +335,15 @@ RSpec.describe SimpleImporter do
routes_count = Chouette::Route.count
journey_pattern_count = Chouette::JourneyPattern.count
stop_areas_count = Chouette::StopArea.count
- expect{importer.import(verbose: false)}.to change{Chouette::StopPoint.count}.by 20
+
+ expect{importer.import(verbose: true)}.to change{Chouette::StopPoint.count}.by 10
expect(importer.status).to eq "success"
expect(Chouette::Route.count).to eq routes_count + 2
expect(Chouette::JourneyPattern.count).to eq journey_pattern_count + 2
expect(Chouette::StopArea.count).to eq stop_areas_count + 5
route = Chouette::Route.find_by number: 1136
expect(route.stop_areas.count).to eq 5
+ expect(route.opposite_route).to eq Chouette::Route.find_by(number: 1137)
journey_pattern = route.full_journey_pattern
expect(journey_pattern.stop_areas.count).to eq 5
start, stop = journey_pattern.stop_points[0..1]
@@ -306,6 +356,7 @@ RSpec.describe SimpleImporter do
expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 177
route = Chouette::Route.find_by number: 1137
+ expect(route.opposite_route).to eq Chouette::Route.find_by(number: 1136)
expect(route.stop_areas.count).to eq 5
journey_pattern = route.full_journey_pattern
expect(journey_pattern.stop_areas.count).to eq 5
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 12:25:00 +0000