Refs #5924 @14h; Extend importers

Mostly add a way to override the default behaviour and process each row
its own way

10 files changed, 335 insertions, 102 deletions

diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb
index b3d40ab96..2e942d520 100644
--- a/app/models/chouette/company.rb
+++ b/app/models/chouette/company.rb
@@ -15,6 +15,7 @@ module Chouette
       [:organizational_unit, :operating_department_name, :code, :phone, :fax, :email, :url, :time_zone]
     end
 
+    def local_id; id end
 
   end
 end
diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb
index 874353752..5df45efbd 100644
--- a/app/models/chouette/line.rb
+++ b/app/models/chouette/line.rb
@@ -49,7 +49,7 @@ module Chouette
       [:published_name, :number, :comment, :url, :color, :text_color, :stable_id]
     end
 
-    def local_id; registration_number end
+    def local_id; registration_number || id end
 
     def geometry_presenter
       Chouette::Geometry::LinePresenter.new self
@@ -83,6 +83,14 @@ module Chouette
       line_referential.companies.where(id: ([company_id] + Array(secondary_company_ids)).compact)
     end
 
+    def deactivate
+      self.deactivated = true
+    end
+
+    def activate
+      self.deactivated = false
+    end
+
     def deactivate!
       update_attribute :deactivated, true
     end
diff --git a/app/models/chouette/purchase_window.rb b/app/models/chouette/purchase_window.rb
index 334493015..157390a21 100644
--- a/app/models/chouette/purchase_window.rb
+++ b/app/models/chouette/purchase_window.rb
@@ -19,6 +19,7 @@ module Chouette
 
     scope :contains_date, ->(date) { where('date ? <@ any (date_ranges)', date) }
     scope :overlap_dates, ->(date_range) { where('daterange(?, ?) && any (date_ranges)', date_range.first, date_range.last + 1.day) }
+    scope :matching_dates, ->(date_range) { where('ARRAY[daterange(?, ?)] = date_ranges', date_range.first, date_range.last + 1.day) }
 
     def self.ransackable_scopes(auth_object = nil)
       [:contains_date]
diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb
index e418134de..3729deb7d 100644
--- a/app/models/chouette/route.rb
+++ b/app/models/chouette/route.rb
@@ -186,15 +186,9 @@ module Chouette
     end
 
     def full_journey_pattern
-      out = journey_patterns.find{|jp| jp.stop_points.count == self.stop_points.count }
-      unless out
-        out = journey_patterns.build name: self.name
-        self.stop_points.each do |sp|
-          out.stop_points.build stop_area: sp.stop_area, position: sp.position
-        end
-        out.save!
-      end
-      out
+      journey_pattern = journey_patterns.find_or_create_by registration_number: self.number, name: self.name
+      journey_pattern.stop_points = self.stop_points
+      journey_pattern
     end
 
     protected
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index c85a8b37d..699616863 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -112,7 +112,6 @@ module Chouette
         id.to_s
       end
     end
-    alias_method :local_id, :user_objectid
 
     alias_method :local_id, :user_objectid
 
diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb
index 15b22b671..b76de852a 100644
--- a/app/models/chouette/time_table.rb
+++ b/app/models/chouette/time_table.rb
@@ -44,10 +44,10 @@ module Chouette
         attrs << self.int_day_types
         dates = self.dates
         dates += TimeTableDate.where(time_table_id: self.id)
-        attrs << dates.map(&:checksum).map(&:to_s).sort
+        attrs << dates.map(&:checksum).map(&:to_s).uniq.sort
         periods = self.periods
         periods += TimeTablePeriod.where(time_table_id: self.id)
-        attrs << periods.map(&:checksum).map(&:to_s).sort
+        attrs << periods.map(&:checksum).map(&:to_s).uniq.sort
       end
     end
 
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index bf6f3b406..dea8f85ad 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -30,7 +30,7 @@ class SimpleImporter < ActiveRecord::Base
   end
 
   def context
-    self.configuration.context || {}
+    self.configuration.context
   end
 
   def resolve col_name, value, &block
@@ -42,99 +42,183 @@ class SimpleImporter < ActiveRecord::Base
 
   def import opts={}
     @verbose = opts.delete :verbose
+
+
     @resolution_queue = Hash.new{|h,k| h[k] = []}
-    number_of_lines = 0
-    padding = 1
+    @errors = []
+    @messages = []
+    @number_of_lines = 0
+    @padding = 1
+    @current_line = 0
     fail_with_error "File not found: #{self.filepath}" do
-      number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
-      padding = [1, Math.log(number_of_lines, 10).ceil()].max
+      @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
+      @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+    end
+
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_csv_file
+    else
+      ActiveRecord::Base.transaction do
+        process_csv_file
+      end
+    end
+    self.status ||= :success
+  rescue FailedImport
+    self.status = :failed
+  ensure
+    self.save!
+  end
+
+  def fail_with_error msg=nil, opts={}
+    begin
+      yield
+    rescue => e
+      msg = msg.call if msg.is_a?(Proc)
+      custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
+      push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
+      @new_status = colorize("x", :red)
+      if self.configuration.ignore_failures
+        raise FailedRow if opts[:abort_row]
+      else
+        raise FailedImport
+      end
     end
+  end
+
+  def encode_string s
+    s.encode("utf-8").force_encoding("utf-8")
+  end
+
+  def dump_csv_from_context
+    filepath = "./#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
+    # for some reason, context[:csv].to_csv does not work
+    CSV.open(filepath, 'w') do |csv|
+      header = true
+      context[:csv].each do |row|
+        csv << row.headers if header
+        csv << row.fields
+        header = false
+      end
+    end
+    log "CSV file dumped in #{filepath}"
+  end
+
+  def log msg, opts={}
+    msg = colorize msg, opts[:color] if opts[:color]
+    if opts[:append]
+      @messages[-1] = (@messages[-1] || "") + msg
+    else
+      @messages << msg
+    end
+    print_state
+  end
 
-    current_line = 0
-    status = :success
-    statuses = ""
-    log "#{"%#{padding}d" % 0}/#{number_of_lines}", clear: true
-    ActiveRecord::Base.transaction do
-      self.configuration.before_actions(:all).each do |action| action.call self end
+  protected
 
-      CSV.foreach(filepath, self.configuration.csv_options) do |row|
-        status = handle_row row, status
+  def process_csv_file
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting import ...", color: :green
 
+    (context[:csv] || CSV.read(filepath, self.configuration.csv_options)).each do |row|
+      @current_row = row
+      @new_status = nil
+      begin
+        handle_row row
         fail_with_error ->(){ @current_record.errors.messages } do
           new_record = @current_record.new_record?
+          @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange)
+          @event = new_record ? :creation : :update
           self.configuration.before_actions(:each_save).each do |action|
             action.call self, @current_record
           end
           ### This could fail if the record has a mandatory relation which is not yet resolved
           ### TODO: do not attempt to save if the current record if waiting for resolution
           ###       and fail at the end if there remains unresolved relations
-          @current_record.save!
-          self.journal.push({event: (new_record ? :creation : :update), kind: :log})
-          statuses += new_record ? colorize("✓", :green) : colorize("-", :orange)
-        end
-        self.configuration.columns.each do |col|
-          if col.name && @resolution_queue.any?
-            val = @current_record.send col[:attribute]
-            (@resolution_queue.delete([col.name, val]) || []).each do |res|
-              record = res[:record]
-              attribute = res[:attribute]
-              value = res[:block].call(val, record)
-              record.send "#{attribute}=", value
-              record.save!
+          if self.configuration.ignore_failures
+            unless @current_record.save
+              @new_status = colorize("x", :red)
+              push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
             end
+          else
+            @current_record.save!
+          end
+          self.configuration.after_actions(:each_save).each do |action|
+            action.call self, @current_record
           end
         end
-        current_line += 1
-        log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
+      rescue FailedRow
+        @new_status = colorize("x", :red)
       end
-      self.configuration.after_actions(:all).each do |action|
-        action.call self
+      push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
+      @statuses += @new_status
+      self.configuration.columns.each do |col|
+      if col.name && @resolution_queue.any?
+        val = @current_record.send col[:attribute]
+        (@resolution_queue.delete([col.name, val]) || []).each do |res|
+          record = res[:record]
+          attribute = res[:attribute]
+          value = res[:block].call(val, record)
+          record.send "#{attribute}=", value
+          record.save!
+        end
       end
     end
-    self.update_attribute :status, status
-  rescue FailedImport
-    self.update_attribute :status, :failed
-  ensure
-    self.save!
-  end
+      print_state
+      @current_line += 1
+    end
 
-  def fail_with_error msg
     begin
-      yield
-    rescue => e
-      msg = msg.call if msg.is_a?(Proc)
-      log "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red
-      self.journal.push({message: msg, error: e.message, event: :error, kind: :error})
-      raise FailedImport
+      self.configuration.after_actions(:all).each do |action|
+        action.call self
+      end
+    rescue FailedRow
     end
   end
 
-  protected
-
-  def handle_row row, status
+  def handle_row row
     if self.configuration.get_custom_handler
       instance_exec(row, &self.configuration.get_custom_handler)
     else
-      @current_record = self.configuration.find_record row
-      self.configuration.columns.each do |col|
-        @current_attribute = col[:attribute]
-        val = col[:value]
-        if val.nil? || val.is_a?(Proc)
-          if row.has_key? col.name
-            if val.is_a?(Proc)
-              val = instance_exec(row[col.name], &val)
+      fail_with_error "", abort_row: true do
+        @current_record = self.configuration.find_record row
+        self.configuration.columns.each do |col|
+          @current_attribute = col[:attribute]
+          val = col[:value]
+          if val.nil? || val.is_a?(Proc)
+            if row.has_key? col.name
+              if val.is_a?(Proc)
+                val = instance_exec(row[col.name], &val)
+              else
+                val = row[col.name]
+              end
             else
-              val = row[col.name]
+              push_in_journal({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+              self.status ||= :success_with_warnings
             end
-          else
-            self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
-            status = :success_with_warnings
           end
+
+          if val.nil? && col.required?
+            raise "MISSING VALUE FOR COLUMN #{col.name}"
+          end
+          val = encode_string(val) if val.is_a?(String)
+          @current_record.send "#{@current_attribute}=", val if val
         end
-        @current_record.send "#{@current_attribute}=", val if val
       end
     end
-    status
+  end
+
+  def push_in_journal data
+    line = @current_line + 1
+    line += 1 if configuration.headers
+    self.journal.push data.update(line: line, row: @current_row)
+    if data[:kind] == :error || data[:kind] == :warning
+      @errors.push data
+    end
   end
 
   def colorize txt, color
@@ -146,23 +230,59 @@ class SimpleImporter < ActiveRecord::Base
     "\e[#{color}m#{txt}\e[0m"
   end
 
-  def log msg, opts={}
+  def print_state
+    return unless @verbose
+
+    @status_width ||= begin
+      term_width = %x(tput cols).to_i
+      term_width - @padding - 10
+    rescue
+      100
+    end
+    full_status = @statuses || ""
+    full_status = full_status.last(@status_width*10) || ""
+    padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
+    full_status = "#{full_status}#{"."*[padding_size, 0].max}"
+
+    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+
+    if @messages.any?
+      msg += "\n\n"
+      msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
+      msg += "[...]\n" if @messages.count > 10
+      msg += @messages.last(10).join("\n")
+    end
+
+    if @errors.any?
+      msg += "\n\n"
+      msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
+      msg += "[...]\n" if @errors.count > 10
+      msg += @errors.last(10).map do |j|
+        kind = j[:kind]
+        kind = colorize(kind, kind == :error ? :red : :orange)
+        encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
+      end.join("\n")
+    end
+    custom_print msg, clear: true
+  end
+
+  def custom_print msg, opts={}
     return unless @verbose
     out = ""
     msg = colorize(msg, opts[:color]) if opts[:color]
-    if opts[:clear] && @prev_msg_size
-      out += "\b"*@prev_msg_size
-    end
+    puts "\e[H\e[2J" if opts[:clear]
     out += msg
     print out
-    @prev_msg_size = msg.size
   end
 
   class FailedImport < RuntimeError
   end
 
+  class FailedRow < RuntimeError
+  end
+
   class Configuration
-    attr_accessor :model, :headers, :separator, :key, :context, :encoding
+    attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
     attr_reader :columns
 
     def initialize import_name, opts={}
@@ -174,6 +294,11 @@ class SimpleImporter < ActiveRecord::Base
       @columns = opts[:columns] || []
       @model = opts[:model]
       @custom_handler = opts[:custom_handler]
+      @before = opts[:before]
+      @after = opts[:after]
+      @ignore_failures = opts[:ignore_failures]
+      @context = opts[:context] || {}
+      @scope = opts[:scope] || {}
     end
 
     def duplicate
@@ -188,7 +313,12 @@ class SimpleImporter < ActiveRecord::Base
         encoding: @encoding,
         columns: @columns.map(&:duplicate),
         model: model,
-        custom_handler: @custom_handler
+        custom_handler: @custom_handler,
+        before: @before,
+        after: @after,
+        ignore_failures: @ignore_failures,
+        context: @context,
+        scope: @scope
       }
     end
 
@@ -201,8 +331,14 @@ class SimpleImporter < ActiveRecord::Base
       column && column[:attribute] || col_name
     end
 
+    def record_scope
+      _scope = @scope
+      _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
+      _scope || model
+    end
+
     def find_record attrs
-      model.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+      record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
     end
 
     def csv_options
@@ -261,6 +397,10 @@ class SimpleImporter < ActiveRecord::Base
         Column.new @options.dup
       end
 
+      def required?
+        !!@options[:required]
+      end
+
       def [](key)
         @options[key]
       end
diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake
index 9c6b1fbcd..eca7e6849 100644
--- a/lib/tasks/imports.rake
+++ b/lib/tasks/imports.rake
@@ -1,3 +1,5 @@
+require 'csv'
+
 namespace :import do
   desc "Notify parent imports when children finish"
   task notify_parent: :environment do
@@ -9,12 +11,35 @@ namespace :import do
     NetexImport.abort_old
   end
 
+  def importer_output_to_csv importer
+    filepath = "./#{importer.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+    cols = %w(line kind event message error)
+    if importer.reload.journal.size > 0
+      keys = importer.journal.first["row"].map(&:first)
+      CSV.open(filepath, "w") do |csv|
+        csv << cols + keys
+        importer.journal.each do |j|
+          csv << cols.map{|c| j[c]} + j["row"].map(&:last)
+        end
+      end
+      puts "Import Output written in #{filepath}"
+    end
+  end
+
   desc "import the given file with the corresponding importer"
-  task :import, [:configuration_name, :filepath] => :environment do |t, args|
+  task :import, [:configuration_name, :filepath, :referential_id] => :environment do |t, args|
     importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath]
+    if args[:referential_id].present?
+      referential = Referential.find args[:referential_id]
+      importer.configure do |config|
+        config.add_value :referential, referential
+        config.context = {referential: referential}
+      end
+    end
     puts "\e[33m***\e[0m Start importing"
     importer.import(verbose: true)
     puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+    importer_output_to_csv importer
   end
 
   desc "import the given file with the corresponding importer in the given StopAreaReferential"
@@ -28,21 +53,23 @@ namespace :import do
     puts "\e[33m***\e[0m Start importing"
     importer.import(verbose: true)
     puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+    importer_output_to_csv importer
   end
 
-  desc "import the given file with the corresponding importer in the given Referential and StopAreaReferential"
-  task :import_in_referential_and_stop_area_referential, [:referential_id, :stop_area_referential_id, :configuration_name, :filepath] => :environment do |t, args|
+  desc "import the given routes files"
+  task :import_routes, [:referential_id, :configuration_name, :mapping_filepath, :filepath] => :environment do |t, args|
     referential = Referential.find args[:referential_id]
     referential.switch
-    stop_area_referential = StopAreaReferential.find args[:stop_area_referential_id]
+    stop_area_referential = referential.stop_area_referential
     importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath]
     importer.configure do |config|
       config.add_value :stop_area_referential, referential
-      config.context = {stop_area_referential: stop_area_referential}
+      config.context = {stop_area_referential: stop_area_referential, mapping_filepath: args[:mapping_filepath]}
     end
     puts "\e[33m***\e[0m Start importing"
     importer.import(verbose: true)
     puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+    importer_output_to_csv importer
   end
 
   desc "import the given file with the corresponding importer in the given LineReferential"
@@ -56,5 +83,6 @@ namespace :import do
     puts "\e[33m***\e[0m Start importing"
     importer.import(verbose: true)
     puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+    importer_output_to_csv importer
   end
 end
diff --git a/spec/fixtures/simple_importer/lines_mapping.csv b/spec/fixtures/simple_importer/lines_mapping.csv
new file mode 100644
index 000000000..b26d0ab59
--- /dev/null
+++ b/spec/fixtures/simple_importer/lines_mapping.csv
@@ -0,0 +1,11 @@
+id;timetable_route_id;route_name;stop_sequence;stop_distance;station_code;station_name;border;Ligne Chouette;Transporteur

+3354;1136;Paris centre - Bercy > Lille > Londres;1;0;XPB;Paris City Center - Bercy;f;Paris <> Londres - OUIBUS;OUIBUS

+3355;1136;Paris centre - Bercy > Lille > Londres;2;232;XDB;Lille;f;Paris <> Londres - OUIBUS;OUIBUS

+3749;1136;Paris centre - Bercy > Lille > Londres;3;350;COF;Coquelles - France;t;Paris <> Londres - OUIBUS;OUIBUS

+4772;1136;Paris centre - Bercy > Lille > Londres;4;350;COU;Coquelles - UK;t;Paris <> Londres - OUIBUS;OUIBUS

+3357;1136;Paris centre - Bercy > Lille > Londres;5;527;ZEP;London;f;Paris <> Londres - OUIBUS;OUIBUS

+3358;1137;Londres > Lille > Paris centre - Bercy;1;0;ZEP;London;f;Paris <> Londres - OUIBUS;OUIBUS

+3559;1137;Londres > Lille > Paris centre - Bercy;2;177;COU;Coquelles - UK;t;Paris <> Londres - OUIBUS;OUIBUS

+3743;1137;Londres > Lille > Paris centre - Bercy;3;177;COF;Coquelles - France;t;Paris <> Londres - OUIBUS;OUIBUS

+3360;1137;Londres > Lille > Paris centre - Bercy;4;295;XDB;Lille;f;Paris <> Londres - OUIBUS;OUIBUS

+3361;1137;Londres > Lille > Paris centre - Bercy;5;527;XPB;Paris City Center - Bercy;f;Paris <> Londres - OUIBUS;OUIBUS

diff --git a/spec/models/simple_importer_spec.rb b/spec/models/simple_importer_spec.rb
index e324399c7..2f520a3f3 100644
--- a/spec/models/simple_importer_spec.rb
+++ b/spec/models/simple_importer_spec.rb
@@ -27,7 +27,7 @@ RSpec.describe SimpleImporter do
 
   describe "#import" do
     let(:importer){ importer = SimpleImporter.new(configuration_name: :test, filepath: filepath) }
-    let(:filepath){ Rails.root + "spec/fixtures/simple_importer/#{filename}" }
+    let(:filepath){ fixtures_path 'simple_importer', filename }
     let(:filename){ "stop_area.csv" }
     let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) }
 
@@ -47,7 +47,7 @@ RSpec.describe SimpleImporter do
     end
 
     it "should import the given file" do
-      expect{importer.import}.to change{Chouette::StopArea.count}.by 1
+      expect{importer.import verbose: false}.to change{Chouette::StopArea.count}.by 1
       expect(importer.status).to eq "success"
       stop = Chouette::StopArea.last
       expect(stop.name).to eq "Nom du Stop"
@@ -90,6 +90,21 @@ RSpec.describe SimpleImporter do
         expect(stop.area_type).to eq "zdep"
         expect(importer.reload.journal.last["event"]).to eq("update")
       end
+
+      context "in another scope" do
+        before(:each) do
+          ref = create(:stop_area_referential)
+          importer.configure do |config|
+            config.context = { stop_area_referential: ref }
+            config.scope = ->{ context[:stop_area_referential].stop_areas }
+          end
+        end
+
+        it "should create the record" do
+          expect{importer.import verbose: true}.to change{Chouette::StopArea.count}.by 1
+          expect(importer.status).to eq "success"
+        end
+      end
     end
 
     context "with a missing column" do
@@ -101,7 +116,7 @@ RSpec.describe SimpleImporter do
       end
     end
 
-    context "with a incomplete dataset" do
+    context "with an incomplete dataset" do
       let(:filename){ "stop_area_incomplete.csv" }
       it "should fail" do
         expect{importer.import}.to_not raise_error
@@ -116,7 +131,7 @@ RSpec.describe SimpleImporter do
 
     context "with a wrong filepath" do
       let(:filename){ "not_found.csv" }
-      it "should create a StopArea" do
+      it "should fail" do
         expect{importer.import}.to_not raise_error
         expect(importer.status).to eq "failed"
         expect(importer.reload.journal.first["message"]).to eq "File not found: #{importer.filepath}"
@@ -221,27 +236,42 @@ RSpec.describe SimpleImporter do
       let(:filename){ "stop_points_full.csv" }
 
       before(:each) do
-        create :line, name: "Paris centre - Bercy > Lille > Londres"
-        create :line, name: "Londres > Lille > Paris centre - Bercy"
+        create :line, name: "Paris <> Londres - OUIBUS"
 
         SimpleImporter.define :test do |config|
           config.model = Chouette::Route
           config.separator = ";"
           config.context = {stop_area_referential: stop_area_referential}
+
+          config.before do |importer|
+            mapping = {}
+            path = Rails.root + "spec/fixtures/simple_importer/lines_mapping.csv"
+            CSV.foreach(path, importer.configuration.csv_options) do |row|
+              if row["Ligne Chouette"].present?
+                mapping[row["timetable_route_id"]] ||= Chouette::Line.find_by(name: importer.encode_string(row["Ligne Chouette"]))
+              end
+            end
+            importer.context[:mapping] = mapping
+          end
+
           config.custom_handler do |row|
             line = nil
-            fail_with_error "MISSING LINE: #{row["route_name"]}" do
-              line = Chouette::Line.find_by! name: row["route_name"]
+            fail_with_error "MISSING LINE FOR ROUTE: #{encode_string row["route_name"]}" do
+              line = context[:mapping][row["timetable_route_id"]]
+              raise unless line
             end
             @current_record = Chouette::Route.find_or_initialize_by number: row["timetable_route_id"]
-            @current_record.name = row["route_name"]
-            @current_record.published_name = row["route_name"]
+            @current_record.name = encode_string row["route_name"]
+            @current_record.published_name = encode_string row["route_name"]
 
             @current_record.line = line
             if @prev_route != @current_record
-              if @prev_route
+              if @prev_route && @prev_route.valid?
                 journey_pattern = @prev_route.full_journey_pattern
-                journey_pattern.set_distances @distances
+                fail_with_error "WRONG DISTANCES FOR ROUTE #{@prev_route.name} (#{@prev_route.number}): #{@distances.count} distances for #{@prev_route.stop_points.count} stops" do
+                  journey_pattern.stop_points = @prev_route.stop_points
+                  journey_pattern.set_distances @distances
+                end
                 fail_with_error ->(){ journey_pattern.errors.messages } do
                   journey_pattern.save!
                 end
@@ -250,6 +280,7 @@ RSpec.describe SimpleImporter do
             end
             @distances.push row["stop_distance"]
             position = row["stop_sequence"].to_i - 1
+
             stop_area = context[:stop_area_referential].stop_areas.where(registration_number: row["station_code"]).last
             unless stop_area
               stop_area = Chouette::StopArea.new registration_number: row["station_code"]
@@ -257,7 +288,7 @@ RSpec.describe SimpleImporter do
               stop_area.kind = row["border"] == "f" ? :commercial : :non_commercial
               stop_area.area_type = row["border"] == "f" ? :zdep : :border
               stop_area.stop_area_referential = context[:stop_area_referential]
-              fail_with_error ->{ stop_area.errors.messages } do
+              fail_with_error ->{p stop_area; "UNABLE TO CREATE STOP_AREA: #{stop_area.errors.messages}" }, abort_row: true do
                 stop_area.save!
               end
             end
@@ -266,15 +297,32 @@ RSpec.describe SimpleImporter do
               stop_point.set_list_position position
             else
               stop_point = @current_record.stop_points.build(stop_area_id: stop_area.id, position: position)
+              stop_point.for_boarding = :normal
+              stop_point.for_alighting = :normal
             end
+
             @prev_route = @current_record
           end
 
+          config.after(:each_save) do |importer, route|
+            opposite_route_name = route.name.split(" > ").reverse.join(' > ')
+            opposite_route = Chouette::Route.where(name: opposite_route_name).where('id < ?', route.id).last
+            if opposite_route && opposite_route.line == route.line
+              route.update_attribute :wayback, :inbound
+              opposite_route.update_attribute :wayback, :outbound
+              route.update_attribute :opposite_route_id, opposite_route.id
+              opposite_route.update_attribute :opposite_route_id, route.id
+            end
+          end
+
           config.after do |importer|
             prev_route = importer.instance_variable_get "@prev_route"
-            if prev_route
+            if prev_route && prev_route.valid?
               journey_pattern = prev_route.full_journey_pattern
-              journey_pattern.set_distances importer.instance_variable_get("@distances")
+              importer.fail_with_error "WRONG DISTANCES FOR ROUTE #{prev_route.name}: #{importer.instance_variable_get("@distances").count} distances for #{prev_route.stop_points.count} stops" do
+                journey_pattern.set_distances importer.instance_variable_get("@distances")
+                journey_pattern.stop_points = prev_route.stop_points
+              end
               importer.fail_with_error ->(){ journey_pattern.errors.messages } do
                 journey_pattern.save!
               end
@@ -287,13 +335,15 @@ RSpec.describe SimpleImporter do
         routes_count = Chouette::Route.count
         journey_pattern_count = Chouette::JourneyPattern.count
         stop_areas_count = Chouette::StopArea.count
-        expect{importer.import(verbose: false)}.to change{Chouette::StopPoint.count}.by 20
+
+        expect{importer.import(verbose: true)}.to change{Chouette::StopPoint.count}.by 10
         expect(importer.status).to eq "success"
         expect(Chouette::Route.count).to eq routes_count + 2
         expect(Chouette::JourneyPattern.count).to eq journey_pattern_count + 2
         expect(Chouette::StopArea.count).to eq stop_areas_count + 5
         route = Chouette::Route.find_by number: 1136
         expect(route.stop_areas.count).to eq 5
+        expect(route.opposite_route).to eq Chouette::Route.find_by(number: 1137)
         journey_pattern = route.full_journey_pattern
         expect(journey_pattern.stop_areas.count).to eq 5
         start, stop = journey_pattern.stop_points[0..1]
@@ -306,6 +356,7 @@ RSpec.describe SimpleImporter do
         expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 177
 
         route = Chouette::Route.find_by number: 1137
+        expect(route.opposite_route).to eq Chouette::Route.find_by(number: 1136)
         expect(route.stop_areas.count).to eq 5
         journey_pattern = route.full_journey_pattern
         expect(journey_pattern.stop_areas.count).to eq 5