Refs #5924 @14h; Extend importers

Mostly add a way to override the default behaviour and process each row
its own way

7 files changed, 224 insertions, 81 deletions

diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb
index b3d40ab96..2e942d520 100644
--- a/app/models/chouette/company.rb
+++ b/app/models/chouette/company.rb
@@ -15,6 +15,7 @@ module Chouette
       [:organizational_unit, :operating_department_name, :code, :phone, :fax, :email, :url, :time_zone]
     end
 
+    def local_id; id end
 
   end
 end
diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb
index 874353752..5df45efbd 100644
--- a/app/models/chouette/line.rb
+++ b/app/models/chouette/line.rb
@@ -49,7 +49,7 @@ module Chouette
       [:published_name, :number, :comment, :url, :color, :text_color, :stable_id]
     end
 
-    def local_id; registration_number end
+    def local_id; registration_number || id end
 
     def geometry_presenter
       Chouette::Geometry::LinePresenter.new self
@@ -83,6 +83,14 @@ module Chouette
       line_referential.companies.where(id: ([company_id] + Array(secondary_company_ids)).compact)
     end
 
+    def deactivate
+      self.deactivated = true
+    end
+
+    def activate
+      self.deactivated = false
+    end
+
     def deactivate!
       update_attribute :deactivated, true
     end
diff --git a/app/models/chouette/purchase_window.rb b/app/models/chouette/purchase_window.rb
index 334493015..157390a21 100644
--- a/app/models/chouette/purchase_window.rb
+++ b/app/models/chouette/purchase_window.rb
@@ -19,6 +19,7 @@ module Chouette
 
     scope :contains_date, ->(date) { where('date ? <@ any (date_ranges)', date) }
     scope :overlap_dates, ->(date_range) { where('daterange(?, ?) && any (date_ranges)', date_range.first, date_range.last + 1.day) }
+    scope :matching_dates, ->(date_range) { where('ARRAY[daterange(?, ?)] = date_ranges', date_range.first, date_range.last + 1.day) }
 
     def self.ransackable_scopes(auth_object = nil)
       [:contains_date]
diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb
index e418134de..3729deb7d 100644
--- a/app/models/chouette/route.rb
+++ b/app/models/chouette/route.rb
@@ -186,15 +186,9 @@ module Chouette
     end
 
     def full_journey_pattern
-      out = journey_patterns.find{|jp| jp.stop_points.count == self.stop_points.count }
-      unless out
-        out = journey_patterns.build name: self.name
-        self.stop_points.each do |sp|
-          out.stop_points.build stop_area: sp.stop_area, position: sp.position
-        end
-        out.save!
-      end
-      out
+      journey_pattern = journey_patterns.find_or_create_by registration_number: self.number, name: self.name
+      journey_pattern.stop_points = self.stop_points
+      journey_pattern
     end
 
     protected
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index c85a8b37d..699616863 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -112,7 +112,6 @@ module Chouette
         id.to_s
       end
     end
-    alias_method :local_id, :user_objectid
 
     alias_method :local_id, :user_objectid
 
diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb
index 15b22b671..b76de852a 100644
--- a/app/models/chouette/time_table.rb
+++ b/app/models/chouette/time_table.rb
@@ -44,10 +44,10 @@ module Chouette
         attrs << self.int_day_types
         dates = self.dates
         dates += TimeTableDate.where(time_table_id: self.id)
-        attrs << dates.map(&:checksum).map(&:to_s).sort
+        attrs << dates.map(&:checksum).map(&:to_s).uniq.sort
         periods = self.periods
         periods += TimeTablePeriod.where(time_table_id: self.id)
-        attrs << periods.map(&:checksum).map(&:to_s).sort
+        attrs << periods.map(&:checksum).map(&:to_s).uniq.sort
       end
     end
 
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index bf6f3b406..dea8f85ad 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -30,7 +30,7 @@ class SimpleImporter < ActiveRecord::Base
   end
 
   def context
-    self.configuration.context || {}
+    self.configuration.context
   end
 
   def resolve col_name, value, &block
@@ -42,99 +42,183 @@ class SimpleImporter < ActiveRecord::Base
 
   def import opts={}
     @verbose = opts.delete :verbose
+
+
     @resolution_queue = Hash.new{|h,k| h[k] = []}
-    number_of_lines = 0
-    padding = 1
+    @errors = []
+    @messages = []
+    @number_of_lines = 0
+    @padding = 1
+    @current_line = 0
     fail_with_error "File not found: #{self.filepath}" do
-      number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
-      padding = [1, Math.log(number_of_lines, 10).ceil()].max
+      @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
+      @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+    end
+
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_csv_file
+    else
+      ActiveRecord::Base.transaction do
+        process_csv_file
+      end
+    end
+    self.status ||= :success
+  rescue FailedImport
+    self.status = :failed
+  ensure
+    self.save!
+  end
+
+  def fail_with_error msg=nil, opts={}
+    begin
+      yield
+    rescue => e
+      msg = msg.call if msg.is_a?(Proc)
+      custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
+      push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
+      @new_status = colorize("x", :red)
+      if self.configuration.ignore_failures
+        raise FailedRow if opts[:abort_row]
+      else
+        raise FailedImport
+      end
     end
+  end
+
+  def encode_string s
+    s.encode("utf-8").force_encoding("utf-8")
+  end
+
+  def dump_csv_from_context
+    filepath = "./#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
+    # for some reason, context[:csv].to_csv does not work
+    CSV.open(filepath, 'w') do |csv|
+      header = true
+      context[:csv].each do |row|
+        csv << row.headers if header
+        csv << row.fields
+        header = false
+      end
+    end
+    log "CSV file dumped in #{filepath}"
+  end
+
+  def log msg, opts={}
+    msg = colorize msg, opts[:color] if opts[:color]
+    if opts[:append]
+      @messages[-1] = (@messages[-1] || "") + msg
+    else
+      @messages << msg
+    end
+    print_state
+  end
 
-    current_line = 0
-    status = :success
-    statuses = ""
-    log "#{"%#{padding}d" % 0}/#{number_of_lines}", clear: true
-    ActiveRecord::Base.transaction do
-      self.configuration.before_actions(:all).each do |action| action.call self end
+  protected
 
-      CSV.foreach(filepath, self.configuration.csv_options) do |row|
-        status = handle_row row, status
+  def process_csv_file
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting import ...", color: :green
 
+    (context[:csv] || CSV.read(filepath, self.configuration.csv_options)).each do |row|
+      @current_row = row
+      @new_status = nil
+      begin
+        handle_row row
         fail_with_error ->(){ @current_record.errors.messages } do
           new_record = @current_record.new_record?
+          @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange)
+          @event = new_record ? :creation : :update
           self.configuration.before_actions(:each_save).each do |action|
             action.call self, @current_record
           end
           ### This could fail if the record has a mandatory relation which is not yet resolved
           ### TODO: do not attempt to save if the current record if waiting for resolution
           ###       and fail at the end if there remains unresolved relations
-          @current_record.save!
-          self.journal.push({event: (new_record ? :creation : :update), kind: :log})
-          statuses += new_record ? colorize("✓", :green) : colorize("-", :orange)
-        end
-        self.configuration.columns.each do |col|
-          if col.name && @resolution_queue.any?
-            val = @current_record.send col[:attribute]
-            (@resolution_queue.delete([col.name, val]) || []).each do |res|
-              record = res[:record]
-              attribute = res[:attribute]
-              value = res[:block].call(val, record)
-              record.send "#{attribute}=", value
-              record.save!
+          if self.configuration.ignore_failures
+            unless @current_record.save
+              @new_status = colorize("x", :red)
+              push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
             end
+          else
+            @current_record.save!
+          end
+          self.configuration.after_actions(:each_save).each do |action|
+            action.call self, @current_record
           end
         end
-        current_line += 1
-        log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
+      rescue FailedRow
+        @new_status = colorize("x", :red)
       end
-      self.configuration.after_actions(:all).each do |action|
-        action.call self
+      push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
+      @statuses += @new_status
+      self.configuration.columns.each do |col|
+      if col.name && @resolution_queue.any?
+        val = @current_record.send col[:attribute]
+        (@resolution_queue.delete([col.name, val]) || []).each do |res|
+          record = res[:record]
+          attribute = res[:attribute]
+          value = res[:block].call(val, record)
+          record.send "#{attribute}=", value
+          record.save!
+        end
       end
     end
-    self.update_attribute :status, status
-  rescue FailedImport
-    self.update_attribute :status, :failed
-  ensure
-    self.save!
-  end
+      print_state
+      @current_line += 1
+    end
 
-  def fail_with_error msg
     begin
-      yield
-    rescue => e
-      msg = msg.call if msg.is_a?(Proc)
-      log "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red
-      self.journal.push({message: msg, error: e.message, event: :error, kind: :error})
-      raise FailedImport
+      self.configuration.after_actions(:all).each do |action|
+        action.call self
+      end
+    rescue FailedRow
     end
   end
 
-  protected
-
-  def handle_row row, status
+  def handle_row row
     if self.configuration.get_custom_handler
       instance_exec(row, &self.configuration.get_custom_handler)
     else
-      @current_record = self.configuration.find_record row
-      self.configuration.columns.each do |col|
-        @current_attribute = col[:attribute]
-        val = col[:value]
-        if val.nil? || val.is_a?(Proc)
-          if row.has_key? col.name
-            if val.is_a?(Proc)
-              val = instance_exec(row[col.name], &val)
+      fail_with_error "", abort_row: true do
+        @current_record = self.configuration.find_record row
+        self.configuration.columns.each do |col|
+          @current_attribute = col[:attribute]
+          val = col[:value]
+          if val.nil? || val.is_a?(Proc)
+            if row.has_key? col.name
+              if val.is_a?(Proc)
+                val = instance_exec(row[col.name], &val)
+              else
+                val = row[col.name]
+              end
             else
-              val = row[col.name]
+              push_in_journal({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+              self.status ||= :success_with_warnings
             end
-          else
-            self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
-            status = :success_with_warnings
           end
+
+          if val.nil? && col.required?
+            raise "MISSING VALUE FOR COLUMN #{col.name}"
+          end
+          val = encode_string(val) if val.is_a?(String)
+          @current_record.send "#{@current_attribute}=", val if val
         end
-        @current_record.send "#{@current_attribute}=", val if val
       end
     end
-    status
+  end
+
+  def push_in_journal data
+    line = @current_line + 1
+    line += 1 if configuration.headers
+    self.journal.push data.update(line: line, row: @current_row)
+    if data[:kind] == :error || data[:kind] == :warning
+      @errors.push data
+    end
   end
 
   def colorize txt, color
@@ -146,23 +230,59 @@ class SimpleImporter < ActiveRecord::Base
     "\e[#{color}m#{txt}\e[0m"
   end
 
-  def log msg, opts={}
+  def print_state
+    return unless @verbose
+
+    @status_width ||= begin
+      term_width = %x(tput cols).to_i
+      term_width - @padding - 10
+    rescue
+      100
+    end
+    full_status = @statuses || ""
+    full_status = full_status.last(@status_width*10) || ""
+    padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
+    full_status = "#{full_status}#{"."*[padding_size, 0].max}"
+
+    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+
+    if @messages.any?
+      msg += "\n\n"
+      msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
+      msg += "[...]\n" if @messages.count > 10
+      msg += @messages.last(10).join("\n")
+    end
+
+    if @errors.any?
+      msg += "\n\n"
+      msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
+      msg += "[...]\n" if @errors.count > 10
+      msg += @errors.last(10).map do |j|
+        kind = j[:kind]
+        kind = colorize(kind, kind == :error ? :red : :orange)
+        encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
+      end.join("\n")
+    end
+    custom_print msg, clear: true
+  end
+
+  def custom_print msg, opts={}
     return unless @verbose
     out = ""
     msg = colorize(msg, opts[:color]) if opts[:color]
-    if opts[:clear] && @prev_msg_size
-      out += "\b"*@prev_msg_size
-    end
+    puts "\e[H\e[2J" if opts[:clear]
     out += msg
     print out
-    @prev_msg_size = msg.size
   end
 
   class FailedImport < RuntimeError
   end
 
+  class FailedRow < RuntimeError
+  end
+
   class Configuration
-    attr_accessor :model, :headers, :separator, :key, :context, :encoding
+    attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
     attr_reader :columns
 
     def initialize import_name, opts={}
@@ -174,6 +294,11 @@ class SimpleImporter < ActiveRecord::Base
       @columns = opts[:columns] || []
       @model = opts[:model]
       @custom_handler = opts[:custom_handler]
+      @before = opts[:before]
+      @after = opts[:after]
+      @ignore_failures = opts[:ignore_failures]
+      @context = opts[:context] || {}
+      @scope = opts[:scope] || {}
     end
 
     def duplicate
@@ -188,7 +313,12 @@ class SimpleImporter < ActiveRecord::Base
         encoding: @encoding,
         columns: @columns.map(&:duplicate),
         model: model,
-        custom_handler: @custom_handler
+        custom_handler: @custom_handler,
+        before: @before,
+        after: @after,
+        ignore_failures: @ignore_failures,
+        context: @context,
+        scope: @scope
       }
     end
 
@@ -201,8 +331,14 @@ class SimpleImporter < ActiveRecord::Base
       column && column[:attribute] || col_name
     end
 
+    def record_scope
+      _scope = @scope
+      _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
+      _scope || model
+    end
+
     def find_record attrs
-      model.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+      record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
     end
 
     def csv_options
@@ -261,6 +397,10 @@ class SimpleImporter < ActiveRecord::Base
         Column.new @options.dup
       end
 
+      def required?
+        !!@options[:required]
+      end
+
       def [](key)
         @options[key]
       end