Merge pull request #359 from af83/6068-simple-exporter

Simple exporter. Refs #6068 

8 files changed, 731 insertions, 306 deletions

diff --git a/app/models/chouette/journey_pattern.rb b/app/models/chouette/journey_pattern.rb
index 2a636a041..ff85f376a 100644
--- a/app/models/chouette/journey_pattern.rb
+++ b/app/models/chouette/journey_pattern.rb
@@ -169,6 +169,19 @@ module Chouette
       full
     end
 
+    def distance_to stop
+      val = 0
+      i = 0
+      _end = stop_points.first
+      while _end != stop
+        i += 1
+        _start = _end
+        _end = stop_points[i]
+        val += costs_between(_start, _end)[:distance]
+      end
+      val
+    end
+
     def set_distances distances
       raise "inconsistent data: #{distances.count} values for #{stop_points.count} stops" unless distances.count == stop_points.count
       prev = distances[0].to_i
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index 7170dd217..f58f97eee 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -390,9 +390,12 @@ module Chouette
       ActiveSupport::TimeZone[time_zone]&.utc_offset
     end
 
-    def country_name
+    def country
       return unless country_code
       country = ISO3166::Country[country_code]
+    end
+
+    def country_name
       return unless country
       country.translations[I18n.locale.to_s] || country.name
     end
diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb
index 78cb6184c..60279422c 100644
--- a/app/models/chouette/vehicle_journey.rb
+++ b/app/models/chouette/vehicle_journey.rb
@@ -88,7 +88,6 @@ module Chouette
       end
     }
 
-
     scope :in_purchase_window, ->(range){
       purchase_windows = Chouette::PurchaseWindow.overlap_dates(range)
       sql = purchase_windows.joins(:vehicle_journeys).select('vehicle_journeys.id').uniq.to_sql
@@ -155,6 +154,14 @@ module Chouette
       end
     end
 
+    def sales_start
+      purchase_windows.map{|p| p.date_ranges.map &:first}.flatten.min
+    end
+
+    def sales_end
+      purchase_windows.map{|p| p.date_ranges.map &:last}.flatten.max
+    end
+
     def calculate_vehicle_journey_at_stop_day_offset
       Chouette::VehicleJourneyAtStopsDayOffset.new(
         vehicle_journey_at_stops
diff --git a/app/models/concerns/application_days_support.rb b/app/models/concerns/application_days_support.rb
index 348436aa4..2d00b5847 100644
--- a/app/models/concerns/application_days_support.rb
+++ b/app/models/concerns/application_days_support.rb
@@ -15,41 +15,34 @@ module ApplicationDaysSupport
   end
 
   def day_by_mask(flag)
-    int_day_types & flag == flag
+    self.class.day_by_mask int_day_types, flag
   end
 
-  def self.day_by_mask(int_day_types,flag)
-    int_day_types & flag == flag
+  def valid_day? wday
+    valid_days.include?(wday)
   end
 
-  def valid_days
-    # Build an array with day of calendar week (1-7, Monday is 1).
-    [].tap do |valid_days|
-      valid_days << 1  if monday
-      valid_days << 2  if tuesday
-      valid_days << 3  if wednesday
-      valid_days << 4  if thursday
-      valid_days << 5  if friday
-      valid_days << 6  if saturday
-      valid_days << 7  if sunday
+  included do
+    def self.valid_days(int_day_types)
+      # Build an array with day of calendar week (1-7, Monday is 1).
+      [].tap do |valid_days|
+        valid_days << 1  if day_by_mask(int_day_types,MONDAY)
+        valid_days << 2  if day_by_mask(int_day_types,TUESDAY)
+        valid_days << 3  if day_by_mask(int_day_types,WEDNESDAY)
+        valid_days << 4  if day_by_mask(int_day_types,THURSDAY)
+        valid_days << 5  if day_by_mask(int_day_types,FRIDAY)
+        valid_days << 6  if day_by_mask(int_day_types,SATURDAY)
+        valid_days << 7  if day_by_mask(int_day_types,SUNDAY)
+      end
     end
-  end
 
-  def valid_day? wday
-    valid_days.include?(wday)
+    def self.day_by_mask(int_day_types,flag)
+      int_day_types & flag == flag
+    end
   end
 
-  def self.valid_days(int_day_types)
-    # Build an array with day of calendar week (1-7, Monday is 1).
-    [].tap do |valid_days|
-      valid_days << 1  if day_by_mask(int_day_types,MONDAY)
-      valid_days << 2  if day_by_mask(int_day_types,TUESDAY)
-      valid_days << 3  if day_by_mask(int_day_types,WEDNESDAY)
-      valid_days << 4  if day_by_mask(int_day_types,THURSDAY)
-      valid_days << 5  if day_by_mask(int_day_types,FRIDAY)
-      valid_days << 6  if day_by_mask(int_day_types,SATURDAY)
-      valid_days << 7  if day_by_mask(int_day_types,SUNDAY)
-    end
+  def valid_days
+    self.class.valid_days int_day_types
   end
 
   def monday
diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
new file mode 100644
index 000000000..c1ba75d0a
--- /dev/null
+++ b/app/models/simple_exporter.rb
@@ -0,0 +1,183 @@
+# coding: utf-8
+class SimpleExporter < SimpleInterface
+  def export opts={}
+    configuration.validate!
+
+    init_env opts
+
+    @csv = nil
+    fail_with_error "Unable to write in file: #{self.filepath}" do
+      dir = Pathname.new(self.filepath).dirname
+      FileUtils.mkdir_p dir
+      @csv = CSV.open(self.filepath, 'w', self.configuration.csv_options)
+    end
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_collection
+    else
+      ActiveRecord::Base.transaction do
+        process_collection
+      end
+    end
+    self.status ||= :success
+  rescue SimpleInterface::FailedOperation
+    self.status = :failed
+  ensure
+    @csv&.close
+    self.save!
+  end
+
+  def collection
+    @collection ||= begin
+      coll = configuration.collection
+      coll = coll.call() if coll.is_a?(Proc)
+      coll
+    end
+  end
+
+  def encode_string s
+    s.encode("utf-8").force_encoding("utf-8")
+  end
+
+  protected
+  def init_env opts
+    @number_of_lines = collection.size
+
+    super opts
+  end
+
+  def process_collection
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting export ...", color: :green
+    log "Export will be written in #{filepath}", color: :green
+    @csv << self.configuration.columns.map(&:name)
+    if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      ids = collection.pluck :id
+      ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+        collection.where(id: batch_ids).each do |item|
+          handle_item item
+        end
+      end
+    else
+      collection.each{|item| handle_item item }
+    end
+    print_state
+  end
+
+  def map_item_to_rows item
+    return [item] unless configuration.item_to_rows_mapping
+    configuration.item_to_rows_mapping.call(item).map {|row| row.is_a?(ActiveRecord::Base) ? row : CustomRow.new(row) }
+  end
+
+  def resolve_value item, col
+    scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
+    val = col[:value]
+    if val.nil? || val.is_a?(Proc)
+      if val.is_a?(Proc)
+        val = instance_exec(scoped_item, &val)
+      else
+        attributes = [col.attribute].flatten
+        val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr) if tmp }
+      end
+    end
+    if val.nil? && !col.omit_nil?
+      push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
+      self.status ||= :success_with_warnings
+    end
+
+    if val.nil? && col.required?
+      @new_status = colorize("x", :red)
+      raise "MISSING VALUE FOR COLUMN #{col.name}"
+    end
+
+    val = encode_string(val) if val.is_a?(String)
+    val
+  end
+
+  def handle_item item
+    number_of_lines = @number_of_lines
+    @current_item = item
+    map_item_to_rows(item).each_with_index do |item, i|
+      @number_of_lines = number_of_lines + i
+      @current_row = item.attributes
+      @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+      row = []
+      @new_status = nil
+      self.configuration.columns.each do |col|
+        val = resolve_value item, col
+        @new_status ||= colorize("✓", :green)
+        row << val
+      end
+      push_in_journal({event: :success, kind: :log})
+      @statuses += @new_status
+      print_state if @current_line % 20 == 0 || i > 0
+      @current_line += 1
+      @csv << row
+    end
+  end
+
+  class CustomRow < OpenStruct
+    def initialize data
+      super data
+      @data = data
+    end
+
+    def attributes
+      flatten_hash @data
+    end
+
+    protected
+    def flatten_hash h
+      h.each_with_object({}) do |(k, v), h|
+        if v.is_a? Hash
+          flatten_hash(v).map do |h_k, h_v|
+            h["#{k}.#{h_k}".to_sym] = h_v
+          end
+        else
+          h[k] = v
+        end
+      end
+    end
+  end
+
+  class Configuration < SimpleInterface::Configuration
+    attr_accessor :collection
+    attr_accessor :batch_size
+    attr_accessor :logged_attributes
+    attr_accessor :item_to_rows_mapping
+
+    def initialize import_name, opts={}
+      super import_name, opts
+      @collection = opts[:collection]
+      @batch_size = opts[:batch_size] || 1000
+      @logged_attributes = opts[:logged_attributes]
+      @item_to_rows_mapping = opts[:item_to_rows_mapping]
+    end
+
+    def options
+      super.update({
+        collection: collection,
+        batch_size: batch_size,
+        logged_attributes: logged_attributes,
+        item_to_rows_mapping: item_to_rows_mapping,
+      })
+    end
+
+    def map_item_to_rows &block
+      @item_to_rows_mapping = block
+    end
+
+    def add_column name, opts={}
+      raise "Column already defined: #{name}" if @columns.any?{|c| c.name == name.to_s}
+      super name, opts
+    end
+
+    def validate!
+      raise "Incomplete configuration, missing collection for #{@import_name}" if collection.nil?
+    end
+  end
+end
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index d6ba64494..e23b3e524 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -1,38 +1,4 @@
-class SimpleImporter < ActiveRecord::Base
-  attr_accessor :configuration
-
-  def self.define name
-    @importers ||= {}
-    configuration = Configuration.new name
-    yield configuration
-    configuration.validate!
-    @importers[name.to_sym] = configuration
-  end
-
-  def self.find_configuration name
-    @importers ||= {}
-    configuration = @importers[name.to_sym]
-    raise "Importer not found: #{name}" unless configuration
-    configuration
-  end
-
-  def initialize *args
-    super *args
-    self.configuration = self.class.find_configuration self.configuration_name
-    self.journal ||= []
-  end
-
-  def configure
-    new_config = configuration.duplicate
-    yield new_config
-    new_config.validate!
-    self.configuration = new_config
-  end
-
-  def context
-    self.configuration.context
-  end
-
+class SimpleImporter < SimpleInterface
   def resolve col_name, value, &block
     val = block.call(value)
     return val if val.present?
@@ -41,20 +7,15 @@ class SimpleImporter < ActiveRecord::Base
   end
 
   def import opts={}
-    @verbose = opts.delete :verbose
-
+    configuration.validate!
 
-    @resolution_queue = Hash.new{|h,k| h[k] = []}
-    @errors = []
-    @messages = []
-    @number_of_lines = 0
-    @padding = 1
-    @current_line = 0
     fail_with_error "File not found: #{self.filepath}" do
       @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
-      @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
     end
 
+    init_env opts
+
+    @resolution_queue = Hash.new{|h,k| h[k] = []}
 
     self.configuration.before_actions(:parsing).each do |action| action.call self end
 
@@ -68,28 +29,12 @@ class SimpleImporter < ActiveRecord::Base
       end
     end
     self.status ||= :success
-  rescue FailedImport
+  rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
     self.save!
   end
 
-  def fail_with_error msg=nil, opts={}
-    begin
-      yield
-    rescue => e
-      msg = msg.call if msg.is_a?(Proc)
-      custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
-      push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
-      @new_status = colorize("x", :red)
-      if self.configuration.ignore_failures
-        raise FailedRow if opts[:abort_row]
-      else
-        raise FailedImport
-      end
-    end
-  end
-
   def encode_string s
     s.encode("utf-8").force_encoding("utf-8")
   end
@@ -109,16 +54,6 @@ class SimpleImporter < ActiveRecord::Base
     log "CSV file dumped in #{filepath}"
   end
 
-  def log msg, opts={}
-    msg = colorize msg, opts[:color] if opts[:color]
-    if opts[:append]
-      @messages[-1] = (@messages[-1] || "") + msg
-    else
-      @messages << msg
-    end
-    print_state
-  end
-
   protected
 
   def process_csv_file
@@ -154,23 +89,23 @@ class SimpleImporter < ActiveRecord::Base
             action.call self, @current_record
           end
         end
-      rescue FailedRow
+      rescue SimpleInterface::FailedRow
         @new_status = colorize("x", :red)
       end
       push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
       @statuses += @new_status
       self.configuration.columns.each do |col|
-      if @current_record && col.name && @resolution_queue.any?
-        val = @current_record.send col[:attribute]
-        (@resolution_queue.delete([col.name, val]) || []).each do |res|
-          record = res[:record]
-          attribute = res[:attribute]
-          value = res[:block].call(val, record)
-          record.send "#{attribute}=", value
-          record.save!
+        if @current_record && col.name && @resolution_queue.any?
+          val = @current_record.send col[:attribute]
+          (@resolution_queue.delete([col.name, val]) || []).each do |res|
+            record = res[:record]
+            attribute = res[:attribute]
+            value = res[:block].call(val, record)
+            record.send "#{attribute}=", value
+            record.save!
+          end
         end
       end
-    end
       print_state
       @current_line += 1
     end
@@ -179,7 +114,7 @@ class SimpleImporter < ActiveRecord::Base
       self.configuration.after_actions(:all).each do |action|
         action.call self
       end
-    rescue FailedRow
+    rescue SimpleInterface::FailedRow
     end
   end
 
@@ -215,211 +150,20 @@ class SimpleImporter < ActiveRecord::Base
     end
   end
 
-  def push_in_journal data
-    line = @current_line + 1
-    line += 1 if configuration.headers
-    self.journal.push data.update(line: line, row: @current_row)
-    if data[:kind] == :error || data[:kind] == :warning
-      @errors.push data
-    end
-  end
-
-  def colorize txt, color
-    color = {
-      red: "31",
-      green: "32",
-      orange: "33",
-    }[color] || "33"
-    "\e[#{color}m#{txt}\e[0m"
-  end
-
-  def print_state
-    return unless @verbose
-
-    @status_width ||= begin
-      term_width = %x(tput cols).to_i
-      term_width - @padding - 10
-    rescue
-      100
-    end
-
-    @status_height ||= begin
-      term_height = %x(tput lines).to_i
-      term_height - 3
-    rescue
-      50
-    end
-
-    full_status = @statuses || ""
-    full_status = full_status.last(@status_width*10) || ""
-    padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
-    full_status = "#{full_status}#{"."*[padding_size, 0].max}"
-
-    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
-
-    lines_count = [(@status_height / 2) - 3, 1].max
-
-    if @messages.any?
-      msg += "\n\n"
-      msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
-      msg += "[...]\n" if @messages.count > lines_count
-      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
-      msg += "\n"*[lines_count-@messages.count, 0].max
-    end
-
-    if @errors.any?
-      msg += "\n\n"
-      msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
-      msg += "[...]\n" if @errors.count > lines_count
-      msg += @errors.last(lines_count).map do |j|
-        kind = j[:kind]
-        kind = colorize(kind, kind == :error ? :red : :orange)
-        kind = "[#{kind}]"
-        kind += " "*(25 - kind.size)
-        encode_string("#{kind}L#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}").truncate(@status_width)
-      end.join("\n")
-    end
-    custom_print msg, clear: true
-  end
-
-  def custom_print msg, opts={}
-    return unless @verbose
-    out = ""
-    msg = colorize(msg, opts[:color]) if opts[:color]
-    puts "\e[H\e[2J" if opts[:clear]
-    out += msg
-    print out
-  end
-
-  class FailedImport < RuntimeError
-  end
-
-  class FailedRow < RuntimeError
-  end
-
-  class Configuration
-    attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
-    attr_reader :columns
+  class Configuration < SimpleInterface::Configuration
+    attr_accessor :model
 
     def initialize import_name, opts={}
-      @import_name = import_name
-      @key = opts[:key] || "id"
-      @headers = opts.has_key?(:headers) ? opts[:headers] : true
-      @separator = opts[:separator] || ","
-      @encoding = opts[:encoding]
-      @columns = opts[:columns] || []
+      super import_name, opts
       @model = opts[:model]
-      @custom_handler = opts[:custom_handler]
-      @before = opts[:before]
-      @after = opts[:after]
-      @ignore_failures = opts[:ignore_failures]
-      @context = opts[:context] || {}
-      @scope = opts[:scope]
-    end
-
-    def duplicate
-      Configuration.new @import_name, self.options
     end
 
     def options
-      {
-        key: @key,
-        headers: @headers,
-        separator: @separator,
-        encoding: @encoding,
-        columns: @columns.map(&:duplicate),
-        model: model,
-        custom_handler: @custom_handler,
-        before: @before,
-        after: @after,
-        ignore_failures: @ignore_failures,
-        context: @context,
-        scope: @scope
-      }
+      super.update({model: model})
     end
 
     def validate!
       raise "Incomplete configuration, missing model for #{@import_name}" unless model.present?
     end
-
-    def attribute_for_col col_name
-      column = self.columns.find{|c| c.name == col_name}
-      column && column[:attribute] || col_name
-    end
-
-    def record_scope
-      _scope = @scope
-      _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
-      _scope || model
-    end
-
-    def find_record attrs
-      record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
-    end
-
-    def csv_options
-      {
-        headers: self.headers,
-        col_sep: self.separator,
-        encoding: self.encoding
-      }
-    end
-
-    def add_column name, opts={}
-      @columns.push Column.new({name: name.to_s}.update(opts))
-    end
-
-    def add_value attribute, value
-      @columns.push Column.new({attribute: attribute, value: value})
-    end
-
-    def before group=:all, &block
-      @before ||= Hash.new{|h, k| h[k] = []}
-      @before[group].push block
-    end
-
-    def after group=:all, &block
-      @after ||= Hash.new{|h, k| h[k] = []}
-      @after[group].push block
-    end
-
-    def before_actions group=:all
-      @before ||= Hash.new{|h, k| h[k] = []}
-      @before[group]
-    end
-
-    def after_actions group=:all
-      @after ||= Hash.new{|h, k| h[k] = []}
-      @after[group]
-    end
-
-    def custom_handler &block
-      @custom_handler = block
-    end
-
-    def get_custom_handler
-      @custom_handler
-    end
-
-    class Column
-      attr_accessor :name
-      def initialize opts={}
-        @name = opts[:name]
-        @options = opts
-        @options[:attribute] ||= @name
-      end
-
-      def duplicate
-        Column.new @options.dup
-      end
-
-      def required?
-        !!@options[:required]
-      end
-
-      def [](key)
-        @options[key]
-      end
-    end
   end
 end
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
new file mode 100644
index 000000000..489419482
--- /dev/null
+++ b/app/models/simple_interface.rb
@@ -0,0 +1,297 @@
+class SimpleInterface < ActiveRecord::Base
+  attr_accessor :configuration
+
+  class << self
+    def configuration_class
+      "#{self.name}::Configuration".constantize
+    end
+
+    def define name
+      @importers ||= {}
+      configuration = configuration_class.new name
+      yield configuration if block_given?
+      @importers[name.to_sym] = configuration
+    end
+
+    def find_configuration name
+      @importers ||= {}
+      configuration = @importers[name.to_sym]
+      raise "Importer not found: #{name}" unless configuration
+      configuration
+    end
+  end
+
+  def initialize *args
+    super *args
+    self.configuration = self.class.find_configuration self.configuration_name
+    self.journal ||= []
+  end
+
+  def init_env opts
+    @verbose = opts.delete :verbose
+
+    @errors = []
+    @messages = []
+    @padding = 1
+    @current_line = -1
+    @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+  end
+
+  def configure
+    new_config = configuration.duplicate
+    yield new_config
+    self.configuration = new_config
+  end
+
+  def context
+    self.configuration.context
+  end
+
+  def fail_with_error msg=nil, opts={}
+    begin
+      yield
+    rescue => e
+      msg = msg.call if msg.is_a?(Proc)
+      custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
+      push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
+      @new_status = colorize("x", :red)
+      if self.configuration.ignore_failures
+        raise SimpleInterface::FailedRow if opts[:abort_row]
+      else
+        raise FailedOperation
+      end
+    end
+  end
+
+  def log msg, opts={}
+    msg = msg.to_s
+    msg = colorize msg, opts[:color] if opts[:color]
+    if opts[:append]
+      @messages[-1] = (@messages[-1] || "") + msg
+    else
+      @messages << msg
+    end
+    print_state
+  end
+
+  protected
+
+  def push_in_journal data
+    line = (@current_line || 0) + 1
+    line += 1 if configuration.headers
+    @errors ||= []
+    self.journal.push data.update(line: line, row: @current_row)
+    if data[:kind] == :error || data[:kind] == :warning
+      @errors.push data
+    end
+  end
+
+  def colorize txt, color
+    color = {
+      red: "31",
+      green: "32",
+      orange: "33",
+    }[color] || "33"
+    "\e[#{color}m#{txt}\e[0m"
+  end
+
+  def print_state
+    return unless @verbose
+
+    @status_width ||= begin
+      term_width = %x(tput cols).to_i
+      term_width - @padding - 10
+    rescue
+      100
+    end
+
+    @status_height ||= begin
+      term_height = %x(tput lines).to_i
+      term_height - 3
+    rescue
+      50
+    end
+
+    full_status = @statuses || ""
+    full_status = full_status.last(@status_width*10) || ""
+    padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
+    full_status = "#{full_status}#{"."*[padding_size, 0].max}"
+
+    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+
+    lines_count = [(@status_height / 2) - 3, 1].max
+
+    if @messages.any?
+      msg += "\n\n"
+      msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
+      msg += "[...]\n" if @messages.count > lines_count
+      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
+      msg += "\n"*[lines_count-@messages.count, 0].max
+    end
+
+    if @errors.any?
+      msg += "\n\n"
+      msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
+      msg += "[...]\n" if @errors.count > lines_count
+      msg += @errors.last(lines_count).map do |j|
+        kind = j[:kind]
+        kind = colorize(kind, kind == :error ? :red : :orange)
+        kind = "[#{kind}]"
+        kind += " "*(25 - kind.size)
+        encode_string("#{kind}L#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}").truncate(@status_width)
+      end.join("\n")
+    end
+    custom_print msg, clear: true
+  end
+
+  def custom_print msg, opts={}
+    return unless @verbose
+    out = ""
+    msg = colorize(msg, opts[:color]) if opts[:color]
+    puts "\e[H\e[2J" if opts[:clear]
+    out += msg
+    print out
+  end
+
+  class FailedRow < RuntimeError
+  end
+
+  class FailedOperation < RuntimeError
+  end
+
+  class Configuration
+    attr_accessor :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
+    attr_reader :columns
+
+    def initialize import_name, opts={}
+      @import_name = import_name
+      @key = opts[:key] || "id"
+      @headers = opts.has_key?(:headers) ? opts[:headers] : true
+      @separator = opts[:separator] || ","
+      @encoding = opts[:encoding]
+      @columns = opts[:columns] || []
+      @custom_handler = opts[:custom_handler]
+      @before = opts[:before]
+      @after = opts[:after]
+      @ignore_failures = opts[:ignore_failures]
+      @context = opts[:context] || {}
+      @scope = opts[:scope]
+    end
+
+    def on_relation relation_name
+      @current_scope ||= []
+      @current_scope.push relation_name
+      yield
+      @current_scope.pop
+    end
+
+    def duplicate
+      self.class.new @import_name, self.options
+    end
+
+    def options
+      {
+        key: @key,
+        headers: @headers,
+        separator: @separator,
+        encoding: @encoding,
+        columns: @columns.map(&:duplicate),
+        custom_handler: @custom_handler,
+        before: @before,
+        after: @after,
+        ignore_failures: @ignore_failures,
+        context: @context,
+        scope: @scope
+      }
+    end
+
+    def attribute_for_col col_name
+      column = self.columns.find{|c| c.name == col_name}
+      column && column[:attribute] || col_name
+    end
+
+    def record_scope
+      _scope = @scope
+      _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
+      _scope || model
+    end
+
+    def find_record attrs
+      record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+    end
+
+    def csv_options
+      {
+        headers: self.headers,
+        col_sep: self.separator,
+        encoding: self.encoding
+      }
+    end
+
+    def add_column name, opts={}
+      @current_scope ||= []
+      @columns.push Column.new({name: name.to_s, scope: @current_scope.dup}.update(opts))
+    end
+
+    def add_value attribute, value
+      @columns.push Column.new({attribute: attribute, value: value})
+    end
+
+    def before group=:all, &block
+      @before ||= Hash.new{|h, k| h[k] = []}
+      @before[group].push block
+    end
+
+    def after group=:all, &block
+      @after ||= Hash.new{|h, k| h[k] = []}
+      @after[group].push block
+    end
+
+    def before_actions group=:all
+      @before ||= Hash.new{|h, k| h[k] = []}
+      @before[group]
+    end
+
+    def after_actions group=:all
+      @after ||= Hash.new{|h, k| h[k] = []}
+      @after[group]
+    end
+
+    def custom_handler &block
+      @custom_handler = block
+    end
+
+    def get_custom_handler
+      @custom_handler
+    end
+
+    class Column
+      attr_accessor :name, :attribute
+      def initialize opts={}
+        @name = opts[:name]
+        @options = opts
+        @attribute = @options[:attribute] ||= @name
+      end
+
+      def duplicate
+        Column.new @options.dup
+      end
+
+      def required?
+        !!@options[:required]
+      end
+
+      def omit_nil?
+        !!@options[:omit_nil]
+      end
+
+      def scope
+        @options[:scope] || []
+      end
+
+      def [](key)
+        @options[key]
+      end
+    end
+  end
+end
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
new file mode 100644
index 000000000..8c44c149a
--- /dev/null
+++ b/app/models/simple_json_exporter.rb
@@ -0,0 +1,185 @@
+class SimpleJsonExporter < SimpleExporter
+
+  def export opts={}
+    configuration.validate!
+
+    init_env opts
+
+    if self.configuration.root
+      @out = {self.configuration.root => []}
+    else
+      @out = []
+    end
+
+    fail_with_error "Unable to write in file: #{self.filepath}" do
+      dir = Pathname.new(self.filepath).dirname
+      FileUtils.mkdir_p dir
+      @file = File.open(self.filepath, 'w', self.configuration.file_options)
+    end
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_collection
+    else
+      ActiveRecord::Base.transaction do
+        process_collection
+      end
+    end
+    self.status ||= :success
+  rescue SimpleInterface::FailedOperation
+    self.status = :failed
+  ensure
+    if @file
+      @file.write @out.to_json
+      @file.close
+    end
+    self.save!
+  end
+
+  protected
+  def root
+    if self.configuration.root
+      @out[self.configuration.root]
+    else
+      @out
+    end
+  end
+
+  def process_collection
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting export ...", color: :green
+    log "Export will be written in #{filepath}", color: :green
+
+    if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      ids = collection.pluck :id
+      ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+        collection.where(id: batch_ids).each do |item|
+          handle_item item
+        end
+      end
+    else
+      collection.each{|item| handle_item item }
+    end
+    print_state
+  end
+
+  def resolve_node item, node
+    vals = []
+    scoped_item = node.scope.inject(item){|tmp, scope| tmp.send(scope)}
+
+    [scoped_item.send(node.attribute)].flatten.each do |node_item|
+      item_val = {}
+      apply_configuration node_item, node.configuration, item_val
+      vals.push item_val
+    end
+    node.multiple ? vals : vals.first
+  end
+
+  def apply_configuration item, configuration, output
+    configuration.columns.each do |col|
+      val = resolve_value item, col
+      output[col.name] = val unless val.nil? && col.omit_nil?
+    end
+
+    configuration.nodes.each do |node|
+      val = resolve_node item, node
+      output[node.name] = val
+    end
+  end
+
+  def handle_item item
+    number_of_lines = @number_of_lines
+    @current_item = item
+    map_item_to_rows(item).each_with_index do |item, i|
+      @number_of_lines = number_of_lines + i
+      serialized_item = {}
+      @current_row = item.attributes
+      @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+      @new_status = nil
+
+      apply_configuration item, self.configuration, serialized_item
+
+      @new_status ||= colorize("✓", :green)
+
+      push_in_journal({event: :success, kind: :log})
+      @statuses += @new_status
+      print_state if @current_line % 20 == 0 || i > 0
+      @current_line += 1
+      append_item serialized_item
+    end
+  end
+
+  def append_item serialized_item
+    root.push serialized_item
+  end
+
+  class Configuration < SimpleExporter::Configuration
+    attr_reader :nodes
+    attr_accessor :root
+
+    alias_method :add_field, :add_column
+
+    def initialize import_name, opts={}
+      super import_name, opts
+      @collection = opts[:collection]
+      @nodes = opts[:nodes] || []
+      @root = opts[:root]
+    end
+
+    def options
+      super.update({
+        nodes: @nodes,
+        root: @root,
+      })
+    end
+
+    def add_node name, opts={}
+      @nodes ||= []
+      @current_scope ||= []
+      node = Node.new({name: name.to_s, scope: @current_scope.dup}.update(opts))
+      yield node.configuration
+      @nodes.push node
+    end
+
+    def add_nodes name, opts={}, &block
+      self.add_node name, opts.update({multiple: true}), &block
+    end
+
+    def file_options
+      {
+        encoding: self.encoding
+      }
+    end
+  end
+
+  class NodeConfiguration < Configuration
+    def initialize node
+      super
+    end
+  end
+
+  class Node
+    attr_accessor :name, :configuration
+
+    def initialize opts={}
+      @name = opts[:name]
+      @options = opts
+      @configuration = NodeConfiguration.new self
+    end
+
+    def attribute
+      @options[:attribute] || name
+    end
+
+    def multiple
+      !!@options[:multiple]
+    end
+
+    def scope
+      @options[:scope] || []
+    end
+  end
+end