Refs #6068; First steps toward JSON exporter

4 files changed, 211 insertions, 39 deletions

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index fe0aa05b5..adc48533f 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -1,17 +1,8 @@
 class SimpleExporter < SimpleInterface
   def export opts={}
     configuration.validate!
-    @verbose = opts.delete :verbose
-
-    @resolution_queue = Hash.new{|h,k| h[k] = []}
-    @errors = []
-    @messages = []
-    @number_of_lines = 0
-    @padding = 1
-    @current_line = -1
-    @number_of_lines = collection.size
 
-    @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+    init_env opts
 
     @csv = nil
     fail_with_error "Unable to write in file: #{self.filepath}" do
@@ -52,6 +43,12 @@ class SimpleExporter < SimpleInterface
   end
 
   protected
+  def init_env opts
+    @number_of_lines = collection.size
+
+    super opts
+  end
+
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
     log "Starting export ...", color: :green
@@ -75,6 +72,31 @@ class SimpleExporter < SimpleInterface
     configuration.item_to_rows_mapping.call(item).map {|row| row.is_a?(ActiveRecord::Base) ? row : CustomRow.new(row) }
   end
 
+  def resolve_value item, col
+    scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
+    val = col[:value]
+    if val.nil? || val.is_a?(Proc)
+      if val.is_a?(Proc)
+        val = instance_exec(scoped_item, &val)
+      else
+        attributes = [col.attribute].flatten
+        val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
+      end
+    end
+    if val.nil?
+      push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
+      self.status ||= :success_with_warnings
+    end
+
+    if val.nil? && col.required?
+      @new_status = colorize("x", :red)
+      raise "MISSING VALUE FOR COLUMN #{col.name}"
+    end
+
+    val = encode_string(val) if val.is_a?(String)
+    val
+  end
+
   def handle_item item
     number_of_lines = @number_of_lines
     map_item_to_rows(item).each_with_index do |item, i|
@@ -84,27 +106,8 @@ class SimpleExporter < SimpleInterface
       row = []
       @new_status = nil
       self.configuration.columns.each do |col|
-        scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
-        val = col[:value]
-        if val.nil? || val.is_a?(Proc)
-          if val.is_a?(Proc)
-            val = instance_exec(scoped_item, &val)
-          else
-            attributes = [col.attribute].flatten
-            val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
-          end
-        end
-        if val.nil?
-          push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
-          self.status ||= :success_with_warnings
-        end
-
-        if val.nil? && col.required?
-          @new_status = colorize("x", :red)
-          raise "MISSING VALUE FOR COLUMN #{col.name}"
-        end
+        val = resolve_value item, col
         @new_status ||= colorize("✓", :green)
-        val = encode_string(val) if val.is_a?(String)
         row << val
       end
       push_in_journal({event: :success, kind: :log})
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index 6db71797a..e23b3e524 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -1,5 +1,4 @@
 class SimpleImporter < SimpleInterface
-
   def resolve col_name, value, &block
     val = block.call(value)
     return val if val.present?
@@ -9,19 +8,15 @@ class SimpleImporter < SimpleInterface
 
   def import opts={}
     configuration.validate!
-    @verbose = opts.delete :verbose
 
-    @resolution_queue = Hash.new{|h,k| h[k] = []}
-    @errors = []
-    @messages = []
-    @number_of_lines = 0
-    @padding = 1
-    @current_line = 0
     fail_with_error "File not found: #{self.filepath}" do
       @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
-      @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
     end
 
+    init_env opts
+
+    @resolution_queue = Hash.new{|h,k| h[k] = []}
+
     self.configuration.before_actions(:parsing).each do |action| action.call self end
 
     @statuses = ""
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 5f022719a..07fabd832 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -27,6 +27,16 @@ class SimpleInterface < ActiveRecord::Base
     self.journal ||= []
   end
 
+  def init_env opts
+    @verbose = opts.delete :verbose
+
+    @errors = []
+    @messages = []
+    @padding = 1
+    @current_line = 0
+    @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+  end
+
   def configure
     new_config = configuration.duplicate
     yield new_config
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
new file mode 100644
index 000000000..706307de1
--- /dev/null
+++ b/app/models/simple_json_exporter.rb
@@ -0,0 +1,164 @@
+class SimpleJsonExporter < SimpleExporter
+
+  def export opts={}
+    configuration.validate!
+
+    init_env opts
+
+    if self.configuration.root
+      @out = {self.configuration.root => []}
+    else
+      @out = []
+    end
+
+    fail_with_error "Unable to write in file: #{self.filepath}" do
+      dir = Pathname.new(self.filepath).dirname
+      FileUtils.mkdir_p dir
+      @file = File.open(self.filepath, 'w', self.configuration.file_options)
+    end
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_collection
+    else
+      ActiveRecord::Base.transaction do
+        process_collection
+      end
+    end
+    self.status ||= :success
+  rescue SimpleInterface::FailedOperation
+    self.status = :failed
+  ensure
+    if @file
+      @file.write @out.to_json
+      @file.close
+    end
+    self.save!
+  end
+
+  protected
+  def root
+    if self.configuration.root
+      @out[self.configuration.root]
+    else
+      @out
+    end
+  end
+
+  def process_collection
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting export ...", color: :green
+    log "Export will be written in #{filepath}", color: :green
+
+    if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      ids = collection.pluck :id
+      ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+        collection.where(id: batch_ids).each do |item|
+          handle_item item
+        end
+      end
+    else
+      collection.each{|item| handle_item item }
+    end
+    print_state
+  end
+
+  def resolve_node item, node
+    vals = []
+    [item.send(node.attribute)].flatten.each do |node_item|
+      item_val = {}
+      apply_configuration node_item, node.configuration, item_val
+      vals.push item_val
+    end
+    node.multiple ? vals : vals.first
+  end
+
+  def apply_configuration item, configuration, output
+    configuration.columns.each do |col|
+      val = resolve_value item, col
+      output[col.name] = val
+    end
+
+    configuration.nodes.each do |node|
+      val = resolve_node item, node
+      output[node.name] = val
+    end
+  end
+
+  def handle_item item
+    serialized_item = {}
+    @current_row = item.attributes
+    @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+    @new_status = nil
+
+    apply_configuration item, self.configuration, serialized_item
+
+    @new_status ||= colorize("✓", :green)
+
+    push_in_journal({event: :success, kind: :log})
+    @statuses += @new_status
+    print_state if @current_line % 20 == 0
+    @current_line += 1
+    append_item serialized_item
+  end
+
+  def append_item serialized_item
+    root.push serialized_item
+  end
+
+  class Configuration < SimpleExporter::Configuration
+    attr_reader :nodes
+    attr_accessor :root
+
+    alias_method :add_field, :add_column
+
+    def initialize import_name, opts={}
+      @nodes = []
+      super import_name, opts
+    end
+
+    def add_node name, opts={}
+      @nodes ||= []
+      node = Node.new({name: name.to_s}.update(opts))
+      yield node.configuration
+      @nodes.push node
+    end
+
+    def add_nodes name, opts={}, &block
+      self.add_node name, opts.update({multiple: true}), &block
+    end
+
+    def file_options
+      {
+        encoding: self.encoding
+      }
+    end
+  end
+
+  class NodeConfiguration < Configuration
+    def initialize node
+      super
+    end
+  end
+
+  class Node
+    attr_accessor :name, :configuration
+
+    def initialize opts={}
+      @name = opts[:name]
+      @options = opts
+      @configuration = NodeConfiguration.new self
+    end
+
+    def attribute
+      name
+    end
+
+    def multiple
+      !!@options[:multiple]
+    end
+  end
+end