Refs #6068; First steps toward JSON exporter

6 files changed, 309 insertions, 42 deletions

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index fe0aa05b5..adc48533f 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -1,17 +1,8 @@
 class SimpleExporter < SimpleInterface
   def export opts={}
     configuration.validate!
-    @verbose = opts.delete :verbose
-
-    @resolution_queue = Hash.new{|h,k| h[k] = []}
-    @errors = []
-    @messages = []
-    @number_of_lines = 0
-    @padding = 1
-    @current_line = -1
-    @number_of_lines = collection.size
 
-    @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+    init_env opts
 
     @csv = nil
     fail_with_error "Unable to write in file: #{self.filepath}" do
@@ -52,6 +43,12 @@ class SimpleExporter < SimpleInterface
   end
 
   protected
+  def init_env opts
+    @number_of_lines = collection.size
+
+    super opts
+  end
+
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
     log "Starting export ...", color: :green
@@ -75,6 +72,31 @@ class SimpleExporter < SimpleInterface
     configuration.item_to_rows_mapping.call(item).map {|row| row.is_a?(ActiveRecord::Base) ? row : CustomRow.new(row) }
   end
 
+  def resolve_value item, col
+    scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
+    val = col[:value]
+    if val.nil? || val.is_a?(Proc)
+      if val.is_a?(Proc)
+        val = instance_exec(scoped_item, &val)
+      else
+        attributes = [col.attribute].flatten
+        val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
+      end
+    end
+    if val.nil?
+      push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
+      self.status ||= :success_with_warnings
+    end
+
+    if val.nil? && col.required?
+      @new_status = colorize("x", :red)
+      raise "MISSING VALUE FOR COLUMN #{col.name}"
+    end
+
+    val = encode_string(val) if val.is_a?(String)
+    val
+  end
+
   def handle_item item
     number_of_lines = @number_of_lines
     map_item_to_rows(item).each_with_index do |item, i|
@@ -84,27 +106,8 @@ class SimpleExporter < SimpleInterface
       row = []
       @new_status = nil
       self.configuration.columns.each do |col|
-        scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
-        val = col[:value]
-        if val.nil? || val.is_a?(Proc)
-          if val.is_a?(Proc)
-            val = instance_exec(scoped_item, &val)
-          else
-            attributes = [col.attribute].flatten
-            val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
-          end
-        end
-        if val.nil?
-          push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
-          self.status ||= :success_with_warnings
-        end
-
-        if val.nil? && col.required?
-          @new_status = colorize("x", :red)
-          raise "MISSING VALUE FOR COLUMN #{col.name}"
-        end
+        val = resolve_value item, col
         @new_status ||= colorize("✓", :green)
-        val = encode_string(val) if val.is_a?(String)
         row << val
       end
       push_in_journal({event: :success, kind: :log})
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index 6db71797a..e23b3e524 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -1,5 +1,4 @@
 class SimpleImporter < SimpleInterface
-
   def resolve col_name, value, &block
     val = block.call(value)
     return val if val.present?
@@ -9,19 +8,15 @@ class SimpleImporter < SimpleInterface
 
   def import opts={}
     configuration.validate!
-    @verbose = opts.delete :verbose
 
-    @resolution_queue = Hash.new{|h,k| h[k] = []}
-    @errors = []
-    @messages = []
-    @number_of_lines = 0
-    @padding = 1
-    @current_line = 0
     fail_with_error "File not found: #{self.filepath}" do
       @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
-      @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
     end
 
+    init_env opts
+
+    @resolution_queue = Hash.new{|h,k| h[k] = []}
+
     self.configuration.before_actions(:parsing).each do |action| action.call self end
 
     @statuses = ""
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 5f022719a..07fabd832 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -27,6 +27,16 @@ class SimpleInterface < ActiveRecord::Base
     self.journal ||= []
   end
 
+  def init_env opts
+    @verbose = opts.delete :verbose
+
+    @errors = []
+    @messages = []
+    @padding = 1
+    @current_line = 0
+    @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+  end
+
   def configure
     new_config = configuration.duplicate
     yield new_config
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
new file mode 100644
index 000000000..706307de1
--- /dev/null
+++ b/app/models/simple_json_exporter.rb
@@ -0,0 +1,164 @@
+class SimpleJsonExporter < SimpleExporter
+
+  def export opts={}
+    configuration.validate!
+
+    init_env opts
+
+    if self.configuration.root
+      @out = {self.configuration.root => []}
+    else
+      @out = []
+    end
+
+    fail_with_error "Unable to write in file: #{self.filepath}" do
+      dir = Pathname.new(self.filepath).dirname
+      FileUtils.mkdir_p dir
+      @file = File.open(self.filepath, 'w', self.configuration.file_options)
+    end
+
+    self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+    @statuses = ""
+
+    if ENV["NO_TRANSACTION"]
+      process_collection
+    else
+      ActiveRecord::Base.transaction do
+        process_collection
+      end
+    end
+    self.status ||= :success
+  rescue SimpleInterface::FailedOperation
+    self.status = :failed
+  ensure
+    if @file
+      @file.write @out.to_json
+      @file.close
+    end
+    self.save!
+  end
+
+  protected
+  def root
+    if self.configuration.root
+      @out[self.configuration.root]
+    else
+      @out
+    end
+  end
+
+  def process_collection
+    self.configuration.before_actions(:all).each do |action| action.call self end
+    log "Starting export ...", color: :green
+    log "Export will be written in #{filepath}", color: :green
+
+    if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      ids = collection.pluck :id
+      ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+        collection.where(id: batch_ids).each do |item|
+          handle_item item
+        end
+      end
+    else
+      collection.each{|item| handle_item item }
+    end
+    print_state
+  end
+
+  def resolve_node item, node
+    vals = []
+    [item.send(node.attribute)].flatten.each do |node_item|
+      item_val = {}
+      apply_configuration node_item, node.configuration, item_val
+      vals.push item_val
+    end
+    node.multiple ? vals : vals.first
+  end
+
+  def apply_configuration item, configuration, output
+    configuration.columns.each do |col|
+      val = resolve_value item, col
+      output[col.name] = val
+    end
+
+    configuration.nodes.each do |node|
+      val = resolve_node item, node
+      output[node.name] = val
+    end
+  end
+
+  def handle_item item
+    serialized_item = {}
+    @current_row = item.attributes
+    @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+    @new_status = nil
+
+    apply_configuration item, self.configuration, serialized_item
+
+    @new_status ||= colorize("✓", :green)
+
+    push_in_journal({event: :success, kind: :log})
+    @statuses += @new_status
+    print_state if @current_line % 20 == 0
+    @current_line += 1
+    append_item serialized_item
+  end
+
+  def append_item serialized_item
+    root.push serialized_item
+  end
+
+  class Configuration < SimpleExporter::Configuration
+    attr_reader :nodes
+    attr_accessor :root
+
+    alias_method :add_field, :add_column
+
+    def initialize import_name, opts={}
+      @nodes = []
+      super import_name, opts
+    end
+
+    def add_node name, opts={}
+      @nodes ||= []
+      node = Node.new({name: name.to_s}.update(opts))
+      yield node.configuration
+      @nodes.push node
+    end
+
+    def add_nodes name, opts={}, &block
+      self.add_node name, opts.update({multiple: true}), &block
+    end
+
+    def file_options
+      {
+        encoding: self.encoding
+      }
+    end
+  end
+
+  class NodeConfiguration < Configuration
+    def initialize node
+      super
+    end
+  end
+
+  class Node
+    attr_accessor :name, :configuration
+
+    def initialize opts={}
+      @name = opts[:name]
+      @options = opts
+      @configuration = NodeConfiguration.new self
+    end
+
+    def attribute
+      name
+    end
+
+    def multiple
+      !!@options[:multiple]
+    end
+  end
+end
diff --git a/spec/models/simple_exporter_spec.rb b/spec/models/simple_exporter_spec.rb
index 18cdc55e8..75051aeb9 100644
--- a/spec/models/simple_exporter_spec.rb
+++ b/spec/models/simple_exporter_spec.rb
@@ -19,7 +19,7 @@ RSpec.describe SimpleExporter do
         expect{SimpleExporter.find_configuration(:foo)}.to_not raise_error
         expect{SimpleExporter.new(configuration_name: :foo, filepath: "").export}.to_not raise_error
         expect{SimpleExporter.find_configuration(:bar)}.to raise_error
-        expect{SimpleExporter.new(configuration_name: :bar, filepath: "")}.to_not raise_error
+        expect{SimpleExporter.new(configuration_name: :bar, filepath: "")}.to raise_error
         expect{SimpleExporter.new(configuration_name: :bar, filepath: "").export}.to raise_error
         expect{SimpleExporter.create(configuration_name: :foo, filepath: "")}.to change{SimpleExporter.count}.by 1
       end
@@ -64,7 +64,7 @@ RSpec.describe SimpleExporter do
       end
 
       it "should export the given file" do
-        expect{exporter.export verbose: true}.to_not raise_error
+        expect{exporter.export verbose: false}.to_not raise_error
         expect(exporter.status).to eq "success"
         expect(File.exists?(filepath)).to be_truthy
         csv = CSV.read(filepath, headers: true, col_sep: ";")
@@ -102,7 +102,7 @@ RSpec.describe SimpleExporter do
       end
 
       it "should export the given file" do
-        expect{exporter.export verbose: true}.to_not raise_error
+        expect{exporter.export verbose: false}.to_not raise_error
         expect(exporter.status).to eq "success"
         expect(File.exists?(filepath)).to be_truthy
         csv = CSV.read(filepath, headers: true)
diff --git a/spec/models/simple_json_exporter_spec.rb b/spec/models/simple_json_exporter_spec.rb
new file mode 100644
index 000000000..4b48dc732
--- /dev/null
+++ b/spec/models/simple_json_exporter_spec.rb
@@ -0,0 +1,95 @@
+RSpec.describe SimpleJsonExporter do
+  describe "#define" do
+    context "with an incomplete configuration" do
+      it "should raise an error" do
+        SimpleJsonExporter.define :foo
+        expect do
+          SimpleJsonExporter.new(configuration_name: :test).export
+        end.to raise_error
+      end
+    end
+    context "with a complete configuration" do
+      before do
+        SimpleJsonExporter.define :foo do |config|
+          config.collection = Chouette::StopArea.all
+        end
+      end
+
+      it "should define an exporter" do
+        expect{SimpleJsonExporter.find_configuration(:foo)}.to_not raise_error
+        expect{SimpleJsonExporter.new(configuration_name: :foo, filepath: "").export}.to_not raise_error
+        expect{SimpleJsonExporter.find_configuration(:bar)}.to raise_error
+        expect{SimpleJsonExporter.new(configuration_name: :bar, filepath: "")}.to raise_error
+        expect{SimpleJsonExporter.new(configuration_name: :bar, filepath: "").export}.to raise_error
+        expect{SimpleJsonExporter.create(configuration_name: :foo, filepath: "")}.to change{SimpleJsonExporter.count}.by 1
+      end
+    end
+
+    context "when defining the same col twice" do
+      it "should raise an error" do
+        expect do
+          SimpleJsonExporter.define :foo do |config|
+            config.collection = Chouette::StopArea.all
+            config.add_field :name
+            config.add_field :name
+          end
+        end.to raise_error
+      end
+    end
+  end
+
+  describe "#export" do
+    let(:exporter){ importer = SimpleJsonExporter.new(configuration_name: :test, filepath: filepath) }
+    let(:filepath){ Rails.root + "tmp/" + filename }
+    let(:filename){ "stop_area.json" }
+    # let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) }
+
+    context "with one row per item" do
+      before(:each) do
+        @stop_area = create :stop_area
+        @stop_1 = create :stop_point, stop_area: @stop_area
+        @stop_2 = create :stop_point, stop_area: @stop_area
+
+        SimpleJsonExporter.define :test do |config|
+          config.collection = ->{ Chouette::StopArea.all }
+          config.root = "stops"
+          config.add_field :name
+          config.add_field :lat, attribute: :latitude
+          config.add_field :lng, attribute: [:latitude, :to_i, :next]
+          config.add_field :type, attribute: :area_type
+          config.add_field :street_name, value: "Lil Exporter"
+          config.add_node :stop_area_referential do |config|
+            config.add_field :id, attribute: :id
+            config.add_field :id_other, value: ->(item){ item.id }
+          end
+
+          config.add_nodes :stop_points do |config|
+            config.add_field :id
+            config.add_node :stop_area do |config|
+              config.add_field :id
+            end
+          end
+          config.add_field :forty_two, value: 42
+        end
+      end
+
+      it "should export the given file" do
+        expect{exporter.export verbose: false}.to_not raise_error
+        expect(exporter.status).to eq "success"
+        expect(File.exists?(filepath)).to be_truthy
+        json = JSON.parse File.read(filepath)
+        row = json["stops"].first
+        expect(row["name"]).to eq @stop_area.name
+        expect(row["lat"]).to eq @stop_area.latitude.to_s
+        expect(row["lng"]).to eq (@stop_area.latitude.to_i + 1)
+        expect(row["street_name"]).to eq "Lil Exporter"
+        expect(row["stop_area_referential"]["id"]).to eq @stop_area.stop_area_referential_id
+        expect(row["stop_area_referential"]["id_other"]).to eq @stop_area.stop_area_referential_id
+        expect(row["stop_points"][0]["id"]).to eq @stop_1.id
+        expect(row["stop_points"][0]["stop_area"]["id"]).to eq @stop_area.id
+        expect(row["stop_points"][1]["id"]).to eq @stop_2.id
+        expect(row["forty_two"]).to eq 42
+      end
+    end
+  end
+end