Merge pull request #361 from af83/6068-simple-exporter

6068 simple exporter

5 files changed, 177 insertions, 35 deletions

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index c1ba75d0a..c267b5b8c 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -16,19 +16,14 @@ class SimpleExporter < SimpleInterface
 
     @statuses = ""
 
-    if ENV["NO_TRANSACTION"]
-      process_collection
-    else
-      ActiveRecord::Base.transaction do
-        process_collection
-      end
-    end
+    process_collection
+
     self.status ||= :success
   rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
     @csv&.close
-    self.save!
+    task_finished
   end
 
   def collection
@@ -46,14 +41,13 @@ class SimpleExporter < SimpleInterface
   protected
   def init_env opts
     @number_of_lines = collection.size
-
     super opts
   end
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
     @csv << self.configuration.columns.map(&:name)
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
       ids = collection.pluck :id
@@ -87,6 +81,7 @@ class SimpleExporter < SimpleInterface
     if val.nil? && !col.omit_nil?
       push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
       self.status ||= :success_with_warnings
+      @new_status ||= colorize("✓", :orange)
     end
 
     if val.nil? && col.required?
@@ -137,6 +132,10 @@ class SimpleExporter < SimpleInterface
           flatten_hash(v).map do |h_k, h_v|
             h["#{k}.#{h_k}".to_sym] = h_v
           end
+        elsif v.is_a? ActiveRecord::Base
+          flatten_hash(v.attributes).map do |h_k, h_v|
+            h["#{k}.#{h_k}".to_sym] = h_v
+          end
         else
           h[k] = v
         end
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index 6f0b8d7a8..cf45fee37 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -33,7 +33,7 @@ class SimpleImporter < SimpleInterface
   rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
-    self.save!
+    task_finished
   end
 
   def encode_string s
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 11b9d0dcd..c20b70875 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -1,5 +1,5 @@
 class SimpleInterface < ActiveRecord::Base
-  attr_accessor :configuration
+  attr_accessor :configuration, :interfaces_group
 
   class << self
     def configuration_class
@@ -39,6 +39,8 @@ class SimpleInterface < ActiveRecord::Base
     @padding = 1
     @current_line = -1
     @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+    @output_dir = opts[:output_dir] || Rails.root.join('tmp', self.class.name.tableize)
+    @start_time = Time.now
   end
 
   def configure
@@ -70,16 +72,56 @@ class SimpleInterface < ActiveRecord::Base
   def log msg, opts={}
     msg = msg.to_s
     msg = colorize msg, opts[:color] if opts[:color]
+    @start_time ||= Time.now
+    time = Time.now - @start_time
+    @messages ||= []
     if opts[:append]
-      @messages[-1] = (@messages[-1] || "") + msg
+      _time, _msg = @messages.pop || []
+      _time ||= time
+      _msg ||= ""
+      @messages.push [_time, _msg+msg]
+    elsif opts[:replace]
+      @messages.pop
+      @messages << [time, msg]
     else
-      @messages << msg
+      @messages << [time, msg]
+    end
+    print_state true
+  end
+
+  def output_filepath
+    @output_filepath ||= File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+  end
+
+  def write_output_to_csv
+    filepath =
+    cols = %i(line kind event message error)
+    if self.journal.size > 0 && self.journal.first[:row].present?
+      log "Writing output log"
+      FileUtils.mkdir_p @output_dir
+      keys = self.journal.first[:row].map(&:first)
+      CSV.open(output_filepath, "w") do |csv|
+        csv << cols + keys
+        self.journal.each do |j|
+          csv << cols.map{|c| j[c]} + j[:row].map(&:last)
+        end
+      end
+      log "Output written in #{filepath}", replace: true
     end
-    print_state
   end
 
   protected
 
+  def task_finished
+    log "Saving..."
+    self.save!
+    log "Saved", replace: true
+    write_output_to_csv
+    log "FINISHED, status: "
+    log status, color: SimpleInterface.status_color(status), append: true
+    print_state true
+  end
+
   def push_in_journal data
     line = (@current_line || 0) + 1
     line += 1 if configuration.headers
@@ -90,7 +132,7 @@ class SimpleInterface < ActiveRecord::Base
     end
   end
 
-  def colorize txt, color
+  def self.colorize txt, color
     color = {
       red: "31",
       green: "32",
@@ -99,12 +141,24 @@ class SimpleInterface < ActiveRecord::Base
     "\e[#{color}m#{txt}\e[0m"
   end
 
-  def print_state
+  def self.status_color status
+    color = :green
+    color = :orange if status.to_s == "success_with_warnings"
+    color = :red if status.to_s == "error"
+    color
+  end
+
+  def colorize txt, color
+    SimpleInterface.colorize txt, color
+  end
+
+  def print_state force=false
     return unless @verbose
+    return if !@last_repaint.nil? && (Time.now - @last_repaint < 0.5) && !force
 
     @status_width ||= begin
-      term_width = %x(tput cols).to_i
-      term_width - @padding - 10
+      @term_width = %x(tput cols).to_i
+      @term_width - @padding - 10
     rescue
       100
     end
@@ -116,12 +170,24 @@ class SimpleInterface < ActiveRecord::Base
       50
     end
 
+    msg = ""
+
+    if @banner.nil? && interfaces_group.present?
+      @banner = interfaces_group.banner @status_width
+      @status_height -= @banner.lines.count + 2
+    end
+
+    if @banner.present?
+      msg += @banner
+      msg += "\n" + "-"*@term_width + "\n"
+    end
+
     full_status = @statuses || ""
     full_status = full_status.last(@status_width*10) || ""
     padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
     full_status = "#{full_status}#{"."*[padding_size, 0].max}"
 
-    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+    msg += "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
 
     lines_count = [(@status_height / 2) - 3, 1].max
 
@@ -129,7 +195,9 @@ class SimpleInterface < ActiveRecord::Base
       msg += "\n\n"
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
       msg += "[...]\n" if @messages.count > lines_count
-      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
+      msg += @messages.last(lines_count).map do |m|
+        "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width - 10)
+      end.join("\n")
       msg += "\n"*[lines_count-@messages.count, 0].max
     end
 
@@ -146,6 +214,7 @@ class SimpleInterface < ActiveRecord::Base
       end.join("\n")
     end
     custom_print msg, clear: true
+    @last_repaint = Time.now
   end
 
   def custom_print msg, opts={}
diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb
new file mode 100644
index 000000000..808be6570
--- /dev/null
+++ b/app/models/simple_interfaces_group.rb
@@ -0,0 +1,76 @@
+class SimpleInterfacesGroup
+  attr_accessor :name, :shared_options
+
+  def initialize name
+    @name = name
+    @interfaces = []
+    @current_step = 0
+  end
+
+  def add_interface interface, name, action, opts={}
+    @interfaces.push({interface: interface, name: name, action: action, opts: opts})
+  end
+
+  def run
+    @interfaces.each do |interface_def|
+      interface = interface_def[:interface]
+      interface.interfaces_group = self
+      interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options || {})
+      return if interface.status == :error
+      @current_step += 1
+    end
+
+    print_summary
+  end
+
+  def banner width=nil
+    width ||= @width
+    width ||= 128
+    @width = width
+
+    name = "### #{self.name} ###"
+    centered_name = " " * ([width - name.size, 0].max / 2) + name
+    banner = [centered_name, ""]
+    banner << @interfaces.each_with_index.map do |interface, i|
+      if interface[:interface].status.present?
+        SimpleInterface.colorize interface[:name], SimpleInterface.status_color(interface[:interface].status)
+      elsif i == @current_step
+        "☕︎ #{interface[:name]}"
+      else
+        interface[:name]
+      end
+    end.join(' > ')
+    banner.join("\n")
+  end
+
+  def print_summary
+    puts "\e[H\e[2J"
+    out = [banner]
+    out << "-" * @width
+    out << ""
+    out << SimpleInterface.colorize("=== STATUSES ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}"
+    end
+    out << ""
+    out << SimpleInterface.colorize("=== OUTPUTS ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      if i[:interface].is_a? SimpleExporter
+        out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].filepath}"
+      end
+    end
+    out << ""
+    out << ""
+    out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].output_filepath}"
+    end
+    out << ""
+    out << ""
+
+    print out.join("\n")
+  end
+end
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
index 8c44c149a..024d75c97 100644
--- a/app/models/simple_json_exporter.rb
+++ b/app/models/simple_json_exporter.rb
@@ -21,22 +21,18 @@ class SimpleJsonExporter < SimpleExporter
 
     @statuses = ""
 
-    if ENV["NO_TRANSACTION"]
-      process_collection
-    else
-      ActiveRecord::Base.transaction do
-        process_collection
-      end
-    end
+    process_collection
     self.status ||= :success
   rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
     if @file
+      log "Writing to JSON file..."
       @file.write @out.to_json
+      log "JSON file written", replace: true
       @file.close
     end
-    self.save!
+    task_finished
   end
 
   protected
@@ -50,20 +46,22 @@ class SimpleJsonExporter < SimpleExporter
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
 
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      log "Using paginated collection", color: :green
       ids = collection.pluck :id
       ids.in_groups_of(configuration.batch_size).each do |batch_ids|
-        collection.where(id: batch_ids).each do |item|
+        collection.where(id: batch_ids.compact).each do |item|
           handle_item item
         end
       end
     else
+      log "Using non-paginated collection", color: :orange
       collection.each{|item| handle_item item }
     end
-    print_state
+    print_state true
   end
 
   def resolve_node item, node
@@ -96,7 +94,7 @@ class SimpleJsonExporter < SimpleExporter
     map_item_to_rows(item).each_with_index do |item, i|
       @number_of_lines = number_of_lines + i
       serialized_item = {}
-      @current_row = item.attributes
+      @current_row = item.attributes.symbolize_keys
       @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
       @new_status = nil