From facce065c8cf455ab6e1fe44c6d9394b4f2aaa42 Mon Sep 17 00:00:00 2001
From: Zog
Date: Tue, 6 Mar 2018 10:16:07 +0100
Subject: Refs #6068; Add aggregated output for multiple interfaces

---
 app/models/simple_exporter.rb         |  7 +--
 app/models/simple_importer.rb         |  2 +-
 app/models/simple_interface.rb        | 86 +++++++++++++++++++++++++++++++----
 app/models/simple_interfaces_group.rb | 66 +++++++++++++++++++++++++++
 app/models/simple_json_exporter.rb    | 10 ++--
 5 files changed, 154 insertions(+), 17 deletions(-)
 create mode 100644 app/models/simple_interfaces_group.rb

(limited to 'app')

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index c1ba75d0a..402d450f0 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -28,7 +28,7 @@ class SimpleExporter < SimpleInterface
     self.status = :failed
   ensure
     @csv&.close
-    self.save!
+    task_finished
   end
 
   def collection
@@ -52,8 +52,8 @@ class SimpleExporter < SimpleInterface
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
     @csv << self.configuration.columns.map(&:name)
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
       ids = collection.pluck :id
@@ -87,6 +87,7 @@ class SimpleExporter < SimpleInterface
     if val.nil? && !col.omit_nil?
       push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
       self.status ||= :success_with_warnings
+      @new_status ||= colorize("✓", :orange)
     end
 
     if val.nil? && col.required?
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index e23b3e524..21e01de89 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -32,7 +32,7 @@ class SimpleImporter < SimpleInterface
   rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
-    self.save!
+    task_finished
   end
 
   def encode_string s
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 489419482..e533dc744 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -1,5 +1,5 @@
 class SimpleInterface < ActiveRecord::Base
-  attr_accessor :configuration
+  attr_accessor :configuration, :interfaces_group
 
   class << self
     def configuration_class
@@ -35,6 +35,8 @@ class SimpleInterface < ActiveRecord::Base
     @padding = 1
     @current_line = -1
     @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+    @output_dir = opts[:output_dir] || Rails.root.join('tmp', self.class.name.tableize)
+    @start_time = Time.now
   end
 
   def configure
@@ -66,16 +68,55 @@ class SimpleInterface < ActiveRecord::Base
   def log msg, opts={}
     msg = msg.to_s
     msg = colorize msg, opts[:color] if opts[:color]
+    @start_time ||= Time.now
+    time = Time.now - @start_time
     if opts[:append]
-      @messages[-1] = (@messages[-1] || "") + msg
+      _time, _msg = @messages.pop || []
+      _time ||= time
+      _msg ||= ""
+      @messages.push [_time, _msg+msg]
+    elsif opts[:replace]
+      @messages.pop
+      @messages << [time, msg]
     else
-      @messages << msg
+      @messages << [time, msg]
     end
     print_state
   end
 
+  def output_filepath
+    File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+  end
+
+  def write_output_to_csv
+    filepath =
+    cols = %i(line kind event message error)
+    if self.journal.size > 0 && self.journal.first[:row].present?
+      log "Writing output log"
+      FileUtils.mkdir_p @output_dir
+      keys = self.journal.first[:row].map(&:first)
+      CSV.open(output_filepath, "w") do |csv|
+        csv << cols + keys
+        self.journal.each do |j|
+          csv << cols.map{|c| j[c]} + j[:row].map(&:last)
+        end
+      end
+      log "Output written in #{filepath}", replace: true
+    end
+  end
+
   protected
 
+  def task_finished
+    log "Saving..."
+    self.save!
+    log "Saved", replace: true
+    write_output_to_csv
+    log "FINISHED, status: "
+    log status, color: SimpleInterface.status_color(status), append: true
+    print_state true
+  end
+
   def push_in_journal data
     line = (@current_line || 0) + 1
     line += 1 if configuration.headers
@@ -86,7 +127,7 @@ class SimpleInterface < ActiveRecord::Base
     end
   end
 
-  def colorize txt, color
+  def self.colorize txt, color
     color = {
       red: "31",
       green: "32",
@@ -95,12 +136,24 @@ class SimpleInterface < ActiveRecord::Base
     "\e[#{color}m#{txt}\e[0m"
   end
 
-  def print_state
+  def self.status_color status
+    color = :green
+    color = :orange if status.to_s == "success_with_warnings"
+    color = :red if status.to_s == "error"
+    color
+  end
+
+  def colorize txt, color
+    SimpleInterface.colorize txt, color
+  end
+
+  def print_state force=false
     return unless @verbose
+    return if !@last_repaint.nil? && (Time.now - @last_repaint < 0.5) && !force
 
     @status_width ||= begin
-      term_width = %x(tput cols).to_i
-      term_width - @padding - 10
+      @term_width = %x(tput cols).to_i
+      @term_width - @padding - 10
     rescue
       100
     end
@@ -112,12 +165,24 @@ class SimpleInterface < ActiveRecord::Base
       50
     end
 
+    msg = ""
+
+    if @banner.nil? && interfaces_group.present?
+      @banner = interfaces_group.banner @status_width
+      @status_height -= @banner.lines.count + 2
+    end
+
+    if @banner.present?
+      msg += @banner
+      msg += "\n" + "-"*@term_width + "\n"
+    end
+
     full_status = @statuses || ""
     full_status = full_status.last(@status_width*10) || ""
     padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
     full_status = "#{full_status}#{"."*[padding_size, 0].max}"
 
-    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+    msg += "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
 
     lines_count = [(@status_height / 2) - 3, 1].max
 
@@ -125,7 +190,9 @@ class SimpleInterface < ActiveRecord::Base
       msg += "\n\n"
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
       msg += "[...]\n" if @messages.count > lines_count
-      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
+      msg += @messages.last(lines_count).map do |m|
+        "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width)
+      end.join("\n")
       msg += "\n"*[lines_count-@messages.count, 0].max
     end
 
@@ -142,6 +209,7 @@ class SimpleInterface < ActiveRecord::Base
       end.join("\n")
     end
     custom_print msg, clear: true
+    @last_repaint = Time.now
   end
 
   def custom_print msg, opts={}
diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb
new file mode 100644
index 000000000..2c931d793
--- /dev/null
+++ b/app/models/simple_interfaces_group.rb
@@ -0,0 +1,66 @@
+class SimpleInterfacesGroup
+  attr_accessor :name, :shared_options
+
+  def initialize name
+    @name = name
+    @interfaces = []
+    @current_step = 0
+  end
+
+  def add_interface interface, name, action, opts={}
+    @interfaces.push({interface: interface, name: name, action: action, opts: opts})
+  end
+
+  def run
+    @interfaces.each do |interface_def|
+      interface = interface_def[:interface]
+      interface.interfaces_group = self
+      interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options)
+      return if interface.status == :error
+      @current_step += 1
+    end
+
+    print_summary
+  end
+
+  def banner width=nil
+    width ||= @width
+    @width = width
+
+    name = "### #{self.name} ###"
+    centered_name = " " * ([width - name.size, 0].max / 2) + name
+    banner = [centered_name, ""]
+    banner << @interfaces.each_with_index.map do |interface, i|
+      if interface[:interface].status.present?
+        SimpleInterface.colorize interface[:name], SimpleInterface.status_color(interface[:interface].status)
+      elsif i == @current_step
+        "☕︎ #{interface[:name]}"
+      else
+        interface[:name]
+      end
+    end.join(' > ')
+    banner.join("\n")
+  end
+
+  def print_summary
+    puts "\e[H\e[2J"
+    out = [banner]
+    out << "-" * @width
+    out << ""
+    out << SimpleInterface.colorize("=== STATUSES ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}"
+    end
+    out << ""
+    out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].output_filepath}"
+    end
+    out << ""
+    out << ""
+
+    print out.join("\n")
+  end
+end
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
index 8c44c149a..6402efea4 100644
--- a/app/models/simple_json_exporter.rb
+++ b/app/models/simple_json_exporter.rb
@@ -33,10 +33,12 @@ class SimpleJsonExporter < SimpleExporter
     self.status = :failed
   ensure
     if @file
+      log "Writing to JSON file..."
       @file.write @out.to_json
+      log "JSON file written", replace: true
       @file.close
     end
-    self.save!
+    task_finished
   end
 
   protected
@@ -50,8 +52,8 @@ class SimpleJsonExporter < SimpleExporter
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
 
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
       ids = collection.pluck :id
@@ -63,7 +65,7 @@ class SimpleJsonExporter < SimpleExporter
     else
       collection.each{|item| handle_item item }
     end
-    print_state
+    print_state true
   end
 
   def resolve_node item, node
-- 
cgit v1.2.3


From 4f7f0a974b432c01493c265dcf338ddd488b6427 Mon Sep 17 00:00:00 2001
From: Zog
Date: Tue, 6 Mar 2018 10:50:46 +0100
Subject: Refs #6068; Remove useless transactions

---
 app/models/simple_exporter.rb      | 13 ++++++-------
 app/models/simple_interface.rb     |  6 +++---
 app/models/simple_json_exporter.rb | 14 +++++---------
 3 files changed, 14 insertions(+), 19 deletions(-)

(limited to 'app')

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index 402d450f0..93a773430 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -16,13 +16,8 @@ class SimpleExporter < SimpleInterface
 
     @statuses = ""
 
-    if ENV["NO_TRANSACTION"]
-      process_collection
-    else
-      ActiveRecord::Base.transaction do
-        process_collection
-      end
-    end
+    process_collection
+
     self.status ||= :success
   rescue SimpleInterface::FailedOperation
     self.status = :failed
@@ -138,6 +133,10 @@ class SimpleExporter < SimpleInterface
           flatten_hash(v).map do |h_k, h_v|
             h["#{k}.#{h_k}".to_sym] = h_v
           end
+        elsif v.is_a? ActiveRecord::Base
+          flatten_hash(v.attributes).map do |h_k, h_v|
+            h["#{k}.#{h_k}".to_sym] = h_v
+          end
         else
           h[k] = v
         end
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index e533dc744..5997a6dd3 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -81,11 +81,11 @@ class SimpleInterface < ActiveRecord::Base
     else
       @messages << [time, msg]
     end
-    print_state
+    print_state true
   end
 
   def output_filepath
-    File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+    @output_filepath ||= File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
   end
 
   def write_output_to_csv
@@ -191,7 +191,7 @@ class SimpleInterface < ActiveRecord::Base
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
       msg += "[...]\n" if @messages.count > lines_count
       msg += @messages.last(lines_count).map do |m|
-        "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width)
+        "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width - 10)
       end.join("\n")
       msg += "\n"*[lines_count-@messages.count, 0].max
     end
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
index 6402efea4..024d75c97 100644
--- a/app/models/simple_json_exporter.rb
+++ b/app/models/simple_json_exporter.rb
@@ -21,13 +21,7 @@ class SimpleJsonExporter < SimpleExporter
 
     @statuses = ""
 
-    if ENV["NO_TRANSACTION"]
-      process_collection
-    else
-      ActiveRecord::Base.transaction do
-        process_collection
-      end
-    end
+    process_collection
     self.status ||= :success
   rescue SimpleInterface::FailedOperation
     self.status = :failed
@@ -56,13 +50,15 @@ class SimpleJsonExporter < SimpleExporter
     log "Export will be written in #{filepath}"
 
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      log "Using paginated collection", color: :green
       ids = collection.pluck :id
       ids.in_groups_of(configuration.batch_size).each do |batch_ids|
-        collection.where(id: batch_ids).each do |item|
+        collection.where(id: batch_ids.compact).each do |item|
           handle_item item
         end
       end
     else
+      log "Using non-paginated collection", color: :orange
       collection.each{|item| handle_item item }
     end
     print_state true
@@ -98,7 +94,7 @@ class SimpleJsonExporter < SimpleExporter
     map_item_to_rows(item).each_with_index do |item, i|
       @number_of_lines = number_of_lines + i
       serialized_item = {}
-      @current_row = item.attributes
+      @current_row = item.attributes.symbolize_keys
       @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
       @new_status = nil
 
-- 
cgit v1.2.3


From c322f76744e68185766b0962066d69f7dc4d9629 Mon Sep 17 00:00:00 2001
From: Zog
Date: Tue, 6 Mar 2018 11:13:11 +0100
Subject: Refs #6068; Better output

---
 app/models/simple_interfaces_group.rb | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'app')

diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb
index 2c931d793..b89a1b696 100644
--- a/app/models/simple_interfaces_group.rb
+++ b/app/models/simple_interfaces_group.rb
@@ -53,6 +53,15 @@ class SimpleInterfacesGroup
       out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}"
     end
     out << ""
+    out << SimpleInterface.colorize("=== OUTPUTS ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      if i[:interface].is_a? SimpleExporter
+        out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].filepath}"
+      end
+    end
+    out << ""
+    out << ""
     out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green)
     out << ""
     @interfaces.each do |i|
-- 
cgit v1.2.3


From b7079226c965e7130a2bcc17b7f278a1e23ac7e8 Mon Sep 17 00:00:00 2001
From: Zog
Date: Tue, 6 Mar 2018 11:49:11 +0100
Subject: Refs #6068; Fix specs

---
 app/models/simple_exporter.rb         | 1 -
 app/models/simple_interface.rb        | 1 +
 app/models/simple_interfaces_group.rb | 3 ++-
 3 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index 93a773430..c267b5b8c 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -41,7 +41,6 @@ class SimpleExporter < SimpleInterface
   protected
   def init_env opts
     @number_of_lines = collection.size
-
     super opts
   end
 
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 5997a6dd3..16d6e86f2 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -70,6 +70,7 @@ class SimpleInterface < ActiveRecord::Base
     msg = colorize msg, opts[:color] if opts[:color]
     @start_time ||= Time.now
     time = Time.now - @start_time
+    @messages ||= []
     if opts[:append]
       _time, _msg = @messages.pop || []
       _time ||= time
diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb
index b89a1b696..808be6570 100644
--- a/app/models/simple_interfaces_group.rb
+++ b/app/models/simple_interfaces_group.rb
@@ -15,7 +15,7 @@ class SimpleInterfacesGroup
     @interfaces.each do |interface_def|
       interface = interface_def[:interface]
       interface.interfaces_group = self
-      interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options)
+      interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options || {})
       return if interface.status == :error
       @current_step += 1
     end
@@ -25,6 +25,7 @@ class SimpleInterfacesGroup
 
   def banner width=nil
     width ||= @width
+    width ||= 128
     @width = width
 
     name = "### #{self.name} ###"
-- 
cgit v1.2.3