Refs #6068; Add aggregated output for multiple interfaces

8 files changed, 196 insertions, 44 deletions

diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index c1ba75d0a..402d450f0 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -28,7 +28,7 @@ class SimpleExporter < SimpleInterface
     self.status = :failed
   ensure
     @csv&.close
-    self.save!
+    task_finished
   end
 
   def collection
@@ -52,8 +52,8 @@ class SimpleExporter < SimpleInterface
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
     @csv << self.configuration.columns.map(&:name)
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
       ids = collection.pluck :id
@@ -87,6 +87,7 @@ class SimpleExporter < SimpleInterface
     if val.nil? && !col.omit_nil?
       push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
       self.status ||= :success_with_warnings
+      @new_status ||= colorize("✓", :orange)
     end
 
     if val.nil? && col.required?
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index e23b3e524..21e01de89 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -32,7 +32,7 @@ class SimpleImporter < SimpleInterface
   rescue SimpleInterface::FailedOperation
     self.status = :failed
   ensure
-    self.save!
+    task_finished
   end
 
   def encode_string s
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 489419482..e533dc744 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -1,5 +1,5 @@
 class SimpleInterface < ActiveRecord::Base
-  attr_accessor :configuration
+  attr_accessor :configuration, :interfaces_group
 
   class << self
     def configuration_class
@@ -35,6 +35,8 @@ class SimpleInterface < ActiveRecord::Base
     @padding = 1
     @current_line = -1
     @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max
+    @output_dir = opts[:output_dir] || Rails.root.join('tmp', self.class.name.tableize)
+    @start_time = Time.now
   end
 
   def configure
@@ -66,16 +68,55 @@ class SimpleInterface < ActiveRecord::Base
   def log msg, opts={}
     msg = msg.to_s
     msg = colorize msg, opts[:color] if opts[:color]
+    @start_time ||= Time.now
+    time = Time.now - @start_time
     if opts[:append]
-      @messages[-1] = (@messages[-1] || "") + msg
+      _time, _msg = @messages.pop || []
+      _time ||= time
+      _msg ||= ""
+      @messages.push [_time, _msg+msg]
+    elsif opts[:replace]
+      @messages.pop
+      @messages << [time, msg]
     else
-      @messages << msg
+      @messages << [time, msg]
     end
     print_state
   end
 
+  def output_filepath
+    File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
+  end
+
+  def write_output_to_csv
+    filepath =
+    cols = %i(line kind event message error)
+    if self.journal.size > 0 && self.journal.first[:row].present?
+      log "Writing output log"
+      FileUtils.mkdir_p @output_dir
+      keys = self.journal.first[:row].map(&:first)
+      CSV.open(output_filepath, "w") do |csv|
+        csv << cols + keys
+        self.journal.each do |j|
+          csv << cols.map{|c| j[c]} + j[:row].map(&:last)
+        end
+      end
+      log "Output written in #{filepath}", replace: true
+    end
+  end
+
   protected
 
+  def task_finished
+    log "Saving..."
+    self.save!
+    log "Saved", replace: true
+    write_output_to_csv
+    log "FINISHED, status: "
+    log status, color: SimpleInterface.status_color(status), append: true
+    print_state true
+  end
+
   def push_in_journal data
     line = (@current_line || 0) + 1
     line += 1 if configuration.headers
@@ -86,7 +127,7 @@ class SimpleInterface < ActiveRecord::Base
     end
   end
 
-  def colorize txt, color
+  def self.colorize txt, color
     color = {
       red: "31",
       green: "32",
@@ -95,12 +136,24 @@ class SimpleInterface < ActiveRecord::Base
     "\e[#{color}m#{txt}\e[0m"
   end
 
-  def print_state
+  def self.status_color status
+    color = :green
+    color = :orange if status.to_s == "success_with_warnings"
+    color = :red if status.to_s == "error"
+    color
+  end
+
+  def colorize txt, color
+    SimpleInterface.colorize txt, color
+  end
+
+  def print_state force=false
     return unless @verbose
+    return if !@last_repaint.nil? && (Time.now - @last_repaint < 0.5) && !force
 
     @status_width ||= begin
-      term_width = %x(tput cols).to_i
-      term_width - @padding - 10
+      @term_width = %x(tput cols).to_i
+      @term_width - @padding - 10
     rescue
       100
     end
@@ -112,12 +165,24 @@ class SimpleInterface < ActiveRecord::Base
       50
     end
 
+    msg = ""
+
+    if @banner.nil? && interfaces_group.present?
+      @banner = interfaces_group.banner @status_width
+      @status_height -= @banner.lines.count + 2
+    end
+
+    if @banner.present?
+      msg += @banner
+      msg += "\n" + "-"*@term_width + "\n"
+    end
+
     full_status = @statuses || ""
     full_status = full_status.last(@status_width*10) || ""
     padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
     full_status = "#{full_status}#{"."*[padding_size, 0].max}"
 
-    msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+    msg += "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
 
     lines_count = [(@status_height / 2) - 3, 1].max
 
@@ -125,7 +190,9 @@ class SimpleInterface < ActiveRecord::Base
       msg += "\n\n"
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
       msg += "[...]\n" if @messages.count > lines_count
-      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
+      msg += @messages.last(lines_count).map do |m|
+        "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width)
+      end.join("\n")
       msg += "\n"*[lines_count-@messages.count, 0].max
     end
 
@@ -142,6 +209,7 @@ class SimpleInterface < ActiveRecord::Base
       end.join("\n")
     end
     custom_print msg, clear: true
+    @last_repaint = Time.now
   end
 
   def custom_print msg, opts={}
diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb
new file mode 100644
index 000000000..2c931d793
--- /dev/null
+++ b/app/models/simple_interfaces_group.rb
@@ -0,0 +1,66 @@
+class SimpleInterfacesGroup
+  attr_accessor :name, :shared_options
+
+  def initialize name
+    @name = name
+    @interfaces = []
+    @current_step = 0
+  end
+
+  def add_interface interface, name, action, opts={}
+    @interfaces.push({interface: interface, name: name, action: action, opts: opts})
+  end
+
+  def run
+    @interfaces.each do |interface_def|
+      interface = interface_def[:interface]
+      interface.interfaces_group = self
+      interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options)
+      return if interface.status == :error
+      @current_step += 1
+    end
+
+    print_summary
+  end
+
+  def banner width=nil
+    width ||= @width
+    @width = width
+
+    name = "### #{self.name} ###"
+    centered_name = " " * ([width - name.size, 0].max / 2) + name
+    banner = [centered_name, ""]
+    banner << @interfaces.each_with_index.map do |interface, i|
+      if interface[:interface].status.present?
+        SimpleInterface.colorize interface[:name], SimpleInterface.status_color(interface[:interface].status)
+      elsif i == @current_step
+        "☕︎ #{interface[:name]}"
+      else
+        interface[:name]
+      end
+    end.join(' > ')
+    banner.join("\n")
+  end
+
+  def print_summary
+    puts "\e[H\e[2J"
+    out = [banner]
+    out << "-" * @width
+    out << ""
+    out << SimpleInterface.colorize("=== STATUSES ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}"
+    end
+    out << ""
+    out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green)
+    out << ""
+    @interfaces.each do |i|
+      out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].output_filepath}"
+    end
+    out << ""
+    out << ""
+
+    print out.join("\n")
+  end
+end
diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb
index 8c44c149a..6402efea4 100644
--- a/app/models/simple_json_exporter.rb
+++ b/app/models/simple_json_exporter.rb
@@ -33,10 +33,12 @@ class SimpleJsonExporter < SimpleExporter
     self.status = :failed
   ensure
     if @file
+      log "Writing to JSON file..."
       @file.write @out.to_json
+      log "JSON file written", replace: true
       @file.close
     end
-    self.save!
+    task_finished
   end
 
   protected
@@ -50,8 +52,8 @@ class SimpleJsonExporter < SimpleExporter
 
   def process_collection
     self.configuration.before_actions(:all).each do |action| action.call self end
-    log "Starting export ...", color: :green
-    log "Export will be written in #{filepath}", color: :green
+    log "Starting export ..."
+    log "Export will be written in #{filepath}"
 
     if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
       ids = collection.pluck :id
@@ -63,7 +65,7 @@ class SimpleJsonExporter < SimpleExporter
     else
       collection.each{|item| handle_item item }
     end
-    print_state
+    print_state true
   end
 
   def resolve_node item, node
diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake
index 547388b35..845d581d3 100644
--- a/lib/tasks/exports.rake
+++ b/lib/tasks/exports.rake
@@ -51,6 +51,9 @@ namespace :export do
     if journeys.count == 0
       puts "No maching journeys were found".red
     else
+      exports_group = SimpleInterfacesGroup.new "Export Complet \"#{referential.name}\" du #{Time.now.to_date} au #{args[:timelapse].to_i.days.from_now.to_date}"
+      exports_group.shared_options = {verbose: true}
+
       exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_companies", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_companies.json"
       ids = journeys.pluck :company_id
       ids += journeys.joins(route: :line).pluck :"lines.company_id"
@@ -59,39 +62,37 @@ namespace :export do
         config.collection = Chouette::Company.where(id: ids.uniq).order('name')
       end
 
-      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
-      break if exporter.status == :error
+      exports_group.add_interface exporter, "Services Types", :export
 
       exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_schedules", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_schedules.json"
       exporter.configure do |config|
         config.collection = journeys
       end
 
-      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
-      break if exporter.status == :error
+      exports_group.add_interface exporter, "Schedules", :export
 
       exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_routes", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_routes.json"
       exporter.configure do |config|
         config.collection = Chouette::JourneyPattern.where(id: journeys.pluck(:journey_pattern_id).uniq)
       end
 
-      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
-      break if exporter.status == :error
+      exports_group.add_interface exporter, "Routes", :export
 
       exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_stops", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_stops.json"
       exporter.configure do |config|
         config.collection = Chouette::StopArea.where(id: journeys.joins(:stop_points).pluck(:"stop_points.stop_area_id").uniq).order('parent_id ASC NULLS FIRST')
       end
 
-      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
-      break if exporter.status == :error
+      exports_group.add_interface exporter, "Stops", :export
 
       exporter = SimpleJsonExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.json"
       exporter.configure do |config|
         config.collection = journeys
       end
 
-      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
+      exports_group.add_interface exporter, "Services", :export
+
+      exports_group.run
     end
   end
 end
diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb
index 5b593be43..61dd38399 100644
--- a/lib/tasks/helpers/simple_interfaces.rb
+++ b/lib/tasks/helpers/simple_interfaces.rb
@@ -1,28 +1,11 @@
 module SimpleInterfacesHelper
-  def self.interface_output_to_csv interface, output_dir
-    FileUtils.mkdir_p output_dir
-    filepath =  File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
-    cols = %w(line kind event message error)
-    if interface.reload.journal.size > 0 && interface.journal.first["row"].present?
-      keys = interface.journal.first["row"].map(&:first)
-      CSV.open(filepath, "w") do |csv|
-        csv << cols + keys
-        interface.journal.each do |j|
-          csv << cols.map{|c| j[c]} + j["row"].map(&:last)
-        end
-      end
-      puts "Task Output written in #{filepath}"
-    end
-  end
-
   def self.run_interface_controlling_interruption interface, method, args
     begin
       interface.send(method, verbose: true)
     rescue Interrupt
+      interface.write_output_to_csv
       raise
     ensure
-      puts "\n\e[33m***\e[0m Done, status: " + (interface.status == "success" ? "\e[32m" : "\e[31m" ) + (interface.status || "") + "\e[0m"
-      interface_output_to_csv interface, args[:logs_output_dir]
     end
   end
 end
diff --git a/spec/models/simple_interfaces_group_spec.rb b/spec/models/simple_interfaces_group_spec.rb
new file mode 100644
index 000000000..65ce9a1bf
--- /dev/null
+++ b/spec/models/simple_interfaces_group_spec.rb
@@ -0,0 +1,31 @@
+RSpec.describe SimpleInterfacesGroup do
+  context "with successful interfaces" do
+    before do
+      create :stop_area
+      SimpleExporter.define :test_1 do |config|
+        config.collection = Chouette::StopArea
+        config.key = "name"
+        config.add_column :name
+      end
+
+      SimpleExporter.define :test_2 do |config|
+        config.collection = Chouette::StopArea
+        config.key = "name"
+        config.add_column :lat, attribute: :latitude
+      end
+    end
+
+    it "should run all interfaces" do
+      test_1 = SimpleExporter.new(configuration_name: :test_1, filepath: "tmp/test1.csv")
+      test_2 = SimpleExporter.new(configuration_name: :test_2, filepath: "tmp/test1.csv")
+
+      expect(test_1).to receive(:export)
+      expect(test_2).to receive(:export)
+
+      group = SimpleInterfacesGroup.new "group"
+      group.add_interface test_1, "Test 1", :export
+      group.add_interface test_2, "Test 1", :export
+      group.run
+    end
+  end
+end