diff options
Diffstat (limited to 'app')
| -rw-r--r-- | app/models/simple_exporter.rb | 21 | ||||
| -rw-r--r-- | app/models/simple_importer.rb | 2 | ||||
| -rw-r--r-- | app/models/simple_interface.rb | 89 | ||||
| -rw-r--r-- | app/models/simple_interfaces_group.rb | 76 | ||||
| -rw-r--r-- | app/models/simple_json_exporter.rb | 24 |
5 files changed, 177 insertions, 35 deletions
diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb index c1ba75d0a..c267b5b8c 100644 --- a/app/models/simple_exporter.rb +++ b/app/models/simple_exporter.rb @@ -16,19 +16,14 @@ class SimpleExporter < SimpleInterface @statuses = "" - if ENV["NO_TRANSACTION"] - process_collection - else - ActiveRecord::Base.transaction do - process_collection - end - end + process_collection + self.status ||= :success rescue SimpleInterface::FailedOperation self.status = :failed ensure @csv&.close - self.save! + task_finished end def collection @@ -46,14 +41,13 @@ class SimpleExporter < SimpleInterface protected def init_env opts @number_of_lines = collection.size - super opts end def process_collection self.configuration.before_actions(:all).each do |action| action.call self end - log "Starting export ...", color: :green - log "Export will be written in #{filepath}", color: :green + log "Starting export ..." + log "Export will be written in #{filepath}" @csv << self.configuration.columns.map(&:name) if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id") ids = collection.pluck :id @@ -87,6 +81,7 @@ class SimpleExporter < SimpleInterface if val.nil? && !col.omit_nil? push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning}) self.status ||= :success_with_warnings + @new_status ||= colorize("✓", :orange) end if val.nil? && col.required? @@ -137,6 +132,10 @@ class SimpleExporter < SimpleInterface flatten_hash(v).map do |h_k, h_v| h["#{k}.#{h_k}".to_sym] = h_v end + elsif v.is_a? ActiveRecord::Base + flatten_hash(v.attributes).map do |h_k, h_v| + h["#{k}.#{h_k}".to_sym] = h_v + end else h[k] = v end diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb index 6f0b8d7a8..cf45fee37 100644 --- a/app/models/simple_importer.rb +++ b/app/models/simple_importer.rb @@ -33,7 +33,7 @@ class SimpleImporter < SimpleInterface rescue SimpleInterface::FailedOperation self.status = :failed ensure - self.save! + task_finished end def encode_string s diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb index 11b9d0dcd..c20b70875 100644 --- a/app/models/simple_interface.rb +++ b/app/models/simple_interface.rb @@ -1,5 +1,5 @@ class SimpleInterface < ActiveRecord::Base - attr_accessor :configuration + attr_accessor :configuration, :interfaces_group class << self def configuration_class @@ -39,6 +39,8 @@ class SimpleInterface < ActiveRecord::Base @padding = 1 @current_line = -1 @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max + @output_dir = opts[:output_dir] || Rails.root.join('tmp', self.class.name.tableize) + @start_time = Time.now end def configure @@ -70,16 +72,56 @@ class SimpleInterface < ActiveRecord::Base def log msg, opts={} msg = msg.to_s msg = colorize msg, opts[:color] if opts[:color] + @start_time ||= Time.now + time = Time.now - @start_time + @messages ||= [] if opts[:append] - @messages[-1] = (@messages[-1] || "") + msg + _time, _msg = @messages.pop || [] + _time ||= time + _msg ||= "" + @messages.push [_time, _msg+msg] + elsif opts[:replace] + @messages.pop + @messages << [time, msg] else - @messages << msg + @messages << [time, msg] + end + print_state true + end + + def output_filepath + @output_filepath ||= File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" + end + + def write_output_to_csv + filepath = + cols = %i(line kind event message error) + if self.journal.size > 0 && self.journal.first[:row].present? + log "Writing output log" + FileUtils.mkdir_p @output_dir + keys = self.journal.first[:row].map(&:first) + CSV.open(output_filepath, "w") do |csv| + csv << cols + keys + self.journal.each do |j| + csv << cols.map{|c| j[c]} + j[:row].map(&:last) + end + end + log "Output written in #{filepath}", replace: true end - print_state end protected + def task_finished + log "Saving..." + self.save! + log "Saved", replace: true + write_output_to_csv + log "FINISHED, status: " + log status, color: SimpleInterface.status_color(status), append: true + print_state true + end + def push_in_journal data line = (@current_line || 0) + 1 line += 1 if configuration.headers @@ -90,7 +132,7 @@ class SimpleInterface < ActiveRecord::Base end end - def colorize txt, color + def self.colorize txt, color color = { red: "31", green: "32", @@ -99,12 +141,24 @@ class SimpleInterface < ActiveRecord::Base "\e[#{color}m#{txt}\e[0m" end - def print_state + def self.status_color status + color = :green + color = :orange if status.to_s == "success_with_warnings" + color = :red if status.to_s == "error" + color + end + + def colorize txt, color + SimpleInterface.colorize txt, color + end + + def print_state force=false return unless @verbose + return if !@last_repaint.nil? && (Time.now - @last_repaint < 0.5) && !force @status_width ||= begin - term_width = %x(tput cols).to_i - term_width - @padding - 10 + @term_width = %x(tput cols).to_i + @term_width - @padding - 10 rescue 100 end @@ -116,12 +170,24 @@ class SimpleInterface < ActiveRecord::Base 50 end + msg = "" + + if @banner.nil? && interfaces_group.present? + @banner = interfaces_group.banner @status_width + @status_height -= @banner.lines.count + 2 + end + + if @banner.present? + msg += @banner + msg += "\n" + "-"*@term_width + "\n" + end + full_status = @statuses || "" full_status = full_status.last(@status_width*10) || "" padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min full_status = "#{full_status}#{"."*[padding_size, 0].max}" - msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}" + msg += "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}" lines_count = [(@status_height / 2) - 3, 1].max @@ -129,7 +195,9 @@ class SimpleInterface < ActiveRecord::Base msg += "\n\n" msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green msg += "[...]\n" if @messages.count > lines_count - msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n") + msg += @messages.last(lines_count).map do |m| + "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width - 10) + end.join("\n") msg += "\n"*[lines_count-@messages.count, 0].max end @@ -146,6 +214,7 @@ class SimpleInterface < ActiveRecord::Base end.join("\n") end custom_print msg, clear: true + @last_repaint = Time.now end def custom_print msg, opts={} diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb new file mode 100644 index 000000000..808be6570 --- /dev/null +++ b/app/models/simple_interfaces_group.rb @@ -0,0 +1,76 @@ +class SimpleInterfacesGroup + attr_accessor :name, :shared_options + + def initialize name + @name = name + @interfaces = [] + @current_step = 0 + end + + def add_interface interface, name, action, opts={} + @interfaces.push({interface: interface, name: name, action: action, opts: opts}) + end + + def run + @interfaces.each do |interface_def| + interface = interface_def[:interface] + interface.interfaces_group = self + interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options || {}) + return if interface.status == :error + @current_step += 1 + end + + print_summary + end + + def banner width=nil + width ||= @width + width ||= 128 + @width = width + + name = "### #{self.name} ###" + centered_name = " " * ([width - name.size, 0].max / 2) + name + banner = [centered_name, ""] + banner << @interfaces.each_with_index.map do |interface, i| + if interface[:interface].status.present? + SimpleInterface.colorize interface[:name], SimpleInterface.status_color(interface[:interface].status) + elsif i == @current_step + "☕︎ #{interface[:name]}" + else + interface[:name] + end + end.join(' > ') + banner.join("\n") + end + + def print_summary + puts "\e[H\e[2J" + out = [banner] + out << "-" * @width + out << "" + out << SimpleInterface.colorize("=== STATUSES ===", :green) + out << "" + @interfaces.each do |i| + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}" + end + out << "" + out << SimpleInterface.colorize("=== OUTPUTS ===", :green) + out << "" + @interfaces.each do |i| + if i[:interface].is_a? SimpleExporter + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].filepath}" + end + end + out << "" + out << "" + out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green) + out << "" + @interfaces.each do |i| + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].output_filepath}" + end + out << "" + out << "" + + print out.join("\n") + end +end diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb index 8c44c149a..024d75c97 100644 --- a/app/models/simple_json_exporter.rb +++ b/app/models/simple_json_exporter.rb @@ -21,22 +21,18 @@ class SimpleJsonExporter < SimpleExporter @statuses = "" - if ENV["NO_TRANSACTION"] - process_collection - else - ActiveRecord::Base.transaction do - process_collection - end - end + process_collection self.status ||= :success rescue SimpleInterface::FailedOperation self.status = :failed ensure if @file + log "Writing to JSON file..." @file.write @out.to_json + log "JSON file written", replace: true @file.close end - self.save! + task_finished end protected @@ -50,20 +46,22 @@ class SimpleJsonExporter < SimpleExporter def process_collection self.configuration.before_actions(:all).each do |action| action.call self end - log "Starting export ...", color: :green - log "Export will be written in #{filepath}", color: :green + log "Starting export ..." + log "Export will be written in #{filepath}" if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id") + log "Using paginated collection", color: :green ids = collection.pluck :id ids.in_groups_of(configuration.batch_size).each do |batch_ids| - collection.where(id: batch_ids).each do |item| + collection.where(id: batch_ids.compact).each do |item| handle_item item end end else + log "Using non-paginated collection", color: :orange collection.each{|item| handle_item item } end - print_state + print_state true end def resolve_node item, node @@ -96,7 +94,7 @@ class SimpleJsonExporter < SimpleExporter map_item_to_rows(item).each_with_index do |item, i| @number_of_lines = number_of_lines + i serialized_item = {} - @current_row = item.attributes + @current_row = item.attributes.symbolize_keys @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present? @new_status = nil |