diff options
| -rw-r--r-- | app/models/simple_exporter.rb | 63 | ||||
| -rw-r--r-- | app/models/simple_importer.rb | 13 | ||||
| -rw-r--r-- | app/models/simple_interface.rb | 10 | ||||
| -rw-r--r-- | app/models/simple_json_exporter.rb | 164 | ||||
| -rw-r--r-- | spec/models/simple_exporter_spec.rb | 6 | ||||
| -rw-r--r-- | spec/models/simple_json_exporter_spec.rb | 95 |
6 files changed, 309 insertions, 42 deletions
diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb index fe0aa05b5..adc48533f 100644 --- a/app/models/simple_exporter.rb +++ b/app/models/simple_exporter.rb @@ -1,17 +1,8 @@ class SimpleExporter < SimpleInterface def export opts={} configuration.validate! - @verbose = opts.delete :verbose - - @resolution_queue = Hash.new{|h,k| h[k] = []} - @errors = [] - @messages = [] - @number_of_lines = 0 - @padding = 1 - @current_line = -1 - @number_of_lines = collection.size - @padding = [1, Math.log(@number_of_lines, 10).ceil()].max + init_env opts @csv = nil fail_with_error "Unable to write in file: #{self.filepath}" do @@ -52,6 +43,12 @@ class SimpleExporter < SimpleInterface end protected + def init_env opts + @number_of_lines = collection.size + + super opts + end + def process_collection self.configuration.before_actions(:all).each do |action| action.call self end log "Starting export ...", color: :green @@ -75,6 +72,31 @@ class SimpleExporter < SimpleInterface configuration.item_to_rows_mapping.call(item).map {|row| row.is_a?(ActiveRecord::Base) ? row : CustomRow.new(row) } end + def resolve_value item, col + scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)} + val = col[:value] + if val.nil? || val.is_a?(Proc) + if val.is_a?(Proc) + val = instance_exec(scoped_item, &val) + else + attributes = [col.attribute].flatten + val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)} + end + end + if val.nil? + push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning}) + self.status ||= :success_with_warnings + end + + if val.nil? && col.required? + @new_status = colorize("x", :red) + raise "MISSING VALUE FOR COLUMN #{col.name}" + end + + val = encode_string(val) if val.is_a?(String) + val + end + def handle_item item number_of_lines = @number_of_lines map_item_to_rows(item).each_with_index do |item, i| @@ -84,27 +106,8 @@ class SimpleExporter < SimpleInterface row = [] @new_status = nil self.configuration.columns.each do |col| - scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)} - val = col[:value] - if val.nil? || val.is_a?(Proc) - if val.is_a?(Proc) - val = instance_exec(scoped_item, &val) - else - attributes = [col.attribute].flatten - val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)} - end - end - if val.nil? - push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning}) - self.status ||= :success_with_warnings - end - - if val.nil? && col.required? - @new_status = colorize("x", :red) - raise "MISSING VALUE FOR COLUMN #{col.name}" - end + val = resolve_value item, col @new_status ||= colorize("✓", :green) - val = encode_string(val) if val.is_a?(String) row << val end push_in_journal({event: :success, kind: :log}) diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb index 6db71797a..e23b3e524 100644 --- a/app/models/simple_importer.rb +++ b/app/models/simple_importer.rb @@ -1,5 +1,4 @@ class SimpleImporter < SimpleInterface - def resolve col_name, value, &block val = block.call(value) return val if val.present? @@ -9,19 +8,15 @@ class SimpleImporter < SimpleInterface def import opts={} configuration.validate! - @verbose = opts.delete :verbose - @resolution_queue = Hash.new{|h,k| h[k] = []} - @errors = [] - @messages = [] - @number_of_lines = 0 - @padding = 1 - @current_line = 0 fail_with_error "File not found: #{self.filepath}" do @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length - @padding = [1, Math.log(@number_of_lines, 10).ceil()].max end + init_env opts + + @resolution_queue = Hash.new{|h,k| h[k] = []} + self.configuration.before_actions(:parsing).each do |action| action.call self end @statuses = "" diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb index 5f022719a..07fabd832 100644 --- a/app/models/simple_interface.rb +++ b/app/models/simple_interface.rb @@ -27,6 +27,16 @@ class SimpleInterface < ActiveRecord::Base self.journal ||= [] end + def init_env opts + @verbose = opts.delete :verbose + + @errors = [] + @messages = [] + @padding = 1 + @current_line = 0 + @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max + end + def configure new_config = configuration.duplicate yield new_config diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb new file mode 100644 index 000000000..706307de1 --- /dev/null +++ b/app/models/simple_json_exporter.rb @@ -0,0 +1,164 @@ +class SimpleJsonExporter < SimpleExporter + + def export opts={} + configuration.validate! + + init_env opts + + if self.configuration.root + @out = {self.configuration.root => []} + else + @out = [] + end + + fail_with_error "Unable to write in file: #{self.filepath}" do + dir = Pathname.new(self.filepath).dirname + FileUtils.mkdir_p dir + @file = File.open(self.filepath, 'w', self.configuration.file_options) + end + + self.configuration.before_actions(:parsing).each do |action| action.call self end + + @statuses = "" + + if ENV["NO_TRANSACTION"] + process_collection + else + ActiveRecord::Base.transaction do + process_collection + end + end + self.status ||= :success + rescue SimpleInterface::FailedOperation + self.status = :failed + ensure + if @file + @file.write @out.to_json + @file.close + end + self.save! + end + + protected + def root + if self.configuration.root + @out[self.configuration.root] + else + @out + end + end + + def process_collection + self.configuration.before_actions(:all).each do |action| action.call self end + log "Starting export ...", color: :green + log "Export will be written in #{filepath}", color: :green + + if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id") + ids = collection.pluck :id + ids.in_groups_of(configuration.batch_size).each do |batch_ids| + collection.where(id: batch_ids).each do |item| + handle_item item + end + end + else + collection.each{|item| handle_item item } + end + print_state + end + + def resolve_node item, node + vals = [] + [item.send(node.attribute)].flatten.each do |node_item| + item_val = {} + apply_configuration node_item, node.configuration, item_val + vals.push item_val + end + node.multiple ? vals : vals.first + end + + def apply_configuration item, configuration, output + configuration.columns.each do |col| + val = resolve_value item, col + output[col.name] = val + end + + configuration.nodes.each do |node| + val = resolve_node item, node + output[node.name] = val + end + end + + def handle_item item + serialized_item = {} + @current_row = item.attributes + @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present? + @new_status = nil + + apply_configuration item, self.configuration, serialized_item + + @new_status ||= colorize("✓", :green) + + push_in_journal({event: :success, kind: :log}) + @statuses += @new_status + print_state if @current_line % 20 == 0 + @current_line += 1 + append_item serialized_item + end + + def append_item serialized_item + root.push serialized_item + end + + class Configuration < SimpleExporter::Configuration + attr_reader :nodes + attr_accessor :root + + alias_method :add_field, :add_column + + def initialize import_name, opts={} + @nodes = [] + super import_name, opts + end + + def add_node name, opts={} + @nodes ||= [] + node = Node.new({name: name.to_s}.update(opts)) + yield node.configuration + @nodes.push node + end + + def add_nodes name, opts={}, &block + self.add_node name, opts.update({multiple: true}), &block + end + + def file_options + { + encoding: self.encoding + } + end + end + + class NodeConfiguration < Configuration + def initialize node + super + end + end + + class Node + attr_accessor :name, :configuration + + def initialize opts={} + @name = opts[:name] + @options = opts + @configuration = NodeConfiguration.new self + end + + def attribute + name + end + + def multiple + !!@options[:multiple] + end + end +end diff --git a/spec/models/simple_exporter_spec.rb b/spec/models/simple_exporter_spec.rb index 18cdc55e8..75051aeb9 100644 --- a/spec/models/simple_exporter_spec.rb +++ b/spec/models/simple_exporter_spec.rb @@ -19,7 +19,7 @@ RSpec.describe SimpleExporter do expect{SimpleExporter.find_configuration(:foo)}.to_not raise_error expect{SimpleExporter.new(configuration_name: :foo, filepath: "").export}.to_not raise_error expect{SimpleExporter.find_configuration(:bar)}.to raise_error - expect{SimpleExporter.new(configuration_name: :bar, filepath: "")}.to_not raise_error + expect{SimpleExporter.new(configuration_name: :bar, filepath: "")}.to raise_error expect{SimpleExporter.new(configuration_name: :bar, filepath: "").export}.to raise_error expect{SimpleExporter.create(configuration_name: :foo, filepath: "")}.to change{SimpleExporter.count}.by 1 end @@ -64,7 +64,7 @@ RSpec.describe SimpleExporter do end it "should export the given file" do - expect{exporter.export verbose: true}.to_not raise_error + expect{exporter.export verbose: false}.to_not raise_error expect(exporter.status).to eq "success" expect(File.exists?(filepath)).to be_truthy csv = CSV.read(filepath, headers: true, col_sep: ";") @@ -102,7 +102,7 @@ RSpec.describe SimpleExporter do end it "should export the given file" do - expect{exporter.export verbose: true}.to_not raise_error + expect{exporter.export verbose: false}.to_not raise_error expect(exporter.status).to eq "success" expect(File.exists?(filepath)).to be_truthy csv = CSV.read(filepath, headers: true) diff --git a/spec/models/simple_json_exporter_spec.rb b/spec/models/simple_json_exporter_spec.rb new file mode 100644 index 000000000..4b48dc732 --- /dev/null +++ b/spec/models/simple_json_exporter_spec.rb @@ -0,0 +1,95 @@ +RSpec.describe SimpleJsonExporter do + describe "#define" do + context "with an incomplete configuration" do + it "should raise an error" do + SimpleJsonExporter.define :foo + expect do + SimpleJsonExporter.new(configuration_name: :test).export + end.to raise_error + end + end + context "with a complete configuration" do + before do + SimpleJsonExporter.define :foo do |config| + config.collection = Chouette::StopArea.all + end + end + + it "should define an exporter" do + expect{SimpleJsonExporter.find_configuration(:foo)}.to_not raise_error + expect{SimpleJsonExporter.new(configuration_name: :foo, filepath: "").export}.to_not raise_error + expect{SimpleJsonExporter.find_configuration(:bar)}.to raise_error + expect{SimpleJsonExporter.new(configuration_name: :bar, filepath: "")}.to raise_error + expect{SimpleJsonExporter.new(configuration_name: :bar, filepath: "").export}.to raise_error + expect{SimpleJsonExporter.create(configuration_name: :foo, filepath: "")}.to change{SimpleJsonExporter.count}.by 1 + end + end + + context "when defining the same col twice" do + it "should raise an error" do + expect do + SimpleJsonExporter.define :foo do |config| + config.collection = Chouette::StopArea.all + config.add_field :name + config.add_field :name + end + end.to raise_error + end + end + end + + describe "#export" do + let(:exporter){ importer = SimpleJsonExporter.new(configuration_name: :test, filepath: filepath) } + let(:filepath){ Rails.root + "tmp/" + filename } + let(:filename){ "stop_area.json" } + # let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) } + + context "with one row per item" do + before(:each) do + @stop_area = create :stop_area + @stop_1 = create :stop_point, stop_area: @stop_area + @stop_2 = create :stop_point, stop_area: @stop_area + + SimpleJsonExporter.define :test do |config| + config.collection = ->{ Chouette::StopArea.all } + config.root = "stops" + config.add_field :name + config.add_field :lat, attribute: :latitude + config.add_field :lng, attribute: [:latitude, :to_i, :next] + config.add_field :type, attribute: :area_type + config.add_field :street_name, value: "Lil Exporter" + config.add_node :stop_area_referential do |config| + config.add_field :id, attribute: :id + config.add_field :id_other, value: ->(item){ item.id } + end + + config.add_nodes :stop_points do |config| + config.add_field :id + config.add_node :stop_area do |config| + config.add_field :id + end + end + config.add_field :forty_two, value: 42 + end + end + + it "should export the given file" do + expect{exporter.export verbose: false}.to_not raise_error + expect(exporter.status).to eq "success" + expect(File.exists?(filepath)).to be_truthy + json = JSON.parse File.read(filepath) + row = json["stops"].first + expect(row["name"]).to eq @stop_area.name + expect(row["lat"]).to eq @stop_area.latitude.to_s + expect(row["lng"]).to eq (@stop_area.latitude.to_i + 1) + expect(row["street_name"]).to eq "Lil Exporter" + expect(row["stop_area_referential"]["id"]).to eq @stop_area.stop_area_referential_id + expect(row["stop_area_referential"]["id_other"]).to eq @stop_area.stop_area_referential_id + expect(row["stop_points"][0]["id"]).to eq @stop_1.id + expect(row["stop_points"][0]["stop_area"]["id"]).to eq @stop_area.id + expect(row["stop_points"][1]["id"]).to eq @stop_2.id + expect(row["forty_two"]).to eq 42 + end + end + end +end |