diff options
| -rw-r--r-- | app/models/simple_importer.rb | 59 | ||||
| -rw-r--r-- | lib/tasks/imports.rake | 12 | ||||
| -rw-r--r-- | spec/models/simple_importer_spec.rb | 35 |
3 files changed, 83 insertions, 23 deletions
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb index 4f549c408..6ab013b04 100644 --- a/app/models/simple_importer.rb +++ b/app/models/simple_importer.rb @@ -22,6 +22,13 @@ class SimpleImporter < ActiveRecord::Base self.journal ||= [] end + def configure + new_config = configuration.duplicate + yield new_config + new_config.validate! + self.configuration = new_config + end + def resolve col_name, value, &block val = block.call(value) return val if val.present? @@ -67,7 +74,7 @@ class SimpleImporter < ActiveRecord::Base new_record = @current_record.new_record? @current_record.save! self.journal.push({event: (new_record ? :creation : :update), kind: :log}) - statuses += new_record ? "✓" : "-" + statuses += new_record ? colorize("✓", :green) : colorize("-", :orange) end self.configuration.columns.each do |col| if col.name && @resolution_queue.any? @@ -105,25 +112,26 @@ class SimpleImporter < ActiveRecord::Base end end + def colorize txt, color + color = { + red: "31", + green: "32", + orange: "33", + }[color] || "33" + "\e[#{color}m#{txt}\e[0m" + end + def log msg, opts={} return unless @verbose out = "" - if opts[:color] - color = { - red: "31", - green: "32", - orange: "33", - }[opts[:color]] || "33" - end - out += "\e[#{color}m" if color + msg = colorize(msg, opts[:color]) if opts[:color] if opts[:clear] && @prev_msg_size out += "\b"*@prev_msg_size end out += msg - out += "\e[0m" if color print out @prev_msg_size = msg.size - @prev_msg_size += 9 if color + # @prev_msg_size += 9 if opts[:color] end class FailedImport < RuntimeError @@ -133,12 +141,27 @@ class SimpleImporter < ActiveRecord::Base attr_accessor :model, :headers, :separator, :key attr_reader :columns - def initialize import_name + def initialize import_name, opts={} @import_name = import_name - @key = "id" - @headers = true - @separator = "," - @columns = [] + @key = opts[:key] || "id" + @headers = opts.has_key?(:headers) ? opts[:headers] : true + @separator = opts[:separator] || "," + @columns = opts[:columns] || [] + @model = opts[:model] + end + + def duplicate + Configuration.new @import_name, self.options + end + + def options + { + key: @key, + headers: @headers, + separator: @separator, + columns: @columns.map(&:duplicate), + model: model + } end def validate! @@ -177,6 +200,10 @@ class SimpleImporter < ActiveRecord::Base @options[:attribute] ||= @name end + def duplicate + Column.new @options.dup + end + def [](key) @options[key] end diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake index 8708d6eed..3e2c3a4f2 100644 --- a/lib/tasks/imports.rake +++ b/lib/tasks/imports.rake @@ -11,4 +11,16 @@ namespace :import do importer.import(verbose: true) puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" end + + desc "import the given file with the corresponding importer in the given StopAreaReferential" + task :import_stop_areas_in_referential, [:referential_id, :configuration_name, :filepath] => :environment do |t, args| + referential = StopAreaReferential.find args[:referential_id] + importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath] + importer.configure do |config| + config.add_value :stop_area_referential, referential + end + puts "\e[33m***\e[0m Start importing" + importer.import(verbose: true) + puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m" + end end diff --git a/spec/models/simple_importer_spec.rb b/spec/models/simple_importer_spec.rb index 50958970c..6b2889d59 100644 --- a/spec/models/simple_importer_spec.rb +++ b/spec/models/simple_importer_spec.rb @@ -26,6 +26,9 @@ RSpec.describe SimpleImporter do end describe "#import" do + let(:importer){ importer = SimpleImporter.new(configuration_name: :test, filepath: filepath) } + let(:filepath){ Rails.root + "spec/fixtures/simple_importer/#{filename}" } + let(:filename){ "stop_area.csv" } before(:each) do SimpleImporter.define :test do |config| config.model = Chouette::StopArea @@ -41,7 +44,6 @@ RSpec.describe SimpleImporter do end it "should import the given file" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area.csv") expect{importer.import}.to change{Chouette::StopArea.count}.by 1 expect(importer.status).to eq "success" stop = Chouette::StopArea.last @@ -52,12 +54,30 @@ RSpec.describe SimpleImporter do expect(importer.reload.journal.last["event"]).to eq("creation") end + context "when overriding configuration" do + before(:each){ + importer.configure do |config| + config.add_value :latitude, 88 + end + } + + it "should import the given file and not mess with the global configuration" do + expect{importer.import}.to change{Chouette::StopArea.count}.by 1 + expect(importer.status).to eq "success" + stop = Chouette::StopArea.last + expect(stop.latitude).to eq 88 + importer = SimpleImporter.new(configuration_name: :test, filepath: filepath) + expect{importer.import}.to change{Chouette::StopArea.count}.by 0 + expect(stop.reload.latitude).to eq 45 + end + end + context "with an already existing record" do + let(:filename){ "stop_area.csv" } before(:each){ create :stop_area, name: "Nom du Stop" } it "should only update the record" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area.csv") expect{importer.import}.to change{Chouette::StopArea.count}.by 0 expect(importer.status).to eq "success" stop = Chouette::StopArea.last @@ -70,8 +90,8 @@ RSpec.describe SimpleImporter do end context "with a missing column" do + let(:filename){ "stop_area_missing_street_name.csv" } it "should set an error message" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_missing_street_name.csv") expect{importer.import}.to_not raise_error expect(importer.status).to eq "success_with_warnings" expect(importer.reload.journal.first["event"]).to eq("column_not_found") @@ -79,8 +99,8 @@ RSpec.describe SimpleImporter do end context "with a incomplete dataset" do + let(:filename){ "stop_area_incomplete.csv" } it "should create a StopArea" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_incomplete.csv") expect{importer.import}.to_not raise_error expect(importer.status).to eq "failed" expect(importer.reload.journal.first["message"]).to eq({"name" => ["doit être rempli(e)"]}) @@ -88,8 +108,8 @@ RSpec.describe SimpleImporter do end context "with a wrong filepath" do + let(:filename){ "not_found.csv" } it "should create a StopArea" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/not_found.csv") expect{importer.import}.to_not raise_error expect(importer.status).to eq "failed" expect(importer.reload.journal.first["message"]).to eq "File not found: #{importer.filepath}" @@ -97,6 +117,7 @@ RSpec.describe SimpleImporter do end context "with a full file" do + let(:filename){ "stop_area_full.csv" } before(:each) do SimpleImporter.define :test do |config| config.model = Chouette::StopArea @@ -121,7 +142,6 @@ RSpec.describe SimpleImporter do end it "should import the given file" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_full.csv") expect{importer.import}.to change{Chouette::StopArea.count}.by 2 expect(importer.status).to eq "success" first = Chouette::StopArea.find_by registration_number: "PAR" @@ -134,8 +154,9 @@ RSpec.describe SimpleImporter do end context "with a relation in reverse order" do + let(:filename){ "stop_area_full_reverse.csv" } + it "should import the given file" do - importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_full_reverse.csv") expect{importer.import}.to change{Chouette::StopArea.count}.by 2 expect(importer.status).to eq "success" first = Chouette::StopArea.find_by registration_number: "XED" |