Refs #5765 @6h; Add a customizable importer mechanism

12 files changed, 375 insertions, 1 deletions

diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index bb8747faa..5afe63747 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -112,6 +112,7 @@ module Chouette
         id.to_s
       end
     end
+    alias_method :local_id, :user_objectid
 
     alias_method :local_id, :user_objectid
 
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
new file mode 100644
index 000000000..4f549c408
--- /dev/null
+++ b/app/models/simple_importer.rb
@@ -0,0 +1,185 @@
+class SimpleImporter < ActiveRecord::Base
+  attr_accessor :configuration
+
+  def self.define name
+    @importers ||= {}
+    configuration = Configuration.new name
+    yield configuration
+    configuration.validate!
+    @importers[name.to_sym] = configuration
+  end
+
+  def self.find_configuration name
+    @importers ||= {}
+    configuration = @importers[name.to_sym]
+    raise "Importer not found: #{name}" unless configuration
+    configuration
+  end
+
+  def initialize *args
+    super *args
+    self.configuration = self.class.find_configuration self.configuration_name
+    self.journal ||= []
+  end
+
+  def resolve col_name, value, &block
+    val = block.call(value)
+    return val if val.present?
+    @resolution_queue[[col_name.to_s, value]].push({record: @current_record, attribute: @current_attribute, block: block})
+    nil
+  end
+
+  def import opts={}
+    @verbose = opts.delete :verbose
+    @resolution_queue = Hash.new{|h,k| h[k] = []}
+    number_of_lines = 0
+    padding = 1
+    fail_with_error "File not found: #{self.filepath}" do
+      number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
+      padding = [1, Math.log(number_of_lines, 10).ceil()].max
+    end
+
+    current_line = 0
+    status = :success
+    statuses = ""
+    log "#{"%#{padding}d" % 0}/#{number_of_lines}", clear: true
+    CSV.foreach(filepath, self.configuration.csv_options) do |row|
+      @current_record = self.configuration.find_record row
+      self.configuration.columns.each do |col|
+        @current_attribute = col[:attribute]
+        val = col[:value]
+        if val.nil? || val.is_a?(Proc)
+          if row.has_key? col.name
+            if val.is_a?(Proc)
+              val = instance_exec(row[col.name], &val)
+            else
+              val = row[col.name]
+            end
+          else
+            self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+            status = :success_with_warnings
+          end
+        end
+        @current_record.send "#{@current_attribute}=", val if val
+      end
+
+      fail_with_error ->(){ @current_record.errors.messages } do
+        new_record = @current_record.new_record?
+        @current_record.save!
+        self.journal.push({event: (new_record ? :creation : :update), kind: :log})
+        statuses += new_record ? "✓" : "-"
+      end
+      self.configuration.columns.each do |col|
+        if col.name && @resolution_queue.any?
+          val = @current_record.send col[:attribute]
+          (@resolution_queue.delete([col.name, val]) || []).each do |res|
+            record = res[:record]
+            attribute = res[:attribute]
+            value = res[:block].call(val, record)
+            record.send "#{attribute}=", value
+            record.save!
+          end
+        end
+      end
+      current_line += 1
+      log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
+    end
+    self.update_attribute :status, status
+  rescue FailedImport
+    self.update_attribute :status, :failed
+  ensure
+    self.save
+  end
+
+  protected
+
+  def fail_with_error msg
+    begin
+      yield
+    rescue => e
+      msg = msg.call if msg.is_a?(Proc)
+      log "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red
+      self.journal.push({message: msg, error: e.message, event: :error, kind: :error})
+      self.save
+      raise FailedImport
+    end
+  end
+
+  def log msg, opts={}
+    return unless @verbose
+    out = ""
+    if opts[:color]
+      color = {
+        red: "31",
+        green: "32",
+        orange: "33",
+      }[opts[:color]] || "33"
+    end
+    out += "\e[#{color}m" if color
+    if opts[:clear] && @prev_msg_size
+      out += "\b"*@prev_msg_size
+    end
+    out += msg
+    out += "\e[0m" if color
+    print out
+    @prev_msg_size = msg.size
+    @prev_msg_size += 9 if color
+  end
+
+  class FailedImport < RuntimeError
+  end
+
+  class Configuration
+    attr_accessor :model, :headers, :separator, :key
+    attr_reader :columns
+
+    def initialize import_name
+      @import_name = import_name
+      @key = "id"
+      @headers = true
+      @separator = ","
+      @columns = []
+    end
+
+    def validate!
+      raise "Incomplete configuration, missing model for #{@import_name}" unless model.present?
+    end
+
+    def attribute_for_col col_name
+      column = self.columns.find{|c| c.name == col_name}
+      column && column[:attribute] || col_name
+    end
+
+    def find_record attrs
+      model.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+    end
+
+    def csv_options
+      {
+        headers: self.headers,
+        col_sep: self.separator
+      }
+    end
+
+    def add_column name, opts={}
+      @columns.push Column.new({name: name.to_s}.update(opts))
+    end
+
+    def add_value attribute, value
+      @columns.push Column.new({attribute: attribute, value: value})
+    end
+
+    class Column
+      attr_accessor :name
+      def initialize opts={}
+        @name = opts[:name]
+        @options = opts
+        @options[:attribute] ||= @name
+      end
+
+      def [](key)
+        @options[key]
+      end
+    end
+  end
+end
diff --git a/config/initializers/apartment.rb b/config/initializers/apartment.rb
index 2d06fb88b..fc652a2da 100644
--- a/config/initializers/apartment.rb
+++ b/config/initializers/apartment.rb
@@ -81,6 +81,7 @@ Apartment.configure do |config|
     'ComplianceCheckMessage',
     'Merge',
     'CustomField',
+    'SimpleImporter',
   ]
 
   # use postgres schemas?
diff --git a/db/migrate/20180129210928_create_simple_importers.rb b/db/migrate/20180129210928_create_simple_importers.rb
new file mode 100644
index 000000000..c2a918900
--- /dev/null
+++ b/db/migrate/20180129210928_create_simple_importers.rb
@@ -0,0 +1,10 @@
+class CreateSimpleImporters < ActiveRecord::Migration
+  def change
+    create_table :simple_importers do |t|
+      t.string :configuration_name
+      t.string :filepath
+      t.string :status
+      t.json :journal
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 596682ce8..f77961f8d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -15,9 +15,10 @@ ActiveRecord::Schema.define(version: 20180202170009) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
-  enable_extension "postgis"
   enable_extension "hstore"
+  enable_extension "postgis"
   enable_extension "unaccent"
+  enable_extension "objectid"
 
   create_table "access_links", id: :bigserial, force: :cascade do |t|
     t.integer  "access_point_id",                        limit: 8
@@ -119,6 +120,7 @@ ActiveRecord::Schema.define(version: 20180202170009) do
     t.datetime "updated_at"
     t.date     "end_date"
     t.string   "date_type"
+    t.string   "mode"
   end
 
   add_index "clean_ups", ["referential_id"], name: "index_clean_ups_on_referential_id", using: :btree
@@ -724,6 +726,13 @@ ActiveRecord::Schema.define(version: 20180202170009) do
     t.integer "line_id",      limit: 8
   end
 
+  create_table "simple_importers", id: :bigserial, force: :cascade do |t|
+    t.string "configuration_name"
+    t.string "filepath"
+    t.string "status"
+    t.json   "journal"
+  end
+
   create_table "stop_area_referential_memberships", id: :bigserial, force: :cascade do |t|
     t.integer "organisation_id",          limit: 8
     t.integer "stop_area_referential_id", limit: 8
diff --git a/lib/tasks/imports.rake b/lib/tasks/imports.rake
index 02e32fd3d..006945c07 100644
--- a/lib/tasks/imports.rake
+++ b/lib/tasks/imports.rake
@@ -8,4 +8,12 @@ namespace :import do
   task netex_abort_old: :environment do
     NetexImport.abort_old
   end
+
+  desc "import the given file with the corresponding importer"
+  task :import, [:configuration_name, :filepath] => :environment do |t, args|
+    importer = SimpleImporter.create configuration_name: args[:configuration_name], filepath: args[:filepath]
+    puts "\e[33m***\e[0m Start importing"
+    importer.import(verbose: true)
+    puts "\n\e[33m***\e[0m Import done, status: " + (importer.status == "success" ? "\e[32m" : "\e[31m" ) + importer.status + "\e[0m"
+  end
 end
diff --git a/spec/fixtures/simple_importer/stop_area.csv b/spec/fixtures/simple_importer/stop_area.csv
new file mode 100644
index 000000000..9361d022b
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_area.csv
@@ -0,0 +1,2 @@
+name;lat;long;type;street_name
+Nom du Stop;45.00;12;ZDEP;99 rue des Poissonieres
diff --git a/spec/fixtures/simple_importer/stop_area_full.csv b/spec/fixtures/simple_importer/stop_area_full.csv
new file mode 100644
index 000000000..250caab30
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_area_full.csv
@@ -0,0 +1,3 @@
+"id";"station_code";"uic_code";"country_code";"province";"district";"county";"station_name";"inactive";"change_timestamp";"longitude";"latitude";"parent_station_code";"additional_info";"external_reference";"timezone";"address";"postal_code";"city"
+5669;"PAR";"PAR";"FRA";"";"";"";"Paris - All stations";f;"2017-07-17 11:56:53.138";2.35222190000002;48.856614;"";"";"{""Région"":""Ile-de-France""}";"Europe/Paris";"";"";""
+5748;"XED";"XED";"FRA";"";"";"";"Paris MLV";t;"2017-05-29 11:24:34.575";2.783409;48.870569;"PAR";"";"{""Région"":""Ile-de-France""}";"Europe/Paris";"";"";""
diff --git a/spec/fixtures/simple_importer/stop_area_full_reverse.csv b/spec/fixtures/simple_importer/stop_area_full_reverse.csv
new file mode 100644
index 000000000..9ea15f6cc
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_area_full_reverse.csv
@@ -0,0 +1,3 @@
+"id";"station_code";"uic_code";"country_code";"province";"district";"county";"station_name";"inactive";"change_timestamp";"longitude";"latitude";"parent_station_code";"additional_info";"external_reference";"timezone";"address";"postal_code";"city"
+5748;"XED";"XED";"FRA";"";"";"";"Paris MLV";t;"2017-05-29 11:24:34.575";2.783409;48.870569;"PAR";"";"{""Région"":""Ile-de-France""}";"Europe/Paris";"";"";""
+5669;"PAR";"PAR";"FRA";"";"";"";"Paris - All stations";f;"2017-07-17 11:56:53.138";2.35222190000002;48.856614;"";"";"{""Région"":""Ile-de-France""}";"Europe/Paris";"";"";""
diff --git a/spec/fixtures/simple_importer/stop_area_incomplete.csv b/spec/fixtures/simple_importer/stop_area_incomplete.csv
new file mode 100644
index 000000000..cd9447acd
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_area_incomplete.csv
@@ -0,0 +1,2 @@
+name;lat;long;type;street_name
+;45.00;12;ZDEP
diff --git a/spec/fixtures/simple_importer/stop_area_missing_street_name.csv b/spec/fixtures/simple_importer/stop_area_missing_street_name.csv
new file mode 100644
index 000000000..aa845c3f5
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_area_missing_street_name.csv
@@ -0,0 +1,2 @@
+name;lat;long;type;foo
+Nom du Stop;45.00;12;ZDEP;blabla
diff --git a/spec/models/simple_importer_spec.rb b/spec/models/simple_importer_spec.rb
new file mode 100644
index 000000000..50958970c
--- /dev/null
+++ b/spec/models/simple_importer_spec.rb
@@ -0,0 +1,148 @@
+RSpec.describe SimpleImporter do
+  describe "#define" do
+    context "with an incomplete configuration" do
+
+      it "should raise an error" do
+        expect do
+          SimpleImporter.define :foo
+        end.to raise_error
+      end
+    end
+    context "with a complete configuration" do
+      before do
+        SimpleImporter.define :foo do |config|
+          config.model = "example"
+        end
+      end
+
+      it "should define an importer" do
+        expect{SimpleImporter.find_configuration(:foo)}.to_not raise_error
+        expect{SimpleImporter.new(configuration_name: :foo, filepath: "")}.to_not raise_error
+        expect{SimpleImporter.find_configuration(:bar)}.to raise_error
+        expect{SimpleImporter.new(configuration_name: :bar, filepath: "")}.to raise_error
+        expect{SimpleImporter.create(configuration_name: :foo, filepath: "")}.to change{SimpleImporter.count}.by 1
+      end
+    end
+  end
+
+  describe "#import" do
+    before(:each) do
+      SimpleImporter.define :test do |config|
+        config.model = Chouette::StopArea
+        config.separator = ";"
+        config.key = "name"
+        config.add_column :name
+        config.add_column :lat, attribute: :latitude
+        config.add_column :lat, attribute: :longitude, value: ->(raw){ raw.to_f + 1 }
+        config.add_column :type, attribute: :area_type, value: ->(raw){ raw&.downcase }
+        config.add_column :street_name
+        config.add_column :stop_area_referential, value: create(:stop_area_referential, objectid_format: :stif_netex)
+      end
+    end
+
+    it "should import the given file" do
+      importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area.csv")
+      expect{importer.import}.to change{Chouette::StopArea.count}.by 1
+      expect(importer.status).to eq "success"
+      stop = Chouette::StopArea.last
+      expect(stop.name).to eq "Nom du Stop"
+      expect(stop.latitude).to eq 45.00
+      expect(stop.longitude).to eq 46.00
+      expect(stop.area_type).to eq "zdep"
+      expect(importer.reload.journal.last["event"]).to eq("creation")
+    end
+
+    context "with an already existing record" do
+      before(:each){
+        create :stop_area, name: "Nom du Stop"
+      }
+      it "should only update the record" do
+        importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area.csv")
+        expect{importer.import}.to change{Chouette::StopArea.count}.by 0
+        expect(importer.status).to eq "success"
+        stop = Chouette::StopArea.last
+        expect(stop.name).to eq "Nom du Stop"
+        expect(stop.latitude).to eq 45.00
+        expect(stop.longitude).to eq 46.00
+        expect(stop.area_type).to eq "zdep"
+        expect(importer.reload.journal.last["event"]).to eq("update")
+      end
+    end
+
+    context "with a missing column" do
+      it "should set an error message" do
+        importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_missing_street_name.csv")
+        expect{importer.import}.to_not raise_error
+        expect(importer.status).to eq "success_with_warnings"
+        expect(importer.reload.journal.first["event"]).to eq("column_not_found")
+      end
+    end
+
+    context "with a incomplete dataset" do
+      it "should create a StopArea" do
+        importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_incomplete.csv")
+        expect{importer.import}.to_not raise_error
+        expect(importer.status).to eq "failed"
+        expect(importer.reload.journal.first["message"]).to eq({"name" => ["doit être rempli(e)"]})
+      end
+    end
+
+    context "with a wrong filepath" do
+      it "should create a StopArea" do
+        importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/not_found.csv")
+        expect{importer.import}.to_not raise_error
+        expect(importer.status).to eq "failed"
+        expect(importer.reload.journal.first["message"]).to eq "File not found: #{importer.filepath}"
+      end
+    end
+
+    context "with a full file" do
+      before(:each) do
+        SimpleImporter.define :test do |config|
+          config.model = Chouette::StopArea
+          config.separator = ";"
+          config.key = "station_code"
+          config.add_column :station_code, attribute: :registration_number
+          config.add_column :country_code
+          config.add_column :station_name, attribute: :name
+          config.add_column :inactive, attribute: :deleted_at, value: ->(raw){ raw == "t" ? Time.now : nil }
+          config.add_column :change_timestamp, attribute: :updated_at
+          config.add_column :longitude
+          config.add_column :latitude
+          config.add_column :parent_station_code, attribute: :parent, value: ->(raw){ raw.present? && resolve(:station_code, raw){|value| Chouette::StopArea.find_by(registration_number: value) } }
+          config.add_column :parent_station_code, attribute: :area_type, value: ->(raw){ raw.present? ? "zdep" : "gdl" }
+          config.add_column :timezone, attribute: :time_zone
+          config.add_column :address, attribute: :street_name
+          config.add_column :postal_code, attribute: :zip_code
+          config.add_column :city, attribute: :city_name
+          config.add_value  :stop_area_referential_id, create(:stop_area_referential, objectid_format: :stif_netex).id
+          config.add_value  :long_lat_type, "WGS84"
+        end
+      end
+
+      it "should import the given file" do
+        importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_full.csv")
+        expect{importer.import}.to change{Chouette::StopArea.count}.by 2
+        expect(importer.status).to eq "success"
+        first = Chouette::StopArea.find_by registration_number: "PAR"
+        last = Chouette::StopArea.find_by registration_number: "XED"
+
+        expect(last.parent).to eq first
+        expect(first.area_type).to eq "gdl"
+        expect(last.area_type).to eq "zdep"
+        expect(first.long_lat_type).to eq "WGS84"
+      end
+
+      context "with a relation in reverse order" do
+        it "should import the given file" do
+          importer = SimpleImporter.new(configuration_name: :test, filepath: Rails.root + "spec/fixtures/simple_importer/stop_area_full_reverse.csv")
+          expect{importer.import}.to change{Chouette::StopArea.count}.by 2
+          expect(importer.status).to eq "success"
+          first = Chouette::StopArea.find_by registration_number: "XED"
+          last = Chouette::StopArea.find_by registration_number: "PAR"
+          expect(first.parent).to eq last
+        end
+      end
+    end
+  end
+end