Refs #5924 @2h; Provide a mechanism to define a custom importer

6 files changed, 233 insertions, 22 deletions

diff --git a/app/models/chouette/journey_pattern.rb b/app/models/chouette/journey_pattern.rb
index aa9fdb810..830e985d9 100644
--- a/app/models/chouette/journey_pattern.rb
+++ b/app/models/chouette/journey_pattern.rb
@@ -170,5 +170,21 @@ module Chouette
       end
       full
     end
+
+    def set_distances distances
+      raise "inconsistent data: #{distances.count} values for #{stop_points.count} stops" unless distances.count == stop_points.count
+      prev = distances[0].to_i
+      _costs = self.costs
+      distances[1..-1].each_with_index do |distance, i|
+        distance = distance.to_i
+        relative = distance - prev
+        prev = distance
+        start, stop = stop_points[i..i+1]
+        key = "#{start.stop_area_id}-#{stop.stop_area_id}"
+        _costs[key] ||= {}
+        _costs[key]["distance"] = relative
+      end
+      self.costs = _costs
+    end
   end
 end
diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb
index 5cc5d8b0d..e418134de 100644
--- a/app/models/chouette/route.rb
+++ b/app/models/chouette/route.rb
@@ -185,6 +185,18 @@ module Chouette
       return true
     end
 
+    def full_journey_pattern
+      out = journey_patterns.find{|jp| jp.stop_points.count == self.stop_points.count }
+      unless out
+        out = journey_patterns.build name: self.name
+        self.stop_points.each do |sp|
+          out.stop_points.build stop_area: sp.stop_area, position: sp.position
+        end
+        out.save!
+      end
+      out
+    end
+
     protected
 
     def self.vehicle_journeys_timeless(stop_point_id)
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index 760b98610..41ce379db 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -29,6 +29,10 @@ class SimpleImporter < ActiveRecord::Base
     self.configuration = new_config
   end
 
+  def context
+    self.configuration.context || {}
+  end
+
   def resolve col_name, value, &block
     val = block.call(value)
     return val if val.present?
@@ -54,24 +58,7 @@ class SimpleImporter < ActiveRecord::Base
       self.configuration.before_actions(:all).each &:call
 
       CSV.foreach(filepath, self.configuration.csv_options) do |row|
-        @current_record = self.configuration.find_record row
-        self.configuration.columns.each do |col|
-          @current_attribute = col[:attribute]
-          val = col[:value]
-          if val.nil? || val.is_a?(Proc)
-            if row.has_key? col.name
-              if val.is_a?(Proc)
-                val = instance_exec(row[col.name], &val)
-              else
-                val = row[col.name]
-              end
-            else
-              self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
-              status = :success_with_warnings
-            end
-          end
-          @current_record.send "#{@current_attribute}=", val if val
-        end
+        status = handle_row row, status
 
         fail_with_error ->(){ @current_record.errors.messages } do
           new_record = @current_record.new_record?
@@ -100,6 +87,9 @@ class SimpleImporter < ActiveRecord::Base
         current_line += 1
         log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
       end
+      self.configuration.after_actions(:all).each do |action|
+        action.call self
+      end
     end
     self.update_attribute :status, status
   rescue FailedImport
@@ -108,8 +98,6 @@ class SimpleImporter < ActiveRecord::Base
     self.save!
   end
 
-  protected
-
   def fail_with_error msg
     begin
       yield
@@ -121,6 +109,34 @@ class SimpleImporter < ActiveRecord::Base
     end
   end
 
+  protected
+
+  def handle_row row, status
+    if self.configuration.get_custom_handler
+      instance_exec(row, &self.configuration.get_custom_handler)
+    else
+      @current_record = self.configuration.find_record row
+      self.configuration.columns.each do |col|
+        @current_attribute = col[:attribute]
+        val = col[:value]
+        if val.nil? || val.is_a?(Proc)
+          if row.has_key? col.name
+            if val.is_a?(Proc)
+              val = instance_exec(row[col.name], &val)
+            else
+              val = row[col.name]
+            end
+          else
+            self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+            status = :success_with_warnings
+          end
+        end
+        @current_record.send "#{@current_attribute}=", val if val
+      end
+    end
+    status
+  end
+
   def colorize txt, color
     color = {
       red: "31",
@@ -146,7 +162,7 @@ class SimpleImporter < ActiveRecord::Base
   end
 
   class Configuration
-    attr_accessor :model, :headers, :separator, :key
+    attr_accessor :model, :headers, :separator, :key, :context
     attr_reader :columns
 
     def initialize import_name, opts={}
@@ -205,11 +221,29 @@ class SimpleImporter < ActiveRecord::Base
       @before[group].push block
     end
 
+    def after group=:all, &block
+      @after ||= Hash.new{|h, k| h[k] = []}
+      @after[group].push block
+    end
+
     def before_actions group=:all
       @before ||= Hash.new{|h, k| h[k] = []}
       @before[group]
     end
 
+    def after_actions group=:all
+      @after ||= Hash.new{|h, k| h[k] = []}
+      @after[group]
+    end
+
+    def custom_handler &block
+      @custom_handler = block
+    end
+
+    def get_custom_handler
+      @custom_handler
+    end
+
     class Column
       attr_accessor :name
       def initialize opts={}
diff --git a/spec/fixtures/simple_importer/stop_points_full.csv b/spec/fixtures/simple_importer/stop_points_full.csv
new file mode 100644
index 000000000..9c5b3c1b7
--- /dev/null
+++ b/spec/fixtures/simple_importer/stop_points_full.csv
@@ -0,0 +1,11 @@
+"id";"timetable_route_id";"route_name";"stop_sequence";"stop_distance";"station_code";"station_name";"border"

+3354;1136;"Paris centre - Bercy > Lille > Londres";1;0;"XPB";"Paris City Center - Bercy";f

+3355;1136;"Paris centre - Bercy > Lille > Londres";2;232;"XDB";"Lille";f

+3749;1136;"Paris centre - Bercy > Lille > Londres";3;350;"COF";"Coquelles - France";t

+4772;1136;"Paris centre - Bercy > Lille > Londres";4;350;"COU";"Coquelles - UK";t

+3357;1136;"Paris centre - Bercy > Lille > Londres";5;527;"ZEP";"London";f

+3358;1137;"Londres > Lille > Paris centre - Bercy";1;0;"ZEP";"London";f

+3559;1137;"Londres > Lille > Paris centre - Bercy";2;177;"COU";"Coquelles - UK";t

+3743;1137;"Londres > Lille > Paris centre - Bercy";3;177;"COF";"Coquelles - France";t

+3360;1137;"Londres > Lille > Paris centre - Bercy";4;295;"XDB";"Lille";f

+3361;1137;"Londres > Lille > Paris centre - Bercy";5;527;"XPB";"Paris City Center - Bercy";f

diff --git a/spec/models/chouette/journey_pattern_spec.rb b/spec/models/chouette/journey_pattern_spec.rb
index 19a74a0e7..7c767e4d1 100644
--- a/spec/models/chouette/journey_pattern_spec.rb
+++ b/spec/models/chouette/journey_pattern_spec.rb
@@ -71,6 +71,30 @@ describe Chouette::JourneyPattern, :type => :model do
     end
   end
 
+  describe "set_distances" do
+    let(:journey_pattern) { create :journey_pattern }
+    let(:distances){ [] }
+    it "should raise an error" do
+      expect{journey_pattern.set_distances(distances)}.to raise_error
+    end
+
+    context "with consistent data" do
+      let(:distances){ [0, 100, "200", 500, 1000] }
+
+      it "should set costs" do
+        expect{journey_pattern.set_distances(distances)}.to_not raise_error
+        start, stop = journey_pattern.stop_points[0..1]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 100
+        start, stop = journey_pattern.stop_points[1..2]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 100
+        start, stop = journey_pattern.stop_points[2..3]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 300
+        start, stop = journey_pattern.stop_points[3..4]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 500
+      end
+    end
+  end
+
   describe "state_update" do
     def journey_pattern_to_state jp
       jp.attributes.slice('name', 'published_name', 'registration_number').tap do |item|
diff --git a/spec/models/simple_importer_spec.rb b/spec/models/simple_importer_spec.rb
index 0f2682473..c8bcf285d 100644
--- a/spec/models/simple_importer_spec.rb
+++ b/spec/models/simple_importer_spec.rb
@@ -95,7 +95,7 @@ RSpec.describe SimpleImporter do
     context "with a missing column" do
       let(:filename){ "stop_area_missing_street_name.csv" }
       it "should set an error message" do
-        expect{importer.import}.to_not raise_error
+        expect{importer.import(verbose: false)}.to_not raise_error
         expect(importer.status).to eq "success_with_warnings"
         expect(importer.reload.journal.first["event"]).to eq("column_not_found")
       end
@@ -216,5 +216,119 @@ RSpec.describe SimpleImporter do
         end
       end
     end
+
+    context "with a specific importer" do
+      let(:filename){ "stop_points_full.csv" }
+
+      before(:each) do
+        create :route, number: 1136, stop_points_count: 0
+        create :route, number: 1137, stop_points_count: 0
+
+        SimpleImporter.define :test do |config|
+          config.model = Chouette::Route
+          config.separator = ";"
+          config.context = {stop_area_referential: stop_area_referential}
+          config.custom_handler do |row|
+            fail_with_error "MISSING ROUTE: #{row["timetable_route_id"]}" do
+              @current_record = Chouette::Route.find_by! number: row["timetable_route_id"]
+            end
+            @current_record.name = row["route_name"]
+            if @prev_route != @current_record
+              @current_record.stop_points.destroy_all
+              if @prev_route
+                journey_pattern = @prev_route.full_journey_pattern
+                journey_pattern.set_distances @distances
+                fail_with_error ->(){ journey_pattern.errors.messages } do
+                  journey_pattern.save!
+                end
+              end
+              @distances = []
+            end
+            @distances.push row["stop_distance"]
+            position = row["stop_sequence"].to_i - 1
+            stop_area = context[:stop_area_referential].stop_areas.where(registration_number: row["station_code"]).last
+            unless stop_area
+              stop_area = Chouette::StopArea.new registration_number: row["station_code"]
+              stop_area.name = row["station_name"]
+              stop_area.kind = row["border"] == "f" ? :commercial : :non_commercial
+              stop_area.area_type = row["border"] == "f" ? :zdep : :border
+              stop_area.stop_area_referential = context[:stop_area_referential]
+              fail_with_error ->{ stop_area.errors.messages } do
+                stop_area.save!
+              end
+            end
+            stop_point = @current_record.stop_points.build(stop_area_id: stop_area.id, position: position)
+            @prev_route = @current_record
+          end
+
+          config.after do |importer|
+            prev_route = importer.instance_variable_get "@prev_route"
+            if prev_route
+              journey_pattern = prev_route.full_journey_pattern
+              journey_pattern.set_distances importer.instance_variable_get("@distances")
+              importer.fail_with_error ->(){ journey_pattern.errors.messages } do
+                journey_pattern.save!
+              end
+            end
+          end
+        end
+      end
+
+      it "should import the given file" do
+        routes_count = Chouette::Route.count
+        journey_pattern_count = Chouette::JourneyPattern.count
+        stop_areas_count = Chouette::StopArea.count
+        expect{importer.import(verbose: false)}.to change{Chouette::StopPoint.count}.by 20
+        expect(importer.status).to eq "success"
+        expect(Chouette::Route.count).to eq routes_count
+        expect(Chouette::JourneyPattern.count).to eq journey_pattern_count + 2
+        expect(Chouette::StopArea.count).to eq stop_areas_count + 5
+        route = Chouette::Route.find_by number: 1136
+        expect(route.stop_areas.count).to eq 5
+        journey_pattern = route.full_journey_pattern
+        expect(journey_pattern.stop_areas.count).to eq 5
+        start, stop = journey_pattern.stop_points[0..1]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 232
+        start, stop = journey_pattern.stop_points[1..2]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 118
+        start, stop = journey_pattern.stop_points[2..3]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 0
+        start, stop = journey_pattern.stop_points[3..4]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 177
+
+        route = Chouette::Route.find_by number: 1137
+        expect(route.stop_areas.count).to eq 5
+        journey_pattern = route.full_journey_pattern
+        expect(journey_pattern.stop_areas.count).to eq 5
+        start, stop = journey_pattern.stop_points[0..1]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 177
+        start, stop = journey_pattern.stop_points[1..2]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 0
+        start, stop = journey_pattern.stop_points[2..3]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 118
+        start, stop = journey_pattern.stop_points[3..4]
+        expect(journey_pattern.costs_between(start, stop)[:distance]).to eq 232
+
+        stop_area = Chouette::StopArea.where(registration_number: "XPB").last
+        expect(stop_area.kind).to eq :commercial
+        expect(stop_area.area_type).to eq :zdep
+
+        stop_area = Chouette::StopArea.where(registration_number: "XDB").last
+        expect(stop_area.kind).to eq :commercial
+        expect(stop_area.area_type).to eq :zdep
+
+        stop_area = Chouette::StopArea.where(registration_number: "COF").last
+        expect(stop_area.kind).to eq :non_commercial
+        expect(stop_area.area_type).to eq :border
+
+        stop_area = Chouette::StopArea.where(registration_number: "COU").last
+        expect(stop_area.kind).to eq :non_commercial
+        expect(stop_area.area_type).to eq :border
+
+        stop_area = Chouette::StopArea.where(registration_number: "ZEP").last
+        expect(stop_area.kind).to eq :commercial
+        expect(stop_area.area_type).to eq :zdep
+      end
+    end
   end
 end