Refs #6068; Export VehicleJourneys

Add a mechanism to allow for several rows in the csv per single object
in the collection.

7 files changed, 211 insertions, 85 deletions

diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb
index 9b94f7f0e..49a2b387e 100644
--- a/app/models/chouette/vehicle_journey.rb
+++ b/app/models/chouette/vehicle_journey.rb
@@ -89,7 +89,6 @@ module Chouette
       end
     }
 
-
     scope :in_purchase_window, ->(range){
       purchase_windows = Chouette::PurchaseWindow.overlap_dates(range)
       sql = purchase_windows.joins(:vehicle_journeys).select('vehicle_journeys.id').uniq.to_sql
@@ -156,6 +155,14 @@ module Chouette
       end
     end
 
+    def sales_start
+      purchase_windows.map{|p| p.date_ranges.first}.min
+    end
+
+    def sales_end
+      purchase_windows.map{|p| p.date_ranges.last}.max
+    end
+
     def calculate_vehicle_journey_at_stop_day_offset
       Chouette::VehicleJourneyAtStopsDayOffset.new(
         vehicle_journey_at_stops
diff --git a/app/models/concerns/application_days_support.rb b/app/models/concerns/application_days_support.rb
index 348436aa4..2d00b5847 100644
--- a/app/models/concerns/application_days_support.rb
+++ b/app/models/concerns/application_days_support.rb
@@ -15,41 +15,34 @@ module ApplicationDaysSupport
   end
 
   def day_by_mask(flag)
-    int_day_types & flag == flag
+    self.class.day_by_mask int_day_types, flag
   end
 
-  def self.day_by_mask(int_day_types,flag)
-    int_day_types & flag == flag
+  def valid_day? wday
+    valid_days.include?(wday)
   end
 
-  def valid_days
-    # Build an array with day of calendar week (1-7, Monday is 1).
-    [].tap do |valid_days|
-      valid_days << 1  if monday
-      valid_days << 2  if tuesday
-      valid_days << 3  if wednesday
-      valid_days << 4  if thursday
-      valid_days << 5  if friday
-      valid_days << 6  if saturday
-      valid_days << 7  if sunday
+  included do
+    def self.valid_days(int_day_types)
+      # Build an array with day of calendar week (1-7, Monday is 1).
+      [].tap do |valid_days|
+        valid_days << 1  if day_by_mask(int_day_types,MONDAY)
+        valid_days << 2  if day_by_mask(int_day_types,TUESDAY)
+        valid_days << 3  if day_by_mask(int_day_types,WEDNESDAY)
+        valid_days << 4  if day_by_mask(int_day_types,THURSDAY)
+        valid_days << 5  if day_by_mask(int_day_types,FRIDAY)
+        valid_days << 6  if day_by_mask(int_day_types,SATURDAY)
+        valid_days << 7  if day_by_mask(int_day_types,SUNDAY)
+      end
     end
-  end
 
-  def valid_day? wday
-    valid_days.include?(wday)
+    def self.day_by_mask(int_day_types,flag)
+      int_day_types & flag == flag
+    end
   end
 
-  def self.valid_days(int_day_types)
-    # Build an array with day of calendar week (1-7, Monday is 1).
-    [].tap do |valid_days|
-      valid_days << 1  if day_by_mask(int_day_types,MONDAY)
-      valid_days << 2  if day_by_mask(int_day_types,TUESDAY)
-      valid_days << 3  if day_by_mask(int_day_types,WEDNESDAY)
-      valid_days << 4  if day_by_mask(int_day_types,THURSDAY)
-      valid_days << 5  if day_by_mask(int_day_types,FRIDAY)
-      valid_days << 6  if day_by_mask(int_day_types,SATURDAY)
-      valid_days << 7  if day_by_mask(int_day_types,SUNDAY)
-    end
+  def valid_days
+    self.class.valid_days int_day_types
   end
 
   def monday
diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index f9b1c0a80..9c12a4e02 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -10,10 +10,13 @@ class SimpleExporter < SimpleInterface
     @padding = 1
     @current_line = -1
     @number_of_lines = collection.size
+
     @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
 
     @csv = nil
     fail_with_error "Unable to write in file: #{self.filepath}" do
+      dir = Pathname.new(self.filepath).dirname
+      FileUtils.mkdir_p dir
       @csv = CSV.open(self.filepath, 'w', self.configuration.csv_options)
     end
 
@@ -54,56 +57,100 @@ class SimpleExporter < SimpleInterface
     log "Starting export ...", color: :green
     log "Export will be written in #{filepath}", color: :green
     @csv << self.configuration.columns.map(&:name)
-    ids = collection.pluck :id
-    ids.in_groups_of(configuration.batch_size).each do |batch_ids|
-      collection.where(id: batch_ids).each do |item|
-        @current_row = item.attributes
-        @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
-        row = []
-        @new_status = nil
-        self.configuration.columns.each do |col|
-          val = col[:value]
-          if val.nil? || val.is_a?(Proc)
-            if item.respond_to? col.attribute
-              if val.is_a?(Proc)
-                val = instance_exec(item.send(col.attribute), &val)
-              else
-                val = item.send(col.attribute)
-              end
-            else
-              push_in_journal({event: :attribute_not_found, message: "Attribute not found: #{col.attribute}", kind: :warning})
-              self.status ||= :success_with_warnings
-            end
+    if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+      ids = collection.pluck :id
+      ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+        collection.where(id: batch_ids).each do |item|
+          handle_item item
+        end
+      end
+    else
+      collection.each{|item| handle_item item }
+    end
+    print_state
+  end
+
+  def map_item_to_rows item
+    return [item] unless configuration.item_to_rows_mapping
+    configuration.item_to_rows_mapping.call(item).map {|row| CustomRow.new row }
+  end
+
+  def handle_item item
+    number_of_lines = @number_of_lines
+    map_item_to_rows(item).each_with_index do |item, i|
+      @number_of_lines = number_of_lines + i
+      @current_row = item.attributes
+      @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+      row = []
+      @new_status = nil
+      self.configuration.columns.each do |col|
+        scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
+        val = col[:value]
+        if val.nil? || val.is_a?(Proc)
+          if val.is_a?(Proc)
+            val = instance_exec(scoped_item, &val)
+          else
+            attributes = [col.attribute].flatten
+            val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
           end
+        end
+        if val.nil?
+          push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
+          self.status ||= :success_with_warnings
+        end
 
-          if val.nil? && col.required?
-            @new_status = colorize("x", :red)
-            raise "MISSING VALUE FOR COLUMN #{col.name}"
+        if val.nil? && col.required?
+          @new_status = colorize("x", :red)
+          raise "MISSING VALUE FOR COLUMN #{col.name}"
+        end
+        @new_status ||= colorize("✓", :green)
+        val = encode_string(val) if val.is_a?(String)
+        row << val
+      end
+      push_in_journal({event: :success, kind: :log})
+      @statuses += @new_status
+      print_state if @current_line % 20 == 0
+      @current_line += 1
+      @csv << row
+    end
+  end
+
+  class CustomRow < OpenStruct
+    def initialize data
+      super data
+      @data = data
+    end
+
+    def attributes
+      flatten_hash @data
+    end
+
+    protected
+    def flatten_hash h
+      h.each_with_object({}) do |(k, v), h|
+        if v.is_a? Hash
+          flatten_hash(v).map do |h_k, h_v|
+            h["#{k}.#{h_k}".to_sym] = h_v
           end
-          @new_status ||= colorize("✓", :green)
-          val = encode_string(val) if val.is_a?(String)
-          row << val
+        else
+          h[k] = v
         end
-        push_in_journal({event: :success, kind: :log})
-        @statuses += @new_status
-        print_state if @current_line % 20 == 0
-        @current_line += 1
-        @csv << row
       end
     end
-    print_state
   end
 
   class Configuration < SimpleInterface::Configuration
     attr_accessor :collection
     attr_accessor :batch_size
     attr_accessor :logged_attributes
+    attr_accessor :item_to_rows_mapping
 
     def initialize import_name, opts={}
       super import_name, opts
       @collection = opts[:collection]
       @batch_size = opts[:batch_size] || 1000
       @logged_attributes = opts[:logged_attributes]
+      @item_to_rows_mapping = opts[:item_to_rows_mapping]
     end
 
     def options
@@ -111,9 +158,14 @@ class SimpleExporter < SimpleInterface
         collection: collection,
         batch_size: batch_size,
         logged_attributes: logged_attributes,
+        item_to_rows_mapping: item_to_rows_mapping,
       })
     end
 
+    def map_item_to_rows &block
+      @item_to_rows_mapping = block
+    end
+
     def add_column name, opts={}
       raise "Column already defined: #{name}" if @columns.any?{|c| c.name == name.to_s}
       super name, opts
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 3d5027bf1..5f022719a 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -167,6 +167,13 @@ class SimpleInterface < ActiveRecord::Base
       @scope = opts[:scope]
     end
 
+    def on_relation relation_name
+      @scope ||= []
+      @scope.push relation_name
+      yield
+      @scope.pop
+    end
+
     def duplicate
       self.class.new @import_name, self.options
     end
@@ -211,7 +218,8 @@ class SimpleInterface < ActiveRecord::Base
     end
 
     def add_column name, opts={}
-      @columns.push Column.new({name: name.to_s}.update(opts))
+      @scope ||= []
+      @columns.push Column.new({name: name.to_s, scope: @scope.dup}.update(opts))
     end
 
     def add_value attribute, value
@@ -262,6 +270,10 @@ class SimpleInterface < ActiveRecord::Base
         !!@options[:required]
       end
 
+      def scope
+        @options[:scope] || []
+      end
+
       def [](key)
         @options[key]
       end
diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake
index 036d96b11..6ff73dac0 100644
--- a/lib/tasks/exports.rake
+++ b/lib/tasks/exports.rake
@@ -40,8 +40,24 @@ namespace :export do
     SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
   end
 
-  desc "export a complete offer from the gicen referential in the given X next days"
-  task :full_offer, [:referential_id, :timelapse, :configuration_name, :output_dir, :logs_output_dir] => :environment do |t, args|
-    args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/")
+  desc "export a complete offer from the given referential in the given X next days"
+  task :full_offer, [:referential_id, :configuration_name, :timelapse, :output_dir, :logs_output_dir] => :environment do |t, args|
+    referential = Referential.find args[:referential_id]
+    args.with_defaults(output_dir: referential.name, logs_output_dir: "./log/exporters/", timelapse: 90)
+
+    referential.switch
+
+    journeys = Chouette::VehicleJourney.with_matching_timetable (Time.now.to_date..args[:timelapse].to_i.days.from_now.to_date)
+    if journeys.count == 0
+      puts "No maching journeys were found".red
+    else
+
+      exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.csv"
+      exporter.configure do |config|
+        config.collection = journeys
+      end
+
+      SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
+    end
   end
 end
diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb
index 68e02e818..1dc051575 100644
--- a/lib/tasks/helpers/simple_interfaces.rb
+++ b/lib/tasks/helpers/simple_interfaces.rb
@@ -2,7 +2,7 @@ module SimpleInterfacesHelper
   def self.interface_output_to_csv interface, output_dir
     filepath =  File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
     cols = %w(line kind event message error)
-    if interface.reload.journal.size > 0
+    if interface.reload.journal.size > 0 && interface.journal.first["row"].present?
       keys = interface.journal.first["row"].map(&:first)
       CSV.open(filepath, "w") do |csv|
         csv << cols + keys
diff --git a/spec/models/simple_exporter_spec.rb b/spec/models/simple_exporter_spec.rb
index 395cb1034..18cdc55e8 100644
--- a/spec/models/simple_exporter_spec.rb
+++ b/spec/models/simple_exporter_spec.rb
@@ -44,29 +44,75 @@ RSpec.describe SimpleExporter do
     let(:filename){ "stop_area.csv" }
     # let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) }
 
-    before(:each) do
-      @stop_area = create :stop_area
-      SimpleExporter.define :test do |config|
-        config.collection = ->{ Chouette::StopArea.all }
-        config.separator = ";"
-        config.add_column :name
-        config.add_column :lat, attribute: :latitude
-        config.add_column :lng, attribute: :latitude, value: ->(raw){ raw.to_f + 1 }
-        config.add_column :type, attribute: :area_type
-        config.add_column :street_name, value: "Lil Exporter"
+    context "with one row per item" do
+      before(:each) do
+        @stop_area = create :stop_area
+        SimpleExporter.define :test do |config|
+          config.collection = ->{ Chouette::StopArea.all }
+          config.separator = ";"
+          config.add_column :name
+          config.add_column :lat, attribute: :latitude
+          config.add_column :lng, attribute: [:latitude, :to_i, :next]
+          config.add_column :type, attribute: :area_type
+          config.add_column :street_name, value: "Lil Exporter"
+          config.on_relation :stop_area_referential do
+            config.add_column :stop_area_referential_id, attribute: :id
+            config.add_column :stop_area_referential_id_other, value: ->(item){ item.id }
+          end
+          config.add_column :forty_two, value: 42
+        end
+      end
+
+      it "should export the given file" do
+        expect{exporter.export verbose: true}.to_not raise_error
+        expect(exporter.status).to eq "success"
+        expect(File.exists?(filepath)).to be_truthy
+        csv = CSV.read(filepath, headers: true, col_sep: ";")
+        row = csv.by_row.values_at(0).last
+        expect(row["name"]).to eq @stop_area.name
+        expect(row["lat"]).to eq @stop_area.latitude.to_s
+        expect(row["lng"]).to eq (@stop_area.latitude.to_i + 1).to_s
+        expect(row["street_name"]).to eq "Lil Exporter"
+        expect(row["stop_area_referential_id"]).to eq @stop_area.stop_area_referential_id.to_s
+        expect(row["stop_area_referential_id_other"]).to eq @stop_area.stop_area_referential_id.to_s
+        expect(row["forty_two"]).to eq "42"
       end
     end
 
-    it "should export the given file" do
-      expect{exporter.export verbose: true}.to_not raise_error
-      expect(exporter.status).to eq "success"
-      expect(File.exists?(filepath)).to be_truthy
-      csv = CSV.read(filepath, headers: true, col_sep: ";")
-      row = csv.by_row.values_at(0).last
-      expect(row["name"]).to eq @stop_area.name
-      expect(row["lat"]).to eq @stop_area.latitude.to_s
-      expect(row["lng"]).to eq (@stop_area.latitude.to_f + 1).to_s
-      expect(row["street_name"]).to eq "Lil Exporter"
+    context "with several rows for the same object" do
+      before(:each) do
+        @stop_area = create :stop_area
+        @stop_1 = create :stop_point, stop_area: @stop_area
+        @stop_2 = create :stop_point, stop_area: @stop_area
+        SimpleExporter.define :test do |config|
+          config.collection = ->{ Chouette::StopArea.all }
+          config.map_item_to_rows do |stop_area|
+            stop_area.stop_points.map do |sp|
+              {
+                id: sp.id,
+                stop_area: stop_area
+              }
+            end
+          end
+          config.add_column :id
+          config.on_relation :stop_area do
+            config.add_column :stop_area_name, attribute: :name
+          end
+        end
+      end
+
+      it "should export the given file" do
+        expect{exporter.export verbose: true}.to_not raise_error
+        expect(exporter.status).to eq "success"
+        expect(File.exists?(filepath)).to be_truthy
+        csv = CSV.read(filepath, headers: true)
+        row = csv.by_row.values_at(0).last
+        expect(row["id"]).to eq @stop_1.id.to_s
+        expect(row["stop_area_name"]).to eq @stop_area.name
+        row = csv.by_row.values_at(1).last
+        expect(row["id"]).to eq @stop_2.id.to_s
+        expect(row["stop_area_name"]).to eq @stop_area.name
+      end
     end
   end
 end