aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--app/models/chouette/vehicle_journey.rb9
-rw-r--r--app/models/concerns/application_days_support.rb47
-rw-r--r--app/models/simple_exporter.rb116
-rw-r--r--app/models/simple_interface.rb14
-rw-r--r--lib/tasks/exports.rake22
-rw-r--r--lib/tasks/helpers/simple_interfaces.rb2
-rw-r--r--spec/models/simple_exporter_spec.rb86
7 files changed, 211 insertions, 85 deletions
diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb
index 9b94f7f0e..49a2b387e 100644
--- a/app/models/chouette/vehicle_journey.rb
+++ b/app/models/chouette/vehicle_journey.rb
@@ -89,7 +89,6 @@ module Chouette
end
}
-
scope :in_purchase_window, ->(range){
purchase_windows = Chouette::PurchaseWindow.overlap_dates(range)
sql = purchase_windows.joins(:vehicle_journeys).select('vehicle_journeys.id').uniq.to_sql
@@ -156,6 +155,14 @@ module Chouette
end
end
+ def sales_start
+ purchase_windows.map{|p| p.date_ranges.first}.min
+ end
+
+ def sales_end
+ purchase_windows.map{|p| p.date_ranges.last}.max
+ end
+
def calculate_vehicle_journey_at_stop_day_offset
Chouette::VehicleJourneyAtStopsDayOffset.new(
vehicle_journey_at_stops
diff --git a/app/models/concerns/application_days_support.rb b/app/models/concerns/application_days_support.rb
index 348436aa4..2d00b5847 100644
--- a/app/models/concerns/application_days_support.rb
+++ b/app/models/concerns/application_days_support.rb
@@ -15,41 +15,34 @@ module ApplicationDaysSupport
end
def day_by_mask(flag)
- int_day_types & flag == flag
+ self.class.day_by_mask int_day_types, flag
end
- def self.day_by_mask(int_day_types,flag)
- int_day_types & flag == flag
+ def valid_day? wday
+ valid_days.include?(wday)
end
- def valid_days
- # Build an array with day of calendar week (1-7, Monday is 1).
- [].tap do |valid_days|
- valid_days << 1 if monday
- valid_days << 2 if tuesday
- valid_days << 3 if wednesday
- valid_days << 4 if thursday
- valid_days << 5 if friday
- valid_days << 6 if saturday
- valid_days << 7 if sunday
+ included do
+ def self.valid_days(int_day_types)
+ # Build an array with day of calendar week (1-7, Monday is 1).
+ [].tap do |valid_days|
+ valid_days << 1 if day_by_mask(int_day_types,MONDAY)
+ valid_days << 2 if day_by_mask(int_day_types,TUESDAY)
+ valid_days << 3 if day_by_mask(int_day_types,WEDNESDAY)
+ valid_days << 4 if day_by_mask(int_day_types,THURSDAY)
+ valid_days << 5 if day_by_mask(int_day_types,FRIDAY)
+ valid_days << 6 if day_by_mask(int_day_types,SATURDAY)
+ valid_days << 7 if day_by_mask(int_day_types,SUNDAY)
+ end
end
- end
- def valid_day? wday
- valid_days.include?(wday)
+ def self.day_by_mask(int_day_types,flag)
+ int_day_types & flag == flag
+ end
end
- def self.valid_days(int_day_types)
- # Build an array with day of calendar week (1-7, Monday is 1).
- [].tap do |valid_days|
- valid_days << 1 if day_by_mask(int_day_types,MONDAY)
- valid_days << 2 if day_by_mask(int_day_types,TUESDAY)
- valid_days << 3 if day_by_mask(int_day_types,WEDNESDAY)
- valid_days << 4 if day_by_mask(int_day_types,THURSDAY)
- valid_days << 5 if day_by_mask(int_day_types,FRIDAY)
- valid_days << 6 if day_by_mask(int_day_types,SATURDAY)
- valid_days << 7 if day_by_mask(int_day_types,SUNDAY)
- end
+ def valid_days
+ self.class.valid_days int_day_types
end
def monday
diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb
index f9b1c0a80..9c12a4e02 100644
--- a/app/models/simple_exporter.rb
+++ b/app/models/simple_exporter.rb
@@ -10,10 +10,13 @@ class SimpleExporter < SimpleInterface
@padding = 1
@current_line = -1
@number_of_lines = collection.size
+
@padding = [1, Math.log(@number_of_lines, 10).ceil()].max
@csv = nil
fail_with_error "Unable to write in file: #{self.filepath}" do
+ dir = Pathname.new(self.filepath).dirname
+ FileUtils.mkdir_p dir
@csv = CSV.open(self.filepath, 'w', self.configuration.csv_options)
end
@@ -54,56 +57,100 @@ class SimpleExporter < SimpleInterface
log "Starting export ...", color: :green
log "Export will be written in #{filepath}", color: :green
@csv << self.configuration.columns.map(&:name)
- ids = collection.pluck :id
- ids.in_groups_of(configuration.batch_size).each do |batch_ids|
- collection.where(id: batch_ids).each do |item|
- @current_row = item.attributes
- @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
- row = []
- @new_status = nil
- self.configuration.columns.each do |col|
- val = col[:value]
- if val.nil? || val.is_a?(Proc)
- if item.respond_to? col.attribute
- if val.is_a?(Proc)
- val = instance_exec(item.send(col.attribute), &val)
- else
- val = item.send(col.attribute)
- end
- else
- push_in_journal({event: :attribute_not_found, message: "Attribute not found: #{col.attribute}", kind: :warning})
- self.status ||= :success_with_warnings
- end
+ if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id")
+ ids = collection.pluck :id
+ ids.in_groups_of(configuration.batch_size).each do |batch_ids|
+ collection.where(id: batch_ids).each do |item|
+ handle_item item
+ end
+ end
+ else
+ collection.each{|item| handle_item item }
+ end
+ print_state
+ end
+
+ def map_item_to_rows item
+ return [item] unless configuration.item_to_rows_mapping
+ configuration.item_to_rows_mapping.call(item).map {|row| CustomRow.new row }
+ end
+
+ def handle_item item
+ number_of_lines = @number_of_lines
+ map_item_to_rows(item).each_with_index do |item, i|
+ @number_of_lines = number_of_lines + i
+ @current_row = item.attributes
+ @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present?
+ row = []
+ @new_status = nil
+ self.configuration.columns.each do |col|
+ scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)}
+ val = col[:value]
+ if val.nil? || val.is_a?(Proc)
+ if val.is_a?(Proc)
+ val = instance_exec(scoped_item, &val)
+ else
+ attributes = [col.attribute].flatten
+ val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr)}
end
+ end
+ if val.nil?
+ push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning})
+ self.status ||= :success_with_warnings
+ end
- if val.nil? && col.required?
- @new_status = colorize("x", :red)
- raise "MISSING VALUE FOR COLUMN #{col.name}"
+ if val.nil? && col.required?
+ @new_status = colorize("x", :red)
+ raise "MISSING VALUE FOR COLUMN #{col.name}"
+ end
+ @new_status ||= colorize("✓", :green)
+ val = encode_string(val) if val.is_a?(String)
+ row << val
+ end
+ push_in_journal({event: :success, kind: :log})
+ @statuses += @new_status
+ print_state if @current_line % 20 == 0
+ @current_line += 1
+ @csv << row
+ end
+ end
+
+ class CustomRow < OpenStruct
+ def initialize data
+ super data
+ @data = data
+ end
+
+ def attributes
+ flatten_hash @data
+ end
+
+ protected
+ def flatten_hash h
+ h.each_with_object({}) do |(k, v), h|
+ if v.is_a? Hash
+ flatten_hash(v).map do |h_k, h_v|
+ h["#{k}.#{h_k}".to_sym] = h_v
end
- @new_status ||= colorize("✓", :green)
- val = encode_string(val) if val.is_a?(String)
- row << val
+ else
+ h[k] = v
end
- push_in_journal({event: :success, kind: :log})
- @statuses += @new_status
- print_state if @current_line % 20 == 0
- @current_line += 1
- @csv << row
end
end
- print_state
end
class Configuration < SimpleInterface::Configuration
attr_accessor :collection
attr_accessor :batch_size
attr_accessor :logged_attributes
+ attr_accessor :item_to_rows_mapping
def initialize import_name, opts={}
super import_name, opts
@collection = opts[:collection]
@batch_size = opts[:batch_size] || 1000
@logged_attributes = opts[:logged_attributes]
+ @item_to_rows_mapping = opts[:item_to_rows_mapping]
end
def options
@@ -111,9 +158,14 @@ class SimpleExporter < SimpleInterface
collection: collection,
batch_size: batch_size,
logged_attributes: logged_attributes,
+ item_to_rows_mapping: item_to_rows_mapping,
})
end
+ def map_item_to_rows &block
+ @item_to_rows_mapping = block
+ end
+
def add_column name, opts={}
raise "Column already defined: #{name}" if @columns.any?{|c| c.name == name.to_s}
super name, opts
diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb
index 3d5027bf1..5f022719a 100644
--- a/app/models/simple_interface.rb
+++ b/app/models/simple_interface.rb
@@ -167,6 +167,13 @@ class SimpleInterface < ActiveRecord::Base
@scope = opts[:scope]
end
+ def on_relation relation_name
+ @scope ||= []
+ @scope.push relation_name
+ yield
+ @scope.pop
+ end
+
def duplicate
self.class.new @import_name, self.options
end
@@ -211,7 +218,8 @@ class SimpleInterface < ActiveRecord::Base
end
def add_column name, opts={}
- @columns.push Column.new({name: name.to_s}.update(opts))
+ @scope ||= []
+ @columns.push Column.new({name: name.to_s, scope: @scope.dup}.update(opts))
end
def add_value attribute, value
@@ -262,6 +270,10 @@ class SimpleInterface < ActiveRecord::Base
!!@options[:required]
end
+ def scope
+ @options[:scope] || []
+ end
+
def [](key)
@options[key]
end
diff --git a/lib/tasks/exports.rake b/lib/tasks/exports.rake
index 036d96b11..6ff73dac0 100644
--- a/lib/tasks/exports.rake
+++ b/lib/tasks/exports.rake
@@ -40,8 +40,24 @@ namespace :export do
SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
end
- desc "export a complete offer from the gicen referential in the given X next days"
- task :full_offer, [:referential_id, :timelapse, :configuration_name, :output_dir, :logs_output_dir] => :environment do |t, args|
- args.with_defaults(filepath: "./companies.csv", logs_output_dir: "./log/exporters/")
+ desc "export a complete offer from the given referential in the given X next days"
+ task :full_offer, [:referential_id, :configuration_name, :timelapse, :output_dir, :logs_output_dir] => :environment do |t, args|
+ referential = Referential.find args[:referential_id]
+ args.with_defaults(output_dir: referential.name, logs_output_dir: "./log/exporters/", timelapse: 90)
+
+ referential.switch
+
+ journeys = Chouette::VehicleJourney.with_matching_timetable (Time.now.to_date..args[:timelapse].to_i.days.from_now.to_date)
+ if journeys.count == 0
+ puts "No maching journeys were found".red
+ else
+
+ exporter = SimpleExporter.create configuration_name: "#{args[:configuration_name]}_journeys", filepath: "#{args[:output_dir]}/#{args[:configuration_name]}_journeys.csv"
+ exporter.configure do |config|
+ config.collection = journeys
+ end
+
+ SimpleInterfacesHelper.run_interface_controlling_interruption exporter, :export, args
+ end
end
end
diff --git a/lib/tasks/helpers/simple_interfaces.rb b/lib/tasks/helpers/simple_interfaces.rb
index 68e02e818..1dc051575 100644
--- a/lib/tasks/helpers/simple_interfaces.rb
+++ b/lib/tasks/helpers/simple_interfaces.rb
@@ -2,7 +2,7 @@ module SimpleInterfacesHelper
def self.interface_output_to_csv interface, output_dir
filepath = File.join output_dir, + "#{interface.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv"
cols = %w(line kind event message error)
- if interface.reload.journal.size > 0
+ if interface.reload.journal.size > 0 && interface.journal.first["row"].present?
keys = interface.journal.first["row"].map(&:first)
CSV.open(filepath, "w") do |csv|
csv << cols + keys
diff --git a/spec/models/simple_exporter_spec.rb b/spec/models/simple_exporter_spec.rb
index 395cb1034..18cdc55e8 100644
--- a/spec/models/simple_exporter_spec.rb
+++ b/spec/models/simple_exporter_spec.rb
@@ -44,29 +44,75 @@ RSpec.describe SimpleExporter do
let(:filename){ "stop_area.csv" }
# let(:stop_area_referential){ create(:stop_area_referential, objectid_format: :stif_netex) }
- before(:each) do
- @stop_area = create :stop_area
- SimpleExporter.define :test do |config|
- config.collection = ->{ Chouette::StopArea.all }
- config.separator = ";"
- config.add_column :name
- config.add_column :lat, attribute: :latitude
- config.add_column :lng, attribute: :latitude, value: ->(raw){ raw.to_f + 1 }
- config.add_column :type, attribute: :area_type
- config.add_column :street_name, value: "Lil Exporter"
+ context "with one row per item" do
+ before(:each) do
+ @stop_area = create :stop_area
+ SimpleExporter.define :test do |config|
+ config.collection = ->{ Chouette::StopArea.all }
+ config.separator = ";"
+ config.add_column :name
+ config.add_column :lat, attribute: :latitude
+ config.add_column :lng, attribute: [:latitude, :to_i, :next]
+ config.add_column :type, attribute: :area_type
+ config.add_column :street_name, value: "Lil Exporter"
+ config.on_relation :stop_area_referential do
+ config.add_column :stop_area_referential_id, attribute: :id
+ config.add_column :stop_area_referential_id_other, value: ->(item){ item.id }
+ end
+ config.add_column :forty_two, value: 42
+ end
+ end
+
+ it "should export the given file" do
+ expect{exporter.export verbose: true}.to_not raise_error
+ expect(exporter.status).to eq "success"
+ expect(File.exists?(filepath)).to be_truthy
+ csv = CSV.read(filepath, headers: true, col_sep: ";")
+ row = csv.by_row.values_at(0).last
+ expect(row["name"]).to eq @stop_area.name
+ expect(row["lat"]).to eq @stop_area.latitude.to_s
+ expect(row["lng"]).to eq (@stop_area.latitude.to_i + 1).to_s
+ expect(row["street_name"]).to eq "Lil Exporter"
+ expect(row["stop_area_referential_id"]).to eq @stop_area.stop_area_referential_id.to_s
+ expect(row["stop_area_referential_id_other"]).to eq @stop_area.stop_area_referential_id.to_s
+ expect(row["forty_two"]).to eq "42"
end
end
- it "should export the given file" do
- expect{exporter.export verbose: true}.to_not raise_error
- expect(exporter.status).to eq "success"
- expect(File.exists?(filepath)).to be_truthy
- csv = CSV.read(filepath, headers: true, col_sep: ";")
- row = csv.by_row.values_at(0).last
- expect(row["name"]).to eq @stop_area.name
- expect(row["lat"]).to eq @stop_area.latitude.to_s
- expect(row["lng"]).to eq (@stop_area.latitude.to_f + 1).to_s
- expect(row["street_name"]).to eq "Lil Exporter"
+ context "with several rows for the same object" do
+ before(:each) do
+ @stop_area = create :stop_area
+ @stop_1 = create :stop_point, stop_area: @stop_area
+ @stop_2 = create :stop_point, stop_area: @stop_area
+ SimpleExporter.define :test do |config|
+ config.collection = ->{ Chouette::StopArea.all }
+ config.map_item_to_rows do |stop_area|
+ stop_area.stop_points.map do |sp|
+ {
+ id: sp.id,
+ stop_area: stop_area
+ }
+ end
+ end
+ config.add_column :id
+ config.on_relation :stop_area do
+ config.add_column :stop_area_name, attribute: :name
+ end
+ end
+ end
+
+ it "should export the given file" do
+ expect{exporter.export verbose: true}.to_not raise_error
+ expect(exporter.status).to eq "success"
+ expect(File.exists?(filepath)).to be_truthy
+ csv = CSV.read(filepath, headers: true)
+ row = csv.by_row.values_at(0).last
+ expect(row["id"]).to eq @stop_1.id.to_s
+ expect(row["stop_area_name"]).to eq @stop_area.name
+ row = csv.by_row.values_at(1).last
+ expect(row["id"]).to eq @stop_2.id.to_s
+ expect(row["stop_area_name"]).to eq @stop_area.name
+ end
end
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-17 06:30:03 +0000