aboutsummaryrefslogtreecommitdiffstats
path: root/app/models
diff options
context:
space:
mode:
Diffstat (limited to 'app/models')
-rw-r--r--app/models/chouette/company.rb1
-rw-r--r--app/models/chouette/line.rb10
-rw-r--r--app/models/chouette/purchase_window.rb1
-rw-r--r--app/models/chouette/route.rb12
-rw-r--r--app/models/chouette/stop_area.rb1
-rw-r--r--app/models/chouette/time_table.rb4
-rw-r--r--app/models/simple_importer.rb276
7 files changed, 224 insertions, 81 deletions
diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb
index b3d40ab96..2e942d520 100644
--- a/app/models/chouette/company.rb
+++ b/app/models/chouette/company.rb
@@ -15,6 +15,7 @@ module Chouette
[:organizational_unit, :operating_department_name, :code, :phone, :fax, :email, :url, :time_zone]
end
+ def local_id; id end
end
end
diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb
index 874353752..5df45efbd 100644
--- a/app/models/chouette/line.rb
+++ b/app/models/chouette/line.rb
@@ -49,7 +49,7 @@ module Chouette
[:published_name, :number, :comment, :url, :color, :text_color, :stable_id]
end
- def local_id; registration_number end
+ def local_id; registration_number || id end
def geometry_presenter
Chouette::Geometry::LinePresenter.new self
@@ -83,6 +83,14 @@ module Chouette
line_referential.companies.where(id: ([company_id] + Array(secondary_company_ids)).compact)
end
+ def deactivate
+ self.deactivated = true
+ end
+
+ def activate
+ self.deactivated = false
+ end
+
def deactivate!
update_attribute :deactivated, true
end
diff --git a/app/models/chouette/purchase_window.rb b/app/models/chouette/purchase_window.rb
index 334493015..157390a21 100644
--- a/app/models/chouette/purchase_window.rb
+++ b/app/models/chouette/purchase_window.rb
@@ -19,6 +19,7 @@ module Chouette
scope :contains_date, ->(date) { where('date ? <@ any (date_ranges)', date) }
scope :overlap_dates, ->(date_range) { where('daterange(?, ?) && any (date_ranges)', date_range.first, date_range.last + 1.day) }
+ scope :matching_dates, ->(date_range) { where('ARRAY[daterange(?, ?)] = date_ranges', date_range.first, date_range.last + 1.day) }
def self.ransackable_scopes(auth_object = nil)
[:contains_date]
diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb
index e418134de..3729deb7d 100644
--- a/app/models/chouette/route.rb
+++ b/app/models/chouette/route.rb
@@ -186,15 +186,9 @@ module Chouette
end
def full_journey_pattern
- out = journey_patterns.find{|jp| jp.stop_points.count == self.stop_points.count }
- unless out
- out = journey_patterns.build name: self.name
- self.stop_points.each do |sp|
- out.stop_points.build stop_area: sp.stop_area, position: sp.position
- end
- out.save!
- end
- out
+ journey_pattern = journey_patterns.find_or_create_by registration_number: self.number, name: self.name
+ journey_pattern.stop_points = self.stop_points
+ journey_pattern
end
protected
diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb
index c85a8b37d..699616863 100644
--- a/app/models/chouette/stop_area.rb
+++ b/app/models/chouette/stop_area.rb
@@ -112,7 +112,6 @@ module Chouette
id.to_s
end
end
- alias_method :local_id, :user_objectid
alias_method :local_id, :user_objectid
diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb
index 15b22b671..b76de852a 100644
--- a/app/models/chouette/time_table.rb
+++ b/app/models/chouette/time_table.rb
@@ -44,10 +44,10 @@ module Chouette
attrs << self.int_day_types
dates = self.dates
dates += TimeTableDate.where(time_table_id: self.id)
- attrs << dates.map(&:checksum).map(&:to_s).sort
+ attrs << dates.map(&:checksum).map(&:to_s).uniq.sort
periods = self.periods
periods += TimeTablePeriod.where(time_table_id: self.id)
- attrs << periods.map(&:checksum).map(&:to_s).sort
+ attrs << periods.map(&:checksum).map(&:to_s).uniq.sort
end
end
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index bf6f3b406..dea8f85ad 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -30,7 +30,7 @@ class SimpleImporter < ActiveRecord::Base
end
def context
- self.configuration.context || {}
+ self.configuration.context
end
def resolve col_name, value, &block
@@ -42,99 +42,183 @@ class SimpleImporter < ActiveRecord::Base
def import opts={}
@verbose = opts.delete :verbose
+
+
@resolution_queue = Hash.new{|h,k| h[k] = []}
- number_of_lines = 0
- padding = 1
+ @errors = []
+ @messages = []
+ @number_of_lines = 0
+ @padding = 1
+ @current_line = 0
fail_with_error "File not found: #{self.filepath}" do
- number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
- padding = [1, Math.log(number_of_lines, 10).ceil()].max
+ @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length
+ @padding = [1, Math.log(@number_of_lines, 10).ceil()].max
+ end
+
+
+ self.configuration.before_actions(:parsing).each do |action| action.call self end
+
+ @statuses = ""
+
+ if ENV["NO_TRANSACTION"]
+ process_csv_file
+ else
+ ActiveRecord::Base.transaction do
+ process_csv_file
+ end
+ end
+ self.status ||= :success
+ rescue FailedImport
+ self.status = :failed
+ ensure
+ self.save!
+ end
+
+ def fail_with_error msg=nil, opts={}
+ begin
+ yield
+ rescue => e
+ msg = msg.call if msg.is_a?(Proc)
+ custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures
+ push_in_journal({message: msg, error: e.message, event: :error, kind: :error})
+ @new_status = colorize("x", :red)
+ if self.configuration.ignore_failures
+ raise FailedRow if opts[:abort_row]
+ else
+ raise FailedImport
+ end
end
+ end
+
+ def encode_string s
+ s.encode("utf-8").force_encoding("utf-8")
+ end
+
+ def dump_csv_from_context
+ filepath = "./#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
+ # for some reason, context[:csv].to_csv does not work
+ CSV.open(filepath, 'w') do |csv|
+ header = true
+ context[:csv].each do |row|
+ csv << row.headers if header
+ csv << row.fields
+ header = false
+ end
+ end
+ log "CSV file dumped in #{filepath}"
+ end
+
+ def log msg, opts={}
+ msg = colorize msg, opts[:color] if opts[:color]
+ if opts[:append]
+ @messages[-1] = (@messages[-1] || "") + msg
+ else
+ @messages << msg
+ end
+ print_state
+ end
- current_line = 0
- status = :success
- statuses = ""
- log "#{"%#{padding}d" % 0}/#{number_of_lines}", clear: true
- ActiveRecord::Base.transaction do
- self.configuration.before_actions(:all).each do |action| action.call self end
+ protected
- CSV.foreach(filepath, self.configuration.csv_options) do |row|
- status = handle_row row, status
+ def process_csv_file
+ self.configuration.before_actions(:all).each do |action| action.call self end
+ log "Starting import ...", color: :green
+ (context[:csv] || CSV.read(filepath, self.configuration.csv_options)).each do |row|
+ @current_row = row
+ @new_status = nil
+ begin
+ handle_row row
fail_with_error ->(){ @current_record.errors.messages } do
new_record = @current_record.new_record?
+ @new_status ||= new_record ? colorize("✓", :green) : colorize("-", :orange)
+ @event = new_record ? :creation : :update
self.configuration.before_actions(:each_save).each do |action|
action.call self, @current_record
end
### This could fail if the record has a mandatory relation which is not yet resolved
### TODO: do not attempt to save if the current record if waiting for resolution
### and fail at the end if there remains unresolved relations
- @current_record.save!
- self.journal.push({event: (new_record ? :creation : :update), kind: :log})
- statuses += new_record ? colorize("✓", :green) : colorize("-", :orange)
- end
- self.configuration.columns.each do |col|
- if col.name && @resolution_queue.any?
- val = @current_record.send col[:attribute]
- (@resolution_queue.delete([col.name, val]) || []).each do |res|
- record = res[:record]
- attribute = res[:attribute]
- value = res[:block].call(val, record)
- record.send "#{attribute}=", value
- record.save!
+ if self.configuration.ignore_failures
+ unless @current_record.save
+ @new_status = colorize("x", :red)
+ push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error})
end
+ else
+ @current_record.save!
+ end
+ self.configuration.after_actions(:each_save).each do |action|
+ action.call self, @current_record
end
end
- current_line += 1
- log "#{"%#{padding}d" % current_line}/#{number_of_lines}: #{statuses}", clear: true
+ rescue FailedRow
+ @new_status = colorize("x", :red)
end
- self.configuration.after_actions(:all).each do |action|
- action.call self
+ push_in_journal({event: @event, kind: :log}) if @current_record&.valid?
+ @statuses += @new_status
+ self.configuration.columns.each do |col|
+ if col.name && @resolution_queue.any?
+ val = @current_record.send col[:attribute]
+ (@resolution_queue.delete([col.name, val]) || []).each do |res|
+ record = res[:record]
+ attribute = res[:attribute]
+ value = res[:block].call(val, record)
+ record.send "#{attribute}=", value
+ record.save!
+ end
end
end
- self.update_attribute :status, status
- rescue FailedImport
- self.update_attribute :status, :failed
- ensure
- self.save!
- end
+ print_state
+ @current_line += 1
+ end
- def fail_with_error msg
begin
- yield
- rescue => e
- msg = msg.call if msg.is_a?(Proc)
- log "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red
- self.journal.push({message: msg, error: e.message, event: :error, kind: :error})
- raise FailedImport
+ self.configuration.after_actions(:all).each do |action|
+ action.call self
+ end
+ rescue FailedRow
end
end
- protected
-
- def handle_row row, status
+ def handle_row row
if self.configuration.get_custom_handler
instance_exec(row, &self.configuration.get_custom_handler)
else
- @current_record = self.configuration.find_record row
- self.configuration.columns.each do |col|
- @current_attribute = col[:attribute]
- val = col[:value]
- if val.nil? || val.is_a?(Proc)
- if row.has_key? col.name
- if val.is_a?(Proc)
- val = instance_exec(row[col.name], &val)
+ fail_with_error "", abort_row: true do
+ @current_record = self.configuration.find_record row
+ self.configuration.columns.each do |col|
+ @current_attribute = col[:attribute]
+ val = col[:value]
+ if val.nil? || val.is_a?(Proc)
+ if row.has_key? col.name
+ if val.is_a?(Proc)
+ val = instance_exec(row[col.name], &val)
+ else
+ val = row[col.name]
+ end
else
- val = row[col.name]
+ push_in_journal({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
+ self.status ||= :success_with_warnings
end
- else
- self.journal.push({event: :column_not_found, message: "Column not found: #{col.name}", kind: :warning})
- status = :success_with_warnings
end
+
+ if val.nil? && col.required?
+ raise "MISSING VALUE FOR COLUMN #{col.name}"
+ end
+ val = encode_string(val) if val.is_a?(String)
+ @current_record.send "#{@current_attribute}=", val if val
end
- @current_record.send "#{@current_attribute}=", val if val
end
end
- status
+ end
+
+ def push_in_journal data
+ line = @current_line + 1
+ line += 1 if configuration.headers
+ self.journal.push data.update(line: line, row: @current_row)
+ if data[:kind] == :error || data[:kind] == :warning
+ @errors.push data
+ end
end
def colorize txt, color
@@ -146,23 +230,59 @@ class SimpleImporter < ActiveRecord::Base
"\e[#{color}m#{txt}\e[0m"
end
- def log msg, opts={}
+ def print_state
+ return unless @verbose
+
+ @status_width ||= begin
+ term_width = %x(tput cols).to_i
+ term_width - @padding - 10
+ rescue
+ 100
+ end
+ full_status = @statuses || ""
+ full_status = full_status.last(@status_width*10) || ""
+ padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min
+ full_status = "#{full_status}#{"."*[padding_size, 0].max}"
+
+ msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}"
+
+ if @messages.any?
+ msg += "\n\n"
+ msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
+ msg += "[...]\n" if @messages.count > 10
+ msg += @messages.last(10).join("\n")
+ end
+
+ if @errors.any?
+ msg += "\n\n"
+ msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red
+ msg += "[...]\n" if @errors.count > 10
+ msg += @errors.last(10).map do |j|
+ kind = j[:kind]
+ kind = colorize(kind, kind == :error ? :red : :orange)
+ encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
+ end.join("\n")
+ end
+ custom_print msg, clear: true
+ end
+
+ def custom_print msg, opts={}
return unless @verbose
out = ""
msg = colorize(msg, opts[:color]) if opts[:color]
- if opts[:clear] && @prev_msg_size
- out += "\b"*@prev_msg_size
- end
+ puts "\e[H\e[2J" if opts[:clear]
out += msg
print out
- @prev_msg_size = msg.size
end
class FailedImport < RuntimeError
end
+ class FailedRow < RuntimeError
+ end
+
class Configuration
- attr_accessor :model, :headers, :separator, :key, :context, :encoding
+ attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope
attr_reader :columns
def initialize import_name, opts={}
@@ -174,6 +294,11 @@ class SimpleImporter < ActiveRecord::Base
@columns = opts[:columns] || []
@model = opts[:model]
@custom_handler = opts[:custom_handler]
+ @before = opts[:before]
+ @after = opts[:after]
+ @ignore_failures = opts[:ignore_failures]
+ @context = opts[:context] || {}
+ @scope = opts[:scope] || {}
end
def duplicate
@@ -188,7 +313,12 @@ class SimpleImporter < ActiveRecord::Base
encoding: @encoding,
columns: @columns.map(&:duplicate),
model: model,
- custom_handler: @custom_handler
+ custom_handler: @custom_handler,
+ before: @before,
+ after: @after,
+ ignore_failures: @ignore_failures,
+ context: @context,
+ scope: @scope
}
end
@@ -201,8 +331,14 @@ class SimpleImporter < ActiveRecord::Base
column && column[:attribute] || col_name
end
+ def record_scope
+ _scope = @scope
+ _scope = instance_exec(&_scope) if _scope.is_a?(Proc)
+ _scope || model
+ end
+
def find_record attrs
- model.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
+ record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s])
end
def csv_options
@@ -261,6 +397,10 @@ class SimpleImporter < ActiveRecord::Base
Column.new @options.dup
end
+ def required?
+ !!@options[:required]
+ end
+
def [](key)
@options[key]
end
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-01 17:25:51 +0000