diff options
Diffstat (limited to 'app/models')
| -rw-r--r-- | app/models/compliance_check_set.rb | 9 | ||||
| -rw-r--r-- | app/models/concerns/iev_interfaces/resource.rb | 20 | ||||
| -rw-r--r-- | app/models/concerns/iev_interfaces/task.rb | 26 | ||||
| -rw-r--r-- | app/models/import/base.rb | 9 | ||||
| -rw-r--r-- | app/models/import/gtfs.rb | 46 | ||||
| -rw-r--r-- | app/models/import/netex.rb | 62 | ||||
| -rw-r--r-- | app/models/import/resource.rb | 44 | ||||
| -rw-r--r-- | app/models/import/workbench.rb | 10 | ||||
| -rw-r--r-- | app/models/referential.rb | 4 | ||||
| -rw-r--r-- | app/models/workbench.rb | 14 | ||||
| -rw-r--r-- | app/models/workgroup.rb | 34 |
11 files changed, 254 insertions, 24 deletions
diff --git a/app/models/compliance_check_set.rb b/app/models/compliance_check_set.rb index 8b1dbdd68..f29ff4de5 100644 --- a/app/models/compliance_check_set.rb +++ b/app/models/compliance_check_set.rb @@ -26,6 +26,10 @@ class ComplianceCheckSet < ApplicationModel %w(successful failed warning aborted canceled) end + def successful? + status.to_s == "successful" + end + def self.abort_old where( 'created_at < ? AND status NOT IN (?)', @@ -68,6 +72,11 @@ class ComplianceCheckSet < ApplicationModel end update attributes + import_resource&.next_step + end + + def import_resource + referential&.import_resources.main_resources.last end diff --git a/app/models/concerns/iev_interfaces/resource.rb b/app/models/concerns/iev_interfaces/resource.rb index 7f8c3eefd..254f88a33 100644 --- a/app/models/concerns/iev_interfaces/resource.rb +++ b/app/models/concerns/iev_interfaces/resource.rb @@ -4,6 +4,24 @@ module IevInterfaces::Resource included do extend Enumerize enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true - validates_presence_of :name, :resource_type, :reference + validates_presence_of :name, :resource_type + end + + def update_status_from_importer importer_status + self.update status: status_from_importer(importer_status) + end + + def status_from_importer importer_status + return nil unless importer_status.present? + { + new: nil, + pending: nil, + successful: :OK, + warning: :WARNING, + failed: :ERROR, + running: nil, + aborted: :ERROR, + canceled: :ERROR + }[importer_status.to_sym] end end diff --git a/app/models/concerns/iev_interfaces/task.rb b/app/models/concerns/iev_interfaces/task.rb index 6be33734b..e40808009 100644 --- a/app/models/concerns/iev_interfaces/task.rb +++ b/app/models/concerns/iev_interfaces/task.rb @@ -31,6 +31,16 @@ module IevInterfaces::Task before_save :initialize_fields, on: :create after_save :notify_parent + + status.values.each do |s| + define_method "#{s}!" do + update status: s + end + + define_method "#{s}?" do + status&.to_s == s + end + end end module ClassMethods @@ -56,13 +66,14 @@ module IevInterfaces::Task end def notify_parent - return unless self.class.finished_statuses.include?(status) + return false unless self.class.finished_statuses.include?(status) - return unless parent.present? - return if notified_parent_at + return false unless parent.present? + return false if notified_parent_at parent.child_change update_column :notified_parent_at, Time.now + true end def children_succeedeed @@ -94,6 +105,10 @@ module IevInterfaces::Task update attributes end + def successful? + status.to_s == "successful" + end + def child_change return if self.class.finished_statuses.include?(status) update_status @@ -112,9 +127,14 @@ module IevInterfaces::Task def call_boiv_iev Rails.logger.error("Begin IEV call for import") + + # Java code expects tasks in NEW status + # Don't change status before calling iev + Net::HTTP.get iev_callback_url Rails.logger.error("End IEV call for import") rescue Exception => e + aborted! logger.error "IEV server error : #{e.message}" logger.error e.backtrace.inspect end diff --git a/app/models/import/base.rb b/app/models/import/base.rb index f98e359d4..dcd710e58 100644 --- a/app/models/import/base.rb +++ b/app/models/import/base.rb @@ -32,8 +32,13 @@ class Import::Base < ApplicationModel Rails.logger.info "update_referentials for #{inspect}" return unless self.class.finished_statuses.include?(status) - children.each do |import| - import.referential.update(ready: true) if import.referential + # We treat all created referentials in a batch + # If a single fails, we consider they all failed + # Ohana means family ! + if self.successful? + children.map(&:referential).compact.each &:active! + else + children.map(&:referential).compact.each &:failed! end end diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb index ceb849bd8..9dab11f0e 100644 --- a/app/models/import/gtfs.rb +++ b/app/models/import/gtfs.rb @@ -1,19 +1,37 @@ class Import::Gtfs < Import::Base after_commit :launch_worker, :on => :create + after_commit do + main_resource.update_status_from_importer self.status + true + end + def launch_worker GtfsImportWorker.perform_async id end + def main_resource + @resource ||= parent.resources.find_or_create_by(name: self.name, resource_type: "referential", reference: self.name) if parent + end + + def next_step + main_resource&.next_step + end + + def create_message args + (main_resource || self).messages.build args + end + def import update status: 'running', started_at: Time.now import_without_status update status: 'successful', ended_at: Time.now - referential&.ready! + referential&.active! rescue Exception => e update status: 'failed', ended_at: Time.now Rails.logger.error "Error in GTFS import: #{e} #{e.backtrace.join('\n')}" + create_message criticity: :error, message_key: :full_text, message_attributes: {text: e.message} referential&.failed! ensure notify_parent @@ -35,6 +53,7 @@ class Import::Gtfs < Import::Base workbench_id: workbench.id, metadatas: [referential_metadata] ) + main_resource.update referential: referential if main_resource end def referential_metadata @@ -131,17 +150,21 @@ class Import::Gtfs < Import::Base end def import_agencies + count = 0 Chouette::Company.transaction do source.agencies.each do |agency| company = line_referential.companies.find_or_initialize_by(registration_number: agency.id) company.attributes = { name: agency.name } save_model company + count += 1 end end + create_message criticity: "info", message_key: "gtfs.agencies.imported", message_attributes: {count: count} end def import_stops + count = 0 Chouette::StopArea.transaction do source.stops.each do |stop| stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id) @@ -155,11 +178,14 @@ class Import::Gtfs < Import::Base # TODO correct default timezone save_model stop_area + count += 1 end + create_message criticity: "info", message_key: "gtfs.stops.imported", message_attributes: {count: count} end end def import_routes + count = 0 Chouette::Line.transaction do source.routes.each do |route| line = line_referential.lines.find_or_initialize_by(registration_number: route.id) @@ -178,7 +204,9 @@ class Import::Gtfs < Import::Base line.url = route.url save_model line + count += 1 end + create_message criticity: "info", message_key: "gtfs.routes.imported", message_attributes: {count: count} end end @@ -187,6 +215,7 @@ class Import::Gtfs < Import::Base end def import_trips + count = 0 source.trips.each_slice(100) do |slice| slice.each do |trip| Chouette::Route.transaction do @@ -205,18 +234,20 @@ class Import::Gtfs < Import::Base vehicle_journey = journey_pattern.vehicle_journeys.build route: route vehicle_journey.published_journey_name = trip.headsign.presence || trip.id save_model vehicle_journey + count += 1 time_table = referential.time_tables.find_by(id: time_tables_by_service_id[trip.service_id]) if time_tables_by_service_id[trip.service_id] if time_table vehicle_journey.time_tables << time_table else - messages.create! criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id} + create_message criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id} end vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id end end end + create_message criticity: "info", message_key: "gtfs.trips.imported", message_attributes: {count: count} end def import_stop_times @@ -262,6 +293,7 @@ class Import::Gtfs < Import::Base end def import_calendars + count = 0 source.calendars.each_slice(500) do |slice| Chouette::TimeTable.transaction do slice.each do |calendar| @@ -272,11 +304,13 @@ class Import::Gtfs < Import::Base time_table.periods.build period_start: calendar.start_date, period_end: calendar.end_date save_model time_table + count += 1 time_tables_by_service_id[calendar.service_id] = time_table.id end end end + create_message criticity: "info", message_key: "gtfs.calendars.imported", message_attributes: {count: count} end def import_calendar_dates @@ -301,10 +335,10 @@ class Import::Gtfs < Import::Base end def notify_parent - return unless parent.present? - return if notified_parent_at - parent.child_change - update_column :notified_parent_at, Time.now + if super + main_resource.update_status_from_importer self.status + next_step + end end end diff --git a/app/models/import/netex.rb b/app/models/import/netex.rb index 49554ee90..b4422328c 100644 --- a/app/models/import/netex.rb +++ b/app/models/import/netex.rb @@ -2,16 +2,39 @@ require 'net/http' class Import::Netex < Import::Base before_destroy :destroy_non_ready_referential - after_commit :call_iev_callback, on: :create + after_commit do + main_resource.update_status_from_importer self.status + true + end before_save do - self.status = 'aborted' unless referential self.referential&.failed! if self.status == 'aborted' || self.status == 'failed' end validates_presence_of :parent + def main_resource + @resource ||= parent.resources.find_or_create_by(name: self.name, resource_type: "referential", reference: self.name) + end + + def notify_parent + if super + main_resource.update_status_from_importer self.status + next_step + end + end + + def next_step + main_resource.next_step + end + + def create_message args + main_resource.messages.create args + end + def create_with_referential! + save unless persisted? + self.referential = Referential.new( name: self.name, @@ -20,15 +43,37 @@ class Import::Netex < Import::Base metadatas: [referential_metadata] ) self.referential.save - if self.referential.invalid? + + if self.referential.valid? + main_resource.update referential: referential + call_iev_callback + save! + else Rails.logger.info "Can't create referential for import #{self.id}: #{referential.inspect} #{referential.metadatas.inspect} #{referential.errors.messages}" - if referential.metadatas.all?{|m| m.line_ids.present? && m.line_ids.empty?} - parent.messages.create criticity: :error, message_key: "referential_creation_missing_lines", message_attributes: {referential_name: referential.name} + + if referential.metadatas.all?{|m| m.line_ids.empty? && m.line_ids.empty?} + create_message criticity: :error, message_key: "referential_creation_missing_lines", message_attributes: {referential_name: referential.name} + elsif (overlapped_referential_ids = referential.overlapped_referential_ids).any? + overlapped = Referential.find overlapped_referential_ids.last + create_message( + criticity: :error, + message_key: "referential_creation_overlapping_existing_referential", + message_attributes: { + referential_name: referential.name, + overlapped_name: overlapped.name, + overlapped_url: Rails.application.routes.url_helpers.referential_path(overlapped) + } + ) else - parent.messages.create criticity: :error, message_key: "referential_creation", message_attributes: {referential_name: referential.name} + create_message( + criticity: :error, + message_key: "referential_creation", + message_attributes: {referential_name: referential.name}, + resource_attributes: referential.errors.messages + ) end - else - save! + self.referential = nil + aborted! end end @@ -55,6 +100,7 @@ class Import::Netex < Import::Base metadata.periodes = frame.periods line_objectids = frame.line_refs.map { |ref| "STIF:CODIFLIGNE:Line:#{ref}" } + create_message criticity: :info, message_key: "referential_creation_lines_found", message_attributes: {line_objectids: line_objectids.to_sentence} metadata.line_ids = workbench.lines.where(objectid: line_objectids).pluck(:id) end end diff --git a/app/models/import/resource.rb b/app/models/import/resource.rb index 1951daacd..43690755d 100644 --- a/app/models/import/resource.rb +++ b/app/models/import/resource.rb @@ -4,5 +4,49 @@ class Import::Resource < ApplicationModel include IevInterfaces::Resource belongs_to :import, class_name: Import::Base + belongs_to :referential has_many :messages, class_name: "Import::Message", foreign_key: :resource_id + + scope :main_resources, ->{ where(resource_type: "referential") } + + def root_import + import = self.import + import = import.parent while import.parent + import + end + + def next_step + if root_import.class == Import::Workbench + + return unless netex_import&.successful? + + workbench.workgroup.import_compliance_control_sets.map do |key, label| + next unless (control_set = workbench.compliance_control_set(key)).present? + compliance_check_set = workbench_import_check_set key + if compliance_check_set.nil? + ComplianceControlSetCopyWorker.perform_async control_set.id, referential_id, root_import.class.name, root_import.id + end + end + end + end + + def workbench + import.workbench + end + + def workgroup + workbench.workgroup + end + + def netex_import + return unless self.resource_type == "referential" + import.children.where(name: self.reference).last + end + + def workbench_import_check_set key + return unless referential.present? + control_set = referential.workbench.compliance_control_set(key) + return unless control_set.present? + referential.compliance_check_sets.where(compliance_control_set_id: control_set.id, referential_id: referential_id).last + end end diff --git a/app/models/import/workbench.rb b/app/models/import/workbench.rb index 124b9b0d8..95d23fe5b 100644 --- a/app/models/import/workbench.rb +++ b/app/models/import/workbench.rb @@ -9,14 +9,18 @@ class Import::Workbench < Import::Base end end + # def main_resource + # @resource ||= resources.find_or_create_by(name: self.name, resource_type: "workbench_import") + # end + def import_gtfs update_column :status, 'running' update_column :started_at, Time.now - Import::Gtfs.create! parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service" + Import::Gtfs.create! parent_type: self.class.name, parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service" - update_column :status, 'successful' - update_column :ended_at, Time.now + # update_column :status, 'successful' + # update_column :ended_at, Time.now rescue Exception => e Rails.logger.error "Error while processing GTFS file: #{e}" diff --git a/app/models/referential.rb b/app/models/referential.rb index b4f64fad1..0c6e71d47 100644 --- a/app/models/referential.rb +++ b/app/models/referential.rb @@ -26,6 +26,7 @@ class Referential < ApplicationModel has_one :user has_many :api_keys, class_name: 'Api::V1::ApiKey', dependent: :destroy + has_many :import_resources, class_name: 'Import::Resource', dependent: :destroy belongs_to :organisation validates_presence_of :organisation @@ -322,6 +323,7 @@ class Referential < ApplicationModel before_create :create_schema after_create :clone_schema, if: :created_from + after_create :active!, unless: :created_from before_destroy :destroy_schema before_destroy :destroy_jobs @@ -397,7 +399,7 @@ class Referential < ApplicationModel query = "select distinct(public.referential_metadata.referential_id) FROM public.referential_metadata, unnest(line_ids) line, LATERAL unnest(periodes) period WHERE public.referential_metadata.referential_id - IN (SELECT public.referentials.id FROM public.referentials WHERE referentials.workbench_id = #{workbench_id} and referentials.archived_at is null and referentials.referential_suite_id is null #{not_myself}) + IN (SELECT public.referentials.id FROM public.referentials WHERE referentials.workbench_id = #{workbench_id} and referentials.archived_at is null and referentials.referential_suite_id is null #{not_myself} AND referentials.failed_at IS NULL) AND line in (#{line_ids.join(',')}) and (#{periods_query});" self.class.connection.select_values(query).map(&:to_i) diff --git a/app/models/workbench.rb b/app/models/workbench.rb index 1c54e8904..1bca91c56 100644 --- a/app/models/workbench.rb +++ b/app/models/workbench.rb @@ -51,6 +51,20 @@ class Workbench < ApplicationModel where(name: DEFAULT_WORKBENCH_NAME).last end + # XXX + # def import_compliance_control_set + # import_compliance_control_set_id && ComplianceControlSet.find(import_compliance_control_set_id) + # end + + def compliance_control_set key + id = (owner_compliance_control_set_ids || {})[key.to_s] + id.present? && ComplianceControlSet.find(id) + end + + def compliance_control_set_ids=(compliance_control_set_ids) + self.owner_compliance_control_set_ids = (owner_compliance_control_set_ids || {}).merge compliance_control_set_ids + end + private def initialize_output diff --git a/app/models/workgroup.rb b/app/models/workgroup.rb index 3e8409634..6bb03c7fa 100644 --- a/app/models/workgroup.rb +++ b/app/models/workgroup.rb @@ -1,6 +1,7 @@ class Workgroup < ApplicationModel belongs_to :line_referential belongs_to :stop_area_referential + belongs_to :owner, class_name: "Organisation" has_many :workbenches has_many :calendars @@ -16,6 +17,8 @@ class Workgroup < ApplicationModel has_many :custom_fields + accepts_nested_attributes_for :workbenches + def custom_fields_definitions Hash[*custom_fields.map{|cf| [cf.code, cf]}.flatten] end @@ -23,4 +26,35 @@ class Workgroup < ApplicationModel def has_export? export_name export_types.include? export_name end + + def all_compliance_control_sets + %i(after_import + after_import_by_workgroup + before_merge + before_merge_by_workgroup + after_merge + after_merge_by_workgroup + automatic_by_workgroup + ) + end + + def compliance_control_sets_by_workgroup + compliance_control_sets_labels all_compliance_control_sets.grep(/by_workgroup$/) + end + + def compliance_control_sets_by_workbench + compliance_control_sets_labels all_compliance_control_sets.grep_v(/by_workgroup$/) + end + + def import_compliance_control_sets + compliance_control_sets_labels all_compliance_control_sets.grep(/^after_import/) + end + + private + def compliance_control_sets_labels(keys) + keys.inject({}) do |h, k| + h[k] = "workgroups.compliance_control_sets.#{k}".t.capitalize + h + end + end end |