diff options
Diffstat (limited to 'app/models')
113 files changed, 2858 insertions, 1409 deletions
diff --git a/app/models/api/v1/api_key.rb b/app/models/api/v1/api_key.rb index 09c6f77ac..e6ceb977a 100644 --- a/app/models/api/v1/api_key.rb +++ b/app/models/api/v1/api_key.rb @@ -1,7 +1,8 @@ module Api module V1 - class ApiKey < ::ActiveRecord::Base - has_paper_trail + class ApiKey < ::ApplicationModel + has_metadata + before_create :generate_access_token belongs_to :referential, :class_name => '::Referential' belongs_to :organisation, :class_name => '::Organisation' @@ -47,4 +48,3 @@ module Api end end end - diff --git a/app/models/application_model.rb b/app/models/application_model.rb new file mode 100644 index 000000000..1a2a5099d --- /dev/null +++ b/app/models/application_model.rb @@ -0,0 +1,5 @@ +class ApplicationModel < ::ActiveRecord::Base + include MetadataSupport + + self.abstract_class = true +end diff --git a/app/models/calendar.rb b/app/models/calendar.rb index 84b569ab4..39e2b2cff 100644 --- a/app/models/calendar.rb +++ b/app/models/calendar.rb @@ -2,18 +2,17 @@ require 'range_ext' require_relative 'calendar/date_value' require_relative 'calendar/period' -class Calendar < ActiveRecord::Base +class Calendar < ApplicationModel include DateSupport include PeriodSupport include ApplicationDaysSupport include TimetableSupport - has_paper_trail class_name: 'PublicVersion' + has_metadata belongs_to :organisation belongs_to :workgroup - validates_presence_of :name, :short_name, :organisation, :workgroup - validates_uniqueness_of :short_name + validates_presence_of :name, :organisation, :workgroup has_many :time_tables diff --git a/app/models/calendar/date_value.rb b/app/models/calendar/date_value.rb index a4a405d43..f50b4237c 100644 --- a/app/models/calendar/date_value.rb +++ b/app/models/calendar/date_value.rb @@ -1,4 +1,4 @@ -class Calendar < ActiveRecord::Base +class Calendar < ApplicationModel class DateValue include ActiveAttr::Model diff --git a/app/models/calendar/period.rb b/app/models/calendar/period.rb index 8b3e4109b..c549a7575 100644 --- a/app/models/calendar/period.rb +++ b/app/models/calendar/period.rb @@ -1,4 +1,4 @@ -class Calendar < ActiveRecord::Base +class Calendar < ApplicationModel class Period include ActiveAttr::Model @@ -16,7 +16,7 @@ class Calendar < ActiveRecord::Base alias_method :period_end=, :end= def check_end_greather_than_begin - if self.begin && self.end && self.begin >= self.end + if self.begin && self.end && self.begin > self.end errors.add(:base, I18n.t('calendars.errors.short_period')) end end diff --git a/app/models/calendar_observer.rb b/app/models/calendar_observer.rb index c81addff4..0414d01d2 100644 --- a/app/models/calendar_observer.rb +++ b/app/models/calendar_observer.rb @@ -3,7 +3,7 @@ class CalendarObserver < ActiveRecord::Observer def after_update calendar return unless calendar.shared - User.with_organisation.each do |user| + User.from_workgroup(calendar.workgroup_id).each do |user| MailerJob.perform_later('CalendarMailer', 'updated', [calendar.id, user.id]) end end @@ -11,7 +11,7 @@ class CalendarObserver < ActiveRecord::Observer def after_create calendar return unless calendar.shared - User.with_organisation.each do |user| + User.from_workgroup(calendar.workgroup_id).each do |user| MailerJob.perform_later('CalendarMailer', 'created', [calendar.id, user.id]) end end diff --git a/app/models/chouette/access_link.rb b/app/models/chouette/access_link.rb index 4b99ab5ba..7ab8ca715 100644 --- a/app/models/chouette/access_link.rb +++ b/app/models/chouette/access_link.rb @@ -1,9 +1,7 @@ module Chouette class AccessLink < Chouette::TridentActiveRecord - has_paper_trail + has_metadata include ObjectidSupport - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" attr_accessor :access_link_type, :link_orientation_type, :link_key diff --git a/app/models/chouette/access_point.rb b/app/models/chouette/access_point.rb index b6f78f239..884460881 100644 --- a/app/models/chouette/access_point.rb +++ b/app/models/chouette/access_point.rb @@ -4,9 +4,7 @@ require 'geo_ruby' module Chouette class AccessPoint < Chouette::ActiveRecord - has_paper_trail - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" + has_metadata include Geokit::Mappable include ProjectionFields diff --git a/app/models/chouette/active_record.rb b/app/models/chouette/active_record.rb index c2aab9d50..27f5426b3 100644 --- a/app/models/chouette/active_record.rb +++ b/app/models/chouette/active_record.rb @@ -1,7 +1,8 @@ #require "active_record" require 'deep_cloneable' module Chouette - class ActiveRecord < ::ActiveRecord::Base + class ActiveRecord < ::ApplicationModel + self.abstract_class = true before_save :nil_if_blank, :set_data_source_ref diff --git a/app/models/chouette/area_type.rb b/app/models/chouette/area_type.rb index e17d2ee8d..4feb5c914 100644 --- a/app/models/chouette/area_type.rb +++ b/app/models/chouette/area_type.rb @@ -34,9 +34,9 @@ class Chouette::AreaType @@options = {} end - def self.options(kind=:all) + def self.options(kind=:all, locale=nil) @@options ||= {} - @@options[kind] ||= self.send(kind).map { |c| find(c) }.map { |t| [ t.label, t.code ] } + @@options[kind] ||= self.send(kind).map { |c| find(c) }.map { |t| [ t.label(locale), t.code ] } end attr_reader :code @@ -48,8 +48,8 @@ class Chouette::AreaType all.index(code) <=> all.index(other.code) end - def label - I18n.translate code, scope: 'area_types.label' + def label locale=nil + I18n.translate code, scope: 'area_types.label', locale: locale end end diff --git a/app/models/chouette/company.rb b/app/models/chouette/company.rb index 53e412600..9d5737a6c 100644 --- a/app/models/chouette/company.rb +++ b/app/models/chouette/company.rb @@ -1,13 +1,15 @@ module Chouette class Company < Chouette::ActiveRecord + has_metadata + include CompanyRestrictions include LineReferentialSupport include ObjectidSupport - has_paper_trail class_name: 'PublicVersion' + include CustomFieldsSupport has_many :lines - validates_format_of :registration_number, :with => %r{\A[0-9A-Za-z_-]+\Z}, :allow_nil => true, :allow_blank => true + # validates_format_of :registration_number, :with => %r{\A[0-9A-Za-z_-]+\Z}, :allow_nil => true, :allow_blank => true validates_presence_of :name validates_format_of :url, :with => %r{\Ahttps?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?\Z}, :allow_nil => true, :allow_blank => true diff --git a/app/models/chouette/connection_link.rb b/app/models/chouette/connection_link.rb index d5ddc606a..fb93e5f90 100644 --- a/app/models/chouette/connection_link.rb +++ b/app/models/chouette/connection_link.rb @@ -1,10 +1,8 @@ module Chouette class ConnectionLink < Chouette::TridentActiveRecord - has_paper_trail + has_metadata include ObjectidSupport include ConnectionLinkRestrictions - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" attr_accessor :connection_link_type diff --git a/app/models/chouette/for_alighting_enumerations.rb b/app/models/chouette/for_alighting_enumerations.rb index ab07a670d..2e15fcb58 100644 --- a/app/models/chouette/for_alighting_enumerations.rb +++ b/app/models/chouette/for_alighting_enumerations.rb @@ -3,6 +3,6 @@ module Chouette extend Enumerize extend ActiveModel::Naming - enumerize :for_alighting, in: %w[normal forbidden request_stop is_flexible] + enumerize :for_alighting, in: %w[normal forbidden request_stop is_flexible], default: :normal end end diff --git a/app/models/chouette/for_boarding_enumerations.rb b/app/models/chouette/for_boarding_enumerations.rb index 48f8762c2..0190bf805 100644 --- a/app/models/chouette/for_boarding_enumerations.rb +++ b/app/models/chouette/for_boarding_enumerations.rb @@ -3,6 +3,6 @@ module Chouette extend Enumerize extend ActiveModel::Naming - enumerize :for_boarding, in: %w[normal forbidden request_stop is_flexible] + enumerize :for_boarding, in: %w[normal forbidden request_stop is_flexible], default: :normal end end diff --git a/app/models/chouette/group_of_line.rb b/app/models/chouette/group_of_line.rb index 75ee1ce73..a30c34ce7 100644 --- a/app/models/chouette/group_of_line.rb +++ b/app/models/chouette/group_of_line.rb @@ -1,12 +1,10 @@ module Chouette class GroupOfLine < Chouette::ActiveRecord - has_paper_trail + has_metadata include ObjectidSupport include GroupOfLineRestrictions include LineReferentialSupport - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" has_and_belongs_to_many :lines, :class_name => 'Chouette::Line', :order => 'lines.name' diff --git a/app/models/chouette/journey_pattern.rb b/app/models/chouette/journey_pattern.rb index 830e985d9..4b4cc2c73 100644 --- a/app/models/chouette/journey_pattern.rb +++ b/app/models/chouette/journey_pattern.rb @@ -1,11 +1,10 @@ module Chouette class JourneyPattern < Chouette::TridentActiveRecord - has_paper_trail + has_metadata include ChecksumSupport + include CustomFieldsSupport include JourneyPatternRestrictions include ObjectidSupport - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" belongs_to :route has_many :vehicle_journeys, :dependent => :destroy @@ -17,7 +16,6 @@ module Chouette validates_presence_of :name #validates :stop_points, length: { minimum: 2, too_short: :minimum }, on: :update - enum section_status: { todo: 0, completed: 1, control: 2 } attr_accessor :control_checked @@ -55,12 +53,19 @@ module Chouette end def self.state_permited_attributes item - { + attrs = { name: item['name'], published_name: item['published_name'], registration_number: item['registration_number'], costs: item['costs'] } + attrs["custom_field_values"] = Hash[ + *(item["custom_fields"] || {}) + .map { |k, v| [k, v["value"]] } + .flatten + ] + + attrs end def self.state_create_instance route, item @@ -81,10 +86,8 @@ module Chouette def state_stop_points_update item item['stop_points'].each do |sp| - exist = stop_area_ids.include?(sp['id']) - next if exist && sp['checked'] - - stop_point = route.stop_points.find_by(stop_area_id: sp['id']) + stop_point = route.stop_points.find_by(stop_area_id: sp['id'], position: sp['position']) + exist = stop_points.include?(stop_point) if !exist && sp['checked'] stop_points << stop_point end @@ -164,13 +167,48 @@ module Chouette next finish unless start.present? costs = costs_between(start, finish) full = false unless costs.present? - full = false unless costs[:distance] && costs[:distance] > 0 full = false unless costs[:time] && costs[:time] > 0 finish end full end + def distance_between start, stop + return 0 unless start.position < stop.position + val = 0 + i = stop_points.index(start) + _end = start + while _end && _end != stop + i += 1 + _start = _end + _end = stop_points[i] + val += costs_between(_start, _end)[:distance] || 0 + end + val + end + + def distance_to stop + distance_between stop_points.first, stop + end + + def journey_length + i = 0 + j = stop_points.length - 1 + start = stop_points[i] + stop = stop_points[j] + while i < j && start.kind == "non_commercial" + i+= 1 + start = stop_points[i] + end + + while i < j && stop.kind == "non_commercial" + j-= 1 + stop = stop_points[j] + end + return 0 unless start && stop + distance_between start, stop + end + def set_distances distances raise "inconsistent data: #{distances.count} values for #{stop_points.count} stops" unless distances.count == stop_points.count prev = distances[0].to_i diff --git a/app/models/chouette/line.rb b/app/models/chouette/line.rb index d077d5c9d..4b5d1a68d 100644 --- a/app/models/chouette/line.rb +++ b/app/models/chouette/line.rb @@ -1,15 +1,11 @@ module Chouette class Line < Chouette::ActiveRecord - has_paper_trail class_name: 'PublicVersion' + has_metadata include LineRestrictions include LineReferentialSupport include ObjectidSupport include StifTransportModeEnumerations include StifTransportSubmodeEnumerations - extend ActiveModel::Naming - - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" belongs_to :company belongs_to :network @@ -33,7 +29,7 @@ module Chouette # validates_presence_of :network # validates_presence_of :company - validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true + # validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true validates_format_of :stable_id, :with => %r{\A[\d\w_\-]+\Z}, :allow_nil => true, :allow_blank => true validates_format_of :url, :with => %r{\Ahttps?:\/\/([\da-z\.-]+)\.([a-z\.]{2,6})([\/\w \.-]*)*\/?\Z}, :allow_nil => true, :allow_blank => true validates_format_of :color, :with => %r{\A[0-9a-fA-F]{6}\Z}, :allow_nil => true, :allow_blank => true @@ -45,6 +41,24 @@ module Chouette scope :by_text, ->(text) { where('lower(name) LIKE :t or lower(published_name) LIKE :t or lower(objectid) LIKE :t or lower(comment) LIKE :t or lower(number) LIKE :t', t: "%#{text.downcase}%") } + scope :by_name, ->(name) { + joins('LEFT OUTER JOIN public.companies ON companies.id = lines.company_id') + .where(' + lines.number LIKE :q + OR lines.name LIKE :q + OR companies.name ILIKE :q', + q: "%#{sanitize_sql_like(name)}%" + ) + } + + scope :for_organisation, ->(organisation){ + if objectids = organisation&.lines_scope + where(objectid: objectids) + else + all + end + } + def self.nullable_attributes [:published_name, :number, :comment, :url, :color, :text_color, :stable_id] end diff --git a/app/models/chouette/network.rb b/app/models/chouette/network.rb index 6843c69ad..4802d7592 100644 --- a/app/models/chouette/network.rb +++ b/app/models/chouette/network.rb @@ -1,12 +1,10 @@ module Chouette class Network < Chouette::ActiveRecord - has_paper_trail class_name: 'PublicVersion' + has_metadata include NetworkRestrictions include LineReferentialSupport include ObjectidSupport extend Enumerize - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" has_many :lines attr_accessor :source_type_name diff --git a/app/models/chouette/pt_link.rb b/app/models/chouette/pt_link.rb index d14d5f29c..680632a14 100644 --- a/app/models/chouette/pt_link.rb +++ b/app/models/chouette/pt_link.rb @@ -2,9 +2,7 @@ require 'geokit' module Chouette class PtLink < Chouette::ActiveRecord - has_paper_trail - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" + has_metadata include Geokit::Mappable def geometry diff --git a/app/models/chouette/purchase_window.rb b/app/models/chouette/purchase_window.rb index 157390a21..bd7cbb41e 100644 --- a/app/models/chouette/purchase_window.rb +++ b/app/models/chouette/purchase_window.rb @@ -11,7 +11,7 @@ module Chouette enumerize :color, in: %w(#9B9B9B #FFA070 #C67300 #7F551B #41CCE3 #09B09C #3655D7 #6321A0 #E796C6 #DD2DAA) - has_paper_trail + has_metadata belongs_to :referential has_and_belongs_to_many :vehicle_journeys, :class_name => 'Chouette::VehicleJourney' @@ -21,6 +21,13 @@ module Chouette scope :overlap_dates, ->(date_range) { where('daterange(?, ?) && any (date_ranges)', date_range.first, date_range.last + 1.day) } scope :matching_dates, ->(date_range) { where('ARRAY[daterange(?, ?)] = date_ranges', date_range.first, date_range.last + 1.day) } + # VehicleJourneys include PurchaseWindow checksums in their checksums + # OPTIMIZEME + def update_vehicle_journey_checksums + vehicle_journeys.find_each(&:update_checksum!) + end + after_commit :update_vehicle_journey_checksums + def self.ransackable_scopes(auth_object = nil) [:contains_date] end @@ -35,12 +42,20 @@ module Chouette def checksum_attributes attrs = ['name', 'color', 'referential_id'] - ranges_attrs = date_ranges.map{|r| [r.first, r.last]}.flatten.sort + ranges_attrs = date_ranges.map{|r| [r.min, r.max]}.flatten.sort self.slice(*attrs).values + ranges_attrs end - # def checksum_attributes - # end + def bounding_dates + [ + date_ranges.map(&:first).min, + date_ranges.map(&:last).max, + ] + end + def color + _color = read_attribute(:color) + _color.present? ? _color : nil + end end end diff --git a/app/models/chouette/route.rb b/app/models/chouette/route.rb index 3729deb7d..928b65f13 100644 --- a/app/models/chouette/route.rb +++ b/app/models/chouette/route.rb @@ -1,20 +1,25 @@ - module Chouette class Route < Chouette::TridentActiveRecord - has_paper_trail + has_metadata + include RouteRestrictions include ChecksumSupport include ObjectidSupport - extend Enumerize - extend ActiveModel::Naming + + if ENV["CHOUETTE_ROUTE_POSITION_CHECK"] == "true" || !Rails.env.production? + after_commit do + positions = stop_points.pluck(:position) + Rails.logger.debug "Check positions in Route #{id} : #{positions.inspect}" + if positions.size != positions.uniq.size + raise "DUPLICATED stop_points positions in Route #{id} : #{positions.inspect}" + end + end + end enumerize :direction, in: %i(straight_forward backward clockwise counter_clockwise north north_west west south_west south south_east east north_east) enumerize :wayback, in: %i(outbound inbound), default: :outbound - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" - def self.nullable_attributes [:published_name, :comment, :number, :name, :direction, :wayback] end @@ -72,32 +77,41 @@ module Chouette validates_presence_of :name validates_presence_of :published_name validates_presence_of :line - - # validates_presence_of :direction - # validates_presence_of :wayback - validates :wayback, inclusion: { in: self.wayback.values } + after_commit :calculate_costs!, + on: [:create, :update], + if: ->() { + # Ensure the call back doesn't run during a referential merge + !referential.in_referential_suite? && + TomTom.enabled? + } + + def duplicate opposite=false + overrides = { + 'opposite_route_id' => nil, + 'name' => I18n.t('activerecord.copy', name: self.name) + } + keys_for_create = attributes.keys - %w{id objectid created_at updated_at} + atts_for_create = attributes + .slice(*keys_for_create) + .merge(overrides) + if opposite + atts_for_create[:wayback] = self.opposite_wayback + atts_for_create[:name] = I18n.t('routes.opposite', name: self.name) + atts_for_create[:published_name] = atts_for_create[:name] + atts_for_create[:opposite_route_id] = self.id + end + new_route = self.class.create!(atts_for_create) + duplicate_stop_points(for_route: new_route, opposite: opposite) + new_route + end - def duplicate - overrides = { - 'opposite_route_id' => nil, - 'name' => I18n.t('activerecord.copy', name: self.name) - } - keys_for_create = attributes.keys - %w{id objectid created_at updated_at} - atts_for_create = attributes - .slice(*keys_for_create) - .merge(overrides) - new_route = self.class.create!(atts_for_create) - duplicate_stop_points(for_route: new_route) - new_route - end - - def duplicate_stop_points(for_route:) - stop_points.each(&duplicate_stop_point(for_route: for_route)) + def duplicate_stop_points(for_route:, opposite: false) + stop_points.each(&duplicate_stop_point(for_route: for_route, opposite: opposite)) end - def duplicate_stop_point(for_route:) + def duplicate_stop_point(for_route:, opposite: false) -> stop_point do - stop_point.duplicate(for_route: for_route) + stop_point.duplicate(for_route: for_route, opposite: opposite) end end @@ -134,10 +148,12 @@ module Chouette values = self.slice(*['name', 'published_name', 'wayback']).values values.tap do |attrs| attrs << self.stop_points.sort_by(&:position).map{|sp| [sp.stop_area.user_objectid, sp.for_boarding, sp.for_alighting]} - attrs << self.routing_constraint_zones.map(&:checksum) + attrs << self.routing_constraint_zones.map(&:checksum).sort end end + has_checksum_children StopPoint + def geometry points = stop_areas.map(&:to_lat_lng).compact.map do |loc| [loc.lng, loc.lat] @@ -191,6 +207,10 @@ module Chouette journey_pattern end + def calculate_costs! + RouteWayCostWorker.perform_async(referential.id, id) + end + protected def self.vehicle_journeys_timeless(stop_point_id) diff --git a/app/models/chouette/routing_constraint_zone.rb b/app/models/chouette/routing_constraint_zone.rb index 58703598e..2cfb60bdd 100644 --- a/app/models/chouette/routing_constraint_zone.rb +++ b/app/models/chouette/routing_constraint_zone.rb @@ -1,6 +1,6 @@ module Chouette class RoutingConstraintZone < Chouette::TridentActiveRecord - has_paper_trail + # has_metadata include ChecksumSupport include ObjectidSupport @@ -30,6 +30,11 @@ module Chouette ] end + def update_route_checksum + route.update_checksum! + end + after_commit :update_route_checksum + def stop_points_belong_to_route return unless route diff --git a/app/models/chouette/stop_area.rb b/app/models/chouette/stop_area.rb index 7170dd217..b933e1944 100644 --- a/app/models/chouette/stop_area.rb +++ b/app/models/chouette/stop_area.rb @@ -2,11 +2,12 @@ require 'geokit' require 'geo_ruby' module Chouette class StopArea < Chouette::ActiveRecord - has_paper_trail class_name: 'PublicVersion' + has_metadata include ProjectionFields include StopAreaRestrictions include StopAreaReferentialSupport include ObjectidSupport + include CustomFieldsSupport extend Enumerize enumerize :area_type, in: Chouette::AreaType::ALL @@ -32,7 +33,7 @@ module Chouette after_update :journey_patterns_control_route_sections, if: Proc.new { |stop_area| ['boarding_position', 'quay'].include? stop_area.stop_area_type } - validates_format_of :registration_number, :with => %r{\A[\d\w_\-]+\Z}, :allow_blank => true + # validates_format_of :registration_number, :with => %r{\A[\d\w_:\-]+\Z}, :allow_blank => true validates_presence_of :name validates_presence_of :kind validates_presence_of :latitude, :if => :longitude @@ -46,6 +47,11 @@ module Chouette validates_numericality_of :waiting_time, greater_than_or_equal_to: 0, only_integer: true, if: :waiting_time validate :parent_area_type_must_be_greater validate :area_type_of_right_kind + validate :registration_number_is_set + + before_validation do + self.registration_number = self.stop_area_referential.generate_registration_number unless self.registration_number.present? + end def self.nullable_attributes [:registration_number, :street_name, :country_code, :fare_code, @@ -73,6 +79,22 @@ module Chouette end end + def registration_number_is_set + return unless self.stop_area_referential.registration_number_format.present? + if self.stop_area_referential.stop_areas.where(registration_number: self.registration_number).\ + where.not(id: self.id).exists? + errors.add(:registration_number, I18n.t('stop_areas.errors.registration_number.already_taken')) + end + + unless self.registration_number.present? + errors.add(:registration_number, I18n.t('stop_areas.errors.registration_number.cannot_be_empty')) + end + + unless self.stop_area_referential.validates_registration_number(self.registration_number) + errors.add(:registration_number, I18n.t('stop_areas.errors.registration_number.invalid', mask: self.stop_area_referential.registration_number_format)) + end + end + after_update :clean_invalid_access_links before_save :coordinates_to_lat_lng @@ -362,14 +384,15 @@ module Chouette end def activated? - deleted_at.nil? + !!(deleted_at.nil? && confirmed_at) end def deactivated? - !activated? + deleted_at.present? end def activate + self.confirmed_at = Time.now self.deleted_at = nil end @@ -378,6 +401,7 @@ module Chouette end def activate! + update_attribute :confirmed_at, Time.now update_attribute :deleted_at, nil end @@ -385,14 +409,45 @@ module Chouette update_attribute :deleted_at, Time.now end + def status + return :deleted if deleted_at + return :confirmed if confirmed_at + + :in_creation + end + + def status=(status) + case status&.to_sym + when :deleted + deactivate + when :confirmed + activate + when :in_creation + self.confirmed_at = self.deleted_at = nil + end + end + + def self.statuses + %i{in_creation confirmed deleted} + end + def time_zone_offset return 0 unless time_zone.present? ActiveSupport::TimeZone[time_zone]&.utc_offset end - def country_name + def full_time_zone_name + return unless time_zone.present? + return unless ActiveSupport::TimeZone[time_zone].present? + ActiveSupport::TimeZone[time_zone].tzinfo.name + end + + def country return unless country_code country = ISO3166::Country[country_code] + end + + def country_name return unless country country.translations[I18n.locale.to_s] || country.name end diff --git a/app/models/chouette/stop_point.rb b/app/models/chouette/stop_point.rb index 3b9eaa2f6..b0906d65f 100644 --- a/app/models/chouette/stop_point.rb +++ b/app/models/chouette/stop_point.rb @@ -1,6 +1,6 @@ module Chouette class StopPoint < Chouette::TridentActiveRecord - has_paper_trail + has_metadata def self.policy_class RoutePolicy end @@ -9,9 +9,6 @@ module Chouette include ForAlightingEnumerations include ObjectidSupport - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" - belongs_to :stop_area belongs_to :route, inverse_of: :stop_points has_many :vehicle_journey_at_stops, :dependent => :destroy @@ -19,7 +16,6 @@ module Chouette acts_as_list :scope => :route, top_of_list: 0 - validates_presence_of :stop_area validate :stop_area_id_validation def stop_area_id_validation @@ -30,7 +26,7 @@ module Chouette scope :default_order, -> { order("position") } - delegate :name, to: :stop_area + delegate :name, :registration_number, :kind, :area_type, to: :stop_area before_destroy :remove_dependent_journey_pattern_stop_points def remove_dependent_journey_pattern_stop_points @@ -41,11 +37,12 @@ module Chouette end end - def duplicate(for_route:) + def duplicate(for_route:, opposite: false) keys_for_create = attributes.keys - %w{id objectid created_at updated_at} atts_for_create = attributes .slice(*keys_for_create) .merge('route_id' => for_route.id) + atts_for_create["position"] = self.route.stop_points.size - atts_for_create["position"] if opposite self.class.create!(atts_for_create) end diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb index 20d6e69ac..29e3808e7 100644 --- a/app/models/chouette/time_table.rb +++ b/app/models/chouette/time_table.rb @@ -1,14 +1,12 @@ module Chouette class TimeTable < Chouette::TridentActiveRecord - has_paper_trail + has_metadata include ChecksumSupport include TimeTableRestrictions include ObjectidSupport include ApplicationDaysSupport include TimetableSupport - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" acts_as_taggable attr_accessor :tag_search @@ -18,7 +16,7 @@ module Chouette end ransacker :unaccented_comment, formatter: ->(val){ val.parameterize } do - Arel.sql('unaccent(comment)') + Arel.sql('unaccent(time_tables.comment)') end has_and_belongs_to_many :vehicle_journeys, :class_name => 'Chouette::VehicleJourney' @@ -83,6 +81,11 @@ module Chouette chunk.values.delete_if {|dates| dates.count < 2} end + def color + _color = read_attribute(:color) + _color.present? ? _color : nil + end + def convert_continuous_dates_to_periods chunks = self.continuous_dates diff --git a/app/models/chouette/time_table_date.rb b/app/models/chouette/time_table_date.rb index 98d8fa765..6a68d7fe1 100644 --- a/app/models/chouette/time_table_date.rb +++ b/app/models/chouette/time_table_date.rb @@ -2,7 +2,6 @@ module Chouette class TimeTableDate < Chouette::ActiveRecord include ChecksumSupport - self.primary_key = "id" belongs_to :time_table, inverse_of: :dates acts_as_list :scope => 'time_table_id = #{time_table_id}',:top_of_list => 0 diff --git a/app/models/chouette/time_table_period.rb b/app/models/chouette/time_table_period.rb index d9b707675..6965d828a 100644 --- a/app/models/chouette/time_table_period.rb +++ b/app/models/chouette/time_table_period.rb @@ -2,7 +2,6 @@ module Chouette class TimeTablePeriod < Chouette::ActiveRecord include ChecksumSupport - self.primary_key = "id" belongs_to :time_table, inverse_of: :periods acts_as_list :scope => 'time_table_id = #{time_table_id}',:top_of_list => 0 diff --git a/app/models/chouette/timeband.rb b/app/models/chouette/timeband.rb index 6155ffc77..38260b755 100644 --- a/app/models/chouette/timeband.rb +++ b/app/models/chouette/timeband.rb @@ -9,8 +9,7 @@ module Chouette class Timeband < Chouette::TridentActiveRecord include ObjectidSupport - has_paper_trail - self.primary_key = "id" + has_metadata validates :start_time, :end_time, presence: true validates_with Chouette::TimebandValidator diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb index 9b94f7f0e..814eeb388 100644 --- a/app/models/chouette/vehicle_journey.rb +++ b/app/models/chouette/vehicle_journey.rb @@ -1,13 +1,12 @@ # coding: utf-8 module Chouette class VehicleJourney < Chouette::TridentActiveRecord - has_paper_trail + has_metadata include ChecksumSupport + include CustomFieldsSupport include VehicleJourneyRestrictions include ObjectidSupport include StifTransportModeEnumerations - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" enum journey_category: { timed: 0, frequency: 1 } @@ -16,7 +15,7 @@ module Chouette attr_reader :time_table_tokens def self.nullable_attributes - [:transport_mode, :published_journey_name, :vehicle_type_identifier, :published_journey_identifier, :comment, :status_value] + [:transport_mode, :published_journey_name, :vehicle_type_identifier, :published_journey_identifier, :comment] end belongs_to :company @@ -63,7 +62,7 @@ module Chouette scope :with_ordered_stop_area_ids, ->(first, second){ if first.present? && second.present? joins(journey_pattern: :stop_points). - joins('INNER JOIN "journey_patterns" ON "journey_patterns"."id" = "vehicle_journeys"."journey_pattern_id" INNER JOIN "journey_patterns_stop_points" ON "journey_patterns_stop_points"."journey_pattern_id" = "journey_patterns"."id" INNER JOIN "stop_points" as "second_stop_points" ON "stop_points"."id" = "journey_patterns_stop_points"."stop_point_id"'). + joins('INNER JOIN "journey_patterns" ON "journey_patterns"."id" = "vehicle_journeys"."journey_pattern_id" INNER JOIN "journey_patterns_stop_points" ON "journey_patterns_stop_points"."journey_pattern_id" = "journey_patterns"."id" INNER JOIN "stop_points" as "second_stop_points" ON "second_stop_points"."id" = "journey_patterns_stop_points"."stop_point_id"'). where('stop_points.stop_area_id = ?', first). where('second_stop_points.stop_area_id = ? and stop_points.position < second_stop_points.position', second) else @@ -89,7 +88,6 @@ module Chouette end } - scope :in_purchase_window, ->(range){ purchase_windows = Chouette::PurchaseWindow.overlap_dates(range) sql = purchase_windows.joins(:vehicle_journeys).select('vehicle_journeys.id').uniq.to_sql @@ -142,9 +140,12 @@ module Chouette attrs << self.published_journey_identifier attrs << self.try(:company).try(:get_objectid).try(:local_id) attrs << self.footnotes.map(&:checksum).sort + vjas = self.vehicle_journey_at_stops vjas += VehicleJourneyAtStop.where(vehicle_journey_id: self.id) unless self.new_record? attrs << vjas.uniq.sort_by { |s| s.stop_point&.position }.map(&:checksum).sort + + attrs << self.purchase_windows.map(&:checksum).sort if purchase_windows.present? end end @@ -156,6 +157,14 @@ module Chouette end end + def sales_start + purchase_windows.map{|p| p.date_ranges.map &:first}.flatten.min + end + + def sales_end + purchase_windows.map{|p| p.date_ranges.map &:last}.flatten.max + end + def calculate_vehicle_journey_at_stop_day_offset Chouette::VehicleJourneyAtStopsDayOffset.new( vehicle_journey_at_stops @@ -238,11 +247,13 @@ module Chouette end def self.state_update route, state + objects = [] transaction do state.each do |item| item.delete('errors') vj = find_by(objectid: item['objectid']) || state_create_instance(route, item) next if item['deletable'] && vj.persisted? && vj.destroy + objects << vj if vj.state_update_vjas?(item['vehicle_journey_at_stops']) vj.update_vjas_from_state(item['vehicle_journey_at_stops']) @@ -270,6 +281,7 @@ module Chouette item['vehicle_journey_at_stops'].map {|vjas| vjas.delete('new_record') } end state.delete_if {|item| item['deletable']} + objects end def self.state_create_instance route, item @@ -340,19 +352,31 @@ module Chouette end end - def self.custom_fields - CustomField.where(resource_type: self.name.split("::").last) - end - - - def custom_fields - Hash[*self.class.custom_fields.map do |v| - [v.code, v.slice(:code, :name, :field_type, :options).update(value: custom_field_value(v.code))] - end.flatten] - end - - def custom_field_value key - (custom_field_values || {})[key.to_s] + def fill_passing_time_at_borders + encountered_borders = [] + previous_stop = nil + vehicle_journey_at_stops.each do |vjas| + sp = vjas.stop_point + if sp.stop_area.area_type == "border" + encountered_borders << vjas + else + if encountered_borders.any? + before_cost = journey_pattern.costs_between previous_stop.stop_point, encountered_borders.first.stop_point + after_cost = journey_pattern.costs_between encountered_borders.last.stop_point, sp + if before_cost && before_cost[:distance] && after_cost && after_cost[:distance] + before_distance = before_cost[:distance].to_f + after_distance = after_cost[:distance].to_f + time = previous_stop.departure_time + before_distance / (before_distance+after_distance) * (vjas.arrival_time - previous_stop.departure_time) + encountered_borders.each do |b| + b.update_attribute :arrival_time, time + b.update_attribute :departure_time, time + end + end + encountered_borders = [] + end + previous_stop = vjas + end + end end def self.matrix(vehicle_journeys) @@ -417,10 +441,5 @@ module Chouette ') .where('"time_tables_vehicle_journeys"."vehicle_journey_id" IS NULL') end - - def self.lines - lines_query = joins(:route).select("routes.line_id").reorder(nil).except(:group).pluck(:'routes.line_id') - Chouette::Line.where(id: lines_query) - end end end diff --git a/app/models/chouette/vehicle_journey_at_stop.rb b/app/models/chouette/vehicle_journey_at_stop.rb index 3b4f35f13..3f5bd5abf 100644 --- a/app/models/chouette/vehicle_journey_at_stop.rb +++ b/app/models/chouette/vehicle_journey_at_stop.rb @@ -4,10 +4,10 @@ module Chouette include Chouette::ForAlightingEnumerations include ChecksumSupport - DAY_OFFSET_MAX = 1 + DAY_OFFSET_MAX = 2 - # FIXME http://jira.codehaus.org/browse/JRUBY-6358 - self.primary_key = "id" + @@day_offset_max = DAY_OFFSET_MAX + mattr_accessor :day_offset_max belongs_to :stop_point belongs_to :vehicle_journey @@ -42,7 +42,7 @@ module Chouette I18n.t( 'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max', short_id: vehicle_journey&.get_objectid&.short_id, - max: DAY_OFFSET_MAX + 1 + max: Chouette::VehicleJourneyAtStop.day_offset_max + 1 ) ) end @@ -53,7 +53,7 @@ module Chouette I18n.t( 'vehicle_journey_at_stops.errors.day_offset_must_not_exceed_max', short_id: vehicle_journey&.get_objectid&.short_id, - max: DAY_OFFSET_MAX + 1 + max: Chouette::VehicleJourneyAtStop.day_offset_max + 1 ) ) end @@ -64,7 +64,7 @@ module Chouette # nil offsets. Handle these gracefully by forcing them to a 0 offset. offset ||= 0 - offset < 0 || offset > DAY_OFFSET_MAX + offset < 0 || offset > Chouette::VehicleJourneyAtStop.day_offset_max end def checksum_attributes @@ -84,12 +84,12 @@ module Chouette format_time arrival_time.utc end - def departure_local_time - local_time departure_time + def departure_local_time offset=nil + local_time departure_time, offset end - def arrival_local_time - local_time arrival_time + def arrival_local_time offset=nil + local_time arrival_time, offset end def departure_local @@ -100,12 +100,15 @@ module Chouette format_time arrival_local_time end + def time_zone_offset + return 0 unless stop_point&.stop_area&.time_zone.present? + ActiveSupport::TimeZone[stop_point.stop_area.time_zone]&.utc_offset || 0 + end + private - def local_time time - return unless time - return time unless stop_point&.stop_area&.time_zone.present? - return time unless ActiveSupport::TimeZone[stop_point.stop_area.time_zone].present? - time + ActiveSupport::TimeZone[stop_point.stop_area.time_zone].utc_offset + def local_time time, offset=nil + return nil unless time + time + (offset || time_zone_offset) end def format_time time diff --git a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb index 7497cd72c..cfa0e8bfc 100644 --- a/app/models/chouette/vehicle_journey_at_stops_day_offset.rb +++ b/app/models/chouette/vehicle_journey_at_stops_day_offset.rb @@ -4,31 +4,32 @@ module Chouette @at_stops = at_stops end - def calculate! - arrival_offset = 0 - departure_offset = 0 + def time_from_fake_date fake_date + fake_date - fake_date.to_date.to_time + end + def calculate! + offset = 0 + tz_offset = @at_stops.first&.time_zone_offset @at_stops.inject(nil) do |prior_stop, stop| next stop if prior_stop.nil? # we only compare time of the day, not actual times - stop_arrival_time = stop.arrival_time - stop.arrival_time.to_date.to_time - stop_departure_time = stop.departure_time - stop.departure_time.to_date.to_time - prior_stop_arrival_time = prior_stop.arrival_time - prior_stop.arrival_time.to_date.to_time - prior_stop_departure_time = prior_stop.departure_time - prior_stop.departure_time.to_date.to_time - - if stop_arrival_time < prior_stop_departure_time || - stop_arrival_time < prior_stop_arrival_time - arrival_offset += 1 + stop_arrival_time = time_from_fake_date stop.arrival_local_time(tz_offset) + stop_departure_time = time_from_fake_date stop.departure_local_time(tz_offset) + prior_stop_departure_time = time_from_fake_date prior_stop.departure_local_time(tz_offset) + + if stop_arrival_time < prior_stop_departure_time + offset += 1 end - if stop_departure_time < stop_arrival_time || - stop_departure_time < prior_stop_departure_time - departure_offset += 1 + stop.arrival_day_offset = offset + + if stop_departure_time < stop_arrival_time + offset += 1 end - stop.arrival_day_offset = arrival_offset - stop.departure_day_offset = departure_offset + stop.departure_day_offset = offset stop end diff --git a/app/models/clean_up.rb b/app/models/clean_up.rb index 7aab7f32e..ec47489e9 100644 --- a/app/models/clean_up.rb +++ b/app/models/clean_up.rb @@ -1,4 +1,4 @@ -class CleanUp < ActiveRecord::Base +class CleanUp < ApplicationModel extend Enumerize include AASM belongs_to :referential diff --git a/app/models/clean_up_result.rb b/app/models/clean_up_result.rb index 24d262deb..dff4f5acd 100644 --- a/app/models/clean_up_result.rb +++ b/app/models/clean_up_result.rb @@ -1,3 +1,3 @@ -class CleanUpResult < ActiveRecord::Base +class CleanUpResult < ApplicationModel belongs_to :clean_up end diff --git a/app/models/compliance_check.rb b/app/models/compliance_check.rb index 55f2ae228..4ef6170e9 100644 --- a/app/models/compliance_check.rb +++ b/app/models/compliance_check.rb @@ -1,14 +1,23 @@ -class ComplianceCheck < ActiveRecord::Base +class ComplianceCheck < ApplicationModel + include ComplianceItemSupport self.inheritance_column = nil extend Enumerize belongs_to :compliance_check_set belongs_to :compliance_check_block - + enumerize :criticity, in: %i(warning error), scope: true, default: :warning validates :criticity, presence: true validates :name, presence: true validates :code, presence: true validates :origin_code, presence: true + + def control_class + compliance_control_name.present? ? compliance_control_name.constantize : nil + end + + delegate :predicate, to: :control_class, allow_nil: true + delegate :prerequisite, to: :control_class, allow_nil: true + end diff --git a/app/models/compliance_check_block.rb b/app/models/compliance_check_block.rb index 059547e1b..e4f4c1c37 100644 --- a/app/models/compliance_check_block.rb +++ b/app/models/compliance_check_block.rb @@ -1,4 +1,4 @@ -class ComplianceCheckBlock < ActiveRecord::Base +class ComplianceCheckBlock < ApplicationModel include StifTransportModeEnumerations include StifTransportSubmodeEnumerations diff --git a/app/models/compliance_check_message.rb b/app/models/compliance_check_message.rb index 738bd4a4b..a4b1062f6 100644 --- a/app/models/compliance_check_message.rb +++ b/app/models/compliance_check_message.rb @@ -1,4 +1,4 @@ -class ComplianceCheckMessage < ActiveRecord::Base +class ComplianceCheckMessage < ApplicationModel extend Enumerize belongs_to :compliance_check_set diff --git a/app/models/compliance_check_message_export.rb b/app/models/compliance_check_message_export.rb index 04e1a9caa..9b7f90fac 100644 --- a/app/models/compliance_check_message_export.rb +++ b/app/models/compliance_check_message_export.rb @@ -22,16 +22,18 @@ class ComplianceCheckMessageExport end def column_names - ["criticity", "message key", "resource objectid", "link", "message"] + ["criticity", "message_key", "resource_objectid", "link", "message"].map {|c| ComplianceCheckMessage.tmf(c)} end def to_csv(options = {}) - CSV.generate(options.slice(:col_sep, :quote_char, :force_quotes)) do |csv| + csv_string = CSV.generate(options.slice(:col_sep, :quote_char, :force_quotes)) do |csv| csv << column_names compliance_check_messages.each do |compliance_check_message| csv << [compliance_check_message.compliance_check.criticity, *compliance_check_message.message_attributes.values_at('test_id', 'source_objectid'), options[:server_url] + compliance_check_message.message_attributes['source_object_path'], I18n.t("compliance_check_messages.#{compliance_check_message.message_key}", compliance_check_message.message_attributes.deep_symbolize_keys)] end end + # We add a BOM to indicate we use UTF-8 + "\uFEFF" + csv_string end def to_zip(temp_file,options = {}) diff --git a/app/models/compliance_check_resource.rb b/app/models/compliance_check_resource.rb index 2989bf3cf..d2f782e2b 100644 --- a/app/models/compliance_check_resource.rb +++ b/app/models/compliance_check_resource.rb @@ -1,9 +1,9 @@ -class ComplianceCheckResource < ActiveRecord::Base +class ComplianceCheckResource < ApplicationModel extend Enumerize belongs_to :compliance_check_set - enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true + enumerize :status, in: %i(OK ERROR WARNING IGNORED) validates_presence_of :compliance_check_set end diff --git a/app/models/compliance_check_set.rb b/app/models/compliance_check_set.rb index 289fc134f..8b1dbdd68 100644 --- a/app/models/compliance_check_set.rb +++ b/app/models/compliance_check_set.rb @@ -1,6 +1,7 @@ -class ComplianceCheckSet < ActiveRecord::Base +class ComplianceCheckSet < ApplicationModel extend Enumerize - has_paper_trail class_name: 'PublicVersion' + + has_metadata belongs_to :referential belongs_to :compliance_control_set @@ -49,39 +50,25 @@ class ComplianceCheckSet < ActiveRecord::Base end def update_status - statuses = compliance_check_resources.map do |resource| - case resource.status - when 'ERROR' - return update(status: 'failed') - when 'WARNING' - return update(status: 'warning') - else - resource.status + status = + if compliance_check_resources.where(status: 'ERROR').count > 0 + 'failed' + elsif compliance_check_resources.where(status: ["WARNING", "IGNORED"]).count > 0 + 'warning' + elsif compliance_check_resources.where(status: "OK").count == compliance_check_resources.count + 'successful' end - end - if statuses_ok_or_ignored?(statuses) - return update(status: 'successful') + attributes = { + status: status + } + + if self.class.finished_statuses.include?(status) + attributes[:ended_at] = Time.now end - true + update attributes end - private - - def statuses_ok_or_ignored?(statuses) - uniform_statuses = statuses.uniq - - ( - # All statuses OK - uniform_statuses.length == 1 && - uniform_statuses.first == 'OK' - ) || - ( - # Statuses OK or IGNORED - uniform_statuses.length == 2 && - uniform_statuses.include?('OK') && - uniform_statuses.include?('IGNORED') - ) - end + end diff --git a/app/models/compliance_control.rb b/app/models/compliance_control.rb index 298a63ab9..672fb128c 100644 --- a/app/models/compliance_control.rb +++ b/app/models/compliance_control.rb @@ -1,18 +1,16 @@ -class ComplianceControl < ActiveRecord::Base +class ComplianceControl < ApplicationModel + include ComplianceItemSupport class << self def criticities; %i(warning error) end def default_code; "" end - def dynamic_attributes - stored_attributes[:control_attributes] || [] - end def policy_class ComplianceControlPolicy end def subclass_patterns - { + { generic: 'Generic', journey_pattern: 'JourneyPattern', line: 'Line', @@ -30,6 +28,9 @@ class ComplianceControl < ActiveRecord::Base end super end + + def predicate; I18n.t("compliance_controls.#{self.name.underscore}.description") end + def prerequisite; I18n.t("compliance_controls.#{self.name.underscore}.prerequisite") end end extend Enumerize @@ -45,26 +46,25 @@ class ComplianceControl < ActiveRecord::Base validates :compliance_control_set, presence: true validate def coherent_control_set - return true if compliance_control_block_id.nil? - ids = [compliance_control_block.compliance_control_set_id, compliance_control_set_id] - return true if ids.first == ids.last - names = ids.map{|id| ComplianceControlSet.find(id).name} - errors.add(:coherent_control_set, - I18n.t('compliance_controls.errors.incoherent_control_sets', - indirect_set_name: names.first, - direct_set_name: names.last)) -end - + return true if compliance_control_block_id.nil? + ids = [compliance_control_block.compliance_control_set_id, compliance_control_set_id] + return true if ids.first == ids.last + names = ids.map{|id| ComplianceControlSet.find(id).name} + errors.add(:coherent_control_set, + I18n.t('compliance_controls.errors.incoherent_control_sets', + indirect_set_name: names.first, + direct_set_name: names.last)) + end -def initialize(attributes = {}) - super - self.name ||= I18n.t("activerecord.models.#{self.class.name.underscore}.one") - self.code ||= self.class.default_code - self.origin_code ||= self.class.default_code -end + def initialize(attributes = {}) + super + self.name ||= I18n.t("activerecord.models.#{self.class.name.underscore}.one") + self.code ||= self.class.default_code + self.origin_code ||= self.class.default_code + end -def predicate; I18n.t("compliance_controls.#{self.class.name.underscore}.description") end -def prerequisite; I18n.t('compliance_controls.metas.no_prerequisite'); end + def predicate; self.class.predicate end + def prerequisite; self.class.prerequisite end end @@ -76,6 +76,7 @@ require_dependency 'generic_attribute_control/uniqueness' require_dependency 'journey_pattern_control/duplicates' require_dependency 'journey_pattern_control/vehicle_journey' require_dependency 'line_control/route' +require_dependency 'line_control/lines_scope' require_dependency 'route_control/duplicates' require_dependency 'route_control/journey_pattern' require_dependency 'route_control/minimum_length' diff --git a/app/models/compliance_control_block.rb b/app/models/compliance_control_block.rb index d7d84fd06..6a3c8a34e 100644 --- a/app/models/compliance_control_block.rb +++ b/app/models/compliance_control_block.rb @@ -1,4 +1,4 @@ -class ComplianceControlBlock < ActiveRecord::Base +class ComplianceControlBlock < ApplicationModel include StifTransportModeEnumerations include StifTransportSubmodeEnumerations @@ -12,6 +12,8 @@ class ComplianceControlBlock < ActiveRecord::Base validates :transport_mode, presence: true validates :compliance_control_set, presence: true + validates_uniqueness_of :condition_attributes, scope: :compliance_control_set_id + def name ApplicationController.helpers.transport_mode_text(self) end diff --git a/app/models/compliance_control_set.rb b/app/models/compliance_control_set.rb index c0ea692f2..4f0f86d08 100644 --- a/app/models/compliance_control_set.rb +++ b/app/models/compliance_control_set.rb @@ -1,5 +1,6 @@ -class ComplianceControlSet < ActiveRecord::Base - has_paper_trail class_name: 'PublicVersion' +class ComplianceControlSet < ApplicationModel + has_metadata + belongs_to :organisation has_many :compliance_control_blocks, dependent: :destroy has_many :compliance_controls, dependent: :destroy diff --git a/app/models/concerns/application_days_support.rb b/app/models/concerns/application_days_support.rb index 348436aa4..6086d9580 100644 --- a/app/models/concerns/application_days_support.rb +++ b/app/models/concerns/application_days_support.rb @@ -10,48 +10,47 @@ module ApplicationDaysSupport SUNDAY = 256 EVERYDAY = MONDAY | TUESDAY | WEDNESDAY | THURSDAY | FRIDAY | SATURDAY | SUNDAY + ALL_DAYS = %w(monday tuesday wednesday thursday friday saturday sunday).freeze + def display_day_types - %w(monday tuesday wednesday thursday friday saturday sunday).select{ |d| self.send(d) }.map{ |d| self.human_attribute_name(d).first(2)}.join(', ') + ALL_DAYS.select{ |d| self.send(d) }.map{ |d| self.human_attribute_name(d).first(2)}.join(', ') end def day_by_mask(flag) - int_day_types & flag == flag + self.class.day_by_mask int_day_types, flag end - def self.day_by_mask(int_day_types,flag) - int_day_types & flag == flag + def valid_day? wday + valid_days.include?(wday) end - def valid_days - # Build an array with day of calendar week (1-7, Monday is 1). - [].tap do |valid_days| - valid_days << 1 if monday - valid_days << 2 if tuesday - valid_days << 3 if wednesday - valid_days << 4 if thursday - valid_days << 5 if friday - valid_days << 6 if saturday - valid_days << 7 if sunday + included do + def self.valid_days(int_day_types) + # Build an array with day of calendar week (1-7, Monday is 1). + [].tap do |valid_days| + valid_days << 1 if day_by_mask(int_day_types,MONDAY) + valid_days << 2 if day_by_mask(int_day_types,TUESDAY) + valid_days << 3 if day_by_mask(int_day_types,WEDNESDAY) + valid_days << 4 if day_by_mask(int_day_types,THURSDAY) + valid_days << 5 if day_by_mask(int_day_types,FRIDAY) + valid_days << 6 if day_by_mask(int_day_types,SATURDAY) + valid_days << 7 if day_by_mask(int_day_types,SUNDAY) + end end - end - def valid_day? wday - valid_days.include?(wday) - end + def self.day_by_mask(int_day_types,flag) + int_day_types & flag == flag + end - def self.valid_days(int_day_types) - # Build an array with day of calendar week (1-7, Monday is 1). - [].tap do |valid_days| - valid_days << 1 if day_by_mask(int_day_types,MONDAY) - valid_days << 2 if day_by_mask(int_day_types,TUESDAY) - valid_days << 3 if day_by_mask(int_day_types,WEDNESDAY) - valid_days << 4 if day_by_mask(int_day_types,THURSDAY) - valid_days << 5 if day_by_mask(int_day_types,FRIDAY) - valid_days << 6 if day_by_mask(int_day_types,SATURDAY) - valid_days << 7 if day_by_mask(int_day_types,SUNDAY) + def self.all_days + ALL_DAYS end end + def valid_days + self.class.valid_days int_day_types + end + def monday day_by_mask(MONDAY) end diff --git a/app/models/concerns/checksum_support.rb b/app/models/concerns/checksum_support.rb index 92103798e..de3a6e16b 100644 --- a/app/models/concerns/checksum_support.rb +++ b/app/models/concerns/checksum_support.rb @@ -40,6 +40,7 @@ module ChecksumSupport def current_checksum_source source = checksum_replace_nil_or_empty_values(self.checksum_attributes) + source += self.custom_fields_checksum if self.respond_to?(:custom_fields_checksum) source.map{ |item| if item.kind_of?(Array) item.map{ |x| x.kind_of?(Array) ? "(#{x.join(',')})" : x }.join(',') diff --git a/app/models/concerns/compliance_item_support.rb b/app/models/concerns/compliance_item_support.rb new file mode 100644 index 000000000..f44f5719f --- /dev/null +++ b/app/models/concerns/compliance_item_support.rb @@ -0,0 +1,13 @@ +module ComplianceItemSupport + extend ActiveSupport::Concern + included do + + end + + module ClassMethods + def dynamic_attributes + stored_attributes[:control_attributes] || [] + end + end + +end diff --git a/app/models/concerns/custom_fields_support.rb b/app/models/concerns/custom_fields_support.rb new file mode 100644 index 000000000..c39dfd1fc --- /dev/null +++ b/app/models/concerns/custom_fields_support.rb @@ -0,0 +1,62 @@ +module CustomFieldsSupport + extend ActiveSupport::Concern + + included do + validate :custom_fields_values_are_valid + after_initialize :initialize_custom_fields + + def self.custom_fields workgroup=:all + fields = CustomField.where(resource_type: self.name.split("::").last) + fields = fields.where(workgroup_id: workgroup&.id) if workgroup != :all + fields + end + + def self.custom_fields_definitions workgroup=:all + Hash[*custom_fields(workgroup).map{|cf| [cf.code, cf]}.flatten] + end + + def method_missing method_name, *args + if method_name =~ /custom_field_*/ && method_name.to_sym != :custom_field_values && !@custom_fields_initialized + initialize_custom_fields + send method_name, *args + else + super method_name, *args + end + end + + def custom_fields workgroup=:all + CustomField::Collection.new self, workgroup + end + + def custom_fields_checksum + custom_fields.values.map(&:checksum) + end + + def custom_field_values= vals + out = {} + custom_fields.each do |code, field| + out[code] = field.preprocess_value_for_assignment(vals.symbolize_keys[code.to_sym]) + end + write_attribute :custom_field_values, out + end + + def initialize_custom_fields + return unless self.attributes.has_key?("custom_field_values") + self.custom_field_values ||= {} + custom_fields(:all).values.each &:initialize_custom_field + custom_fields(:all).each do |k, v| + custom_field_values[k] ||= v.default_value + end + @custom_fields_initialized = true + end + + def custom_field_value key + (custom_field_values&.stringify_keys || {})[key.to_s] + end + + private + def custom_fields_values_are_valid + custom_fields(:all).values.all?{|cf| cf.valid?} + end + end +end diff --git a/app/models/concerns/iev_interfaces/message.rb b/app/models/concerns/iev_interfaces/message.rb new file mode 100644 index 000000000..ad41e98b7 --- /dev/null +++ b/app/models/concerns/iev_interfaces/message.rb @@ -0,0 +1,9 @@ +module IevInterfaces::Message + extend ActiveSupport::Concern + + included do + extend Enumerize + enumerize :criticity, in: %i(info warning error) + validates :criticity, presence: true + end +end diff --git a/app/models/concerns/iev_interfaces/resource.rb b/app/models/concerns/iev_interfaces/resource.rb new file mode 100644 index 000000000..7f8c3eefd --- /dev/null +++ b/app/models/concerns/iev_interfaces/resource.rb @@ -0,0 +1,9 @@ +module IevInterfaces::Resource + extend ActiveSupport::Concern + + included do + extend Enumerize + enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true + validates_presence_of :name, :resource_type, :reference + end +end diff --git a/app/models/concerns/iev_interfaces/task.rb b/app/models/concerns/iev_interfaces/task.rb new file mode 100644 index 000000000..f052b3a8f --- /dev/null +++ b/app/models/concerns/iev_interfaces/task.rb @@ -0,0 +1,121 @@ +module IevInterfaces::Task + extend ActiveSupport::Concern + + included do + belongs_to :parent, polymorphic: true + belongs_to :workbench, class_name: "::Workbench" + belongs_to :referential + + mount_uploader :file, ImportUploader + + has_many :children, foreign_key: :parent_id, class_name: self.name, dependent: :destroy + + extend Enumerize + enumerize :status, in: %w(new pending successful warning failed running aborted canceled), scope: true, default: :new + + validates :name, presence: true + validates_presence_of :workbench, :creator + + has_many :messages, class_name: messages_class_name, dependent: :destroy, foreign_key: "#{messages_class_name.split('::').first.downcase}_id" + has_many :resources, class_name: resources_class_name, dependent: :destroy, foreign_key: "#{resources_class_name.split('::').first.downcase}_id" + + scope :where_started_at_in, ->(period_range) do + where('started_at BETWEEN :begin AND :end', begin: period_range.begin, end: period_range.end) + end + + scope :blocked, -> { where('created_at < ? AND status = ?', 4.hours.ago, 'running') } + + before_save :initialize_fields, on: :create + after_save :notify_parent + end + + module ClassMethods + def launched_statuses + %w(new pending) + end + + def failed_statuses + %w(failed aborted canceled) + end + + def finished_statuses + %w(successful failed warning aborted canceled) + end + + def abort_old + where( + 'created_at < ? AND status NOT IN (?)', + 4.hours.ago, + finished_statuses + ).update_all(status: 'aborted') + end + end + + def notify_parent + return unless self.class.finished_statuses.include?(status) + + return unless parent.present? + return if notified_parent_at + parent.child_change + + update_column :notified_parent_at, Time.now + end + + def children_succeedeed + children.with_status(:successful, :warning).count + end + + def update_status + Rails.logger.info "update_status for #{inspect}" + status = + if children.where(status: self.class.failed_statuses).count > 0 + 'failed' + elsif children.where(status: "warning").count > 0 + 'warning' + elsif children.where(status: "successful").count == children.count + 'successful' + else + 'running' + end + + attributes = { + current_step: children.count, + status: status + } + + if self.class.finished_statuses.include?(status) + attributes[:ended_at] = Time.now + end + + update attributes + end + + def child_change + return if self.class.finished_statuses.include?(status) + update_status + end + + def call_iev_callback + return if self.class.finished_statuses.include?(status) + threaded_call_boiv_iev + end + + private + + def threaded_call_boiv_iev + Thread.new(&method(:call_boiv_iev)) + end + + def call_boiv_iev + Rails.logger.error("Begin IEV call for import") + Net::HTTP.get iev_callback_url + Rails.logger.error("End IEV call for import") + rescue Exception => e + logger.error "IEV server error : #{e.message}" + logger.error e.backtrace.inspect + end + + private + def initialize_fields + end +end diff --git a/app/models/concerns/metadata_support.rb b/app/models/concerns/metadata_support.rb new file mode 100644 index 000000000..c4bedbcda --- /dev/null +++ b/app/models/concerns/metadata_support.rb @@ -0,0 +1,107 @@ +module MetadataSupport + extend ActiveSupport::Concern + + included do + class << self + def has_metadata? + !!@has_metadata + end + + def has_metadata opts={} + @has_metadata = true + + define_method :metadata do + attr_name = opts[:attr_name] || :metadata + @wrapped_metadata ||= begin + wrapped = MetadataSupport::MetadataWrapper.new self.read_attribute(attr_name) + wrapped.attribute_name = attr_name + wrapped.owner = self + wrapped + end + end + + define_method :metadata= do |val| + @wrapped_metadata = nil + super val + end + + define_method :set_metadata! do |name, value| + self.metadata.send "#{name}=", value + self.save! + end + end + end + end + + def has_metadata? + self.class.has_metadata? + end + + def merge_metadata_from source + return unless source.has_metadata? + source_metadata = source.metadata + res = {} + self.metadata.each do |k, v| + unless self.metadata.is_timestamp_attr?(k) + ts = self.metadata.timestamp_attr(k) + if source_metadata[ts] && source_metadata[ts] > self.metadata[ts] + res[k] = source_metadata[k] + else + res[k] = v + end + end + end + self.metadata = res + self + end + + class MetadataWrapper < OpenStruct + attr_accessor :attribute_name, :owner + + def is_timestamp_attr? name + name =~ /_updated_at$/ + end + + def timestamp_attr name + "#{name}_updated_at".to_sym + end + + def method_missing(mid, *args) + out = super(mid, *args) + owner.send :write_attribute, attribute_name, @table + out = out&.to_time if args.length == 0 && is_timestamp_attr?(mid) + out + end + + def each + @table.each do |k,v| + yield k, v + end + end + + def new_ostruct_member name + unless is_timestamp_attr?(name) + timestamp_attr_name = timestamp_attr(name) + end + + name = name.to_sym + unless respond_to?(name) + if timestamp_attr_name + define_singleton_method(timestamp_attr_name) { @table[timestamp_attr_name]&.to_time } + define_singleton_method(name) { @table[name] } + else + # we are defining an accessor for a timestamp + define_singleton_method(name) { @table[name]&.to_time } + end + + define_singleton_method("#{name}=") do |x| + modifiable[timestamp_attr_name] = Time.now if timestamp_attr_name + modifiable[name] = x + owner.send :write_attribute, attribute_name, @table + end + modifiable[timestamp_attr_name] = Time.now if timestamp_attr_name + end + name + end + end +end diff --git a/app/models/concerns/min_max_values_validation.rb b/app/models/concerns/min_max_values_validation.rb index eff779d81..a79f5ec85 100644 --- a/app/models/concerns/min_max_values_validation.rb +++ b/app/models/concerns/min_max_values_validation.rb @@ -3,11 +3,13 @@ module MinMaxValuesValidation included do validates_presence_of :minimum, :maximum + validates_numericality_of :minimum, :maximum, allow_nil: true, greater_than_or_equal_to: 0 + validates_format_of :minimum, :maximum, with: %r{\A\d+(\.\d+)?\Z} validate :min_max_values_validation end def min_max_values_validation - return true if (minimum && maximum) && (minimum.to_i < maximum.to_i) + return true if (minimum && maximum) && (minimum.to_f < maximum.to_f) errors.add(:minimum, I18n.t('compliance_controls.min_max_values', min: minimum, max: maximum)) end end diff --git a/app/models/custom_field.rb b/app/models/custom_field.rb index 774c8b0f6..88783b5b4 100644 --- a/app/models/custom_field.rb +++ b/app/models/custom_field.rb @@ -1,9 +1,298 @@ -class CustomField < ActiveRecord::Base +class CustomField < ApplicationModel extend Enumerize belongs_to :workgroup - enumerize :field_type, in: %i{list} + enumerize :field_type, in: %i{list integer string attachment} validates :name, uniqueness: {scope: [:resource_type, :workgroup_id]} - validates :code, uniqueness: {scope: [:resource_type, :workgroup_id], case_sensitive: false} + validates :code, uniqueness: {scope: [:resource_type, :workgroup_id], case_sensitive: false}, presence: true + + class Collection < HashWithIndifferentAccess + def initialize object, workgroup=:all + vals = object.class.custom_fields(workgroup).map do |v| + [v.code, CustomField::Instance.new(object, v, object.custom_field_value(v.code))] + end + super Hash[*vals.flatten] + end + + def to_hash + HashWithIndifferentAccess[*self.map{|k, v| [k, v.to_hash]}.flatten(1)] + end + end + + class Instance + def self.new owner, custom_field, value + field_type = custom_field.field_type + klass_name = field_type && "CustomField::Instance::#{field_type.classify}" + klass = klass_name.safe_constantize || CustomField::Instance::Base + klass.new owner, custom_field, value + end + + class Base + def initialize owner, custom_field, value + @custom_field = custom_field + @raw_value = value + @owner = owner + @errors = [] + @validated = false + @valid = false + end + + attr_accessor :owner, :custom_field + + delegate :code, :name, :field_type, to: :@custom_field + + def default_value + options["default"] + end + + def options + @custom_field.options || {} + end + + def validate + @valid = true + end + + def valid? + validate unless @validated + @valid + end + + def value + @raw_value + end + + def checksum + @raw_value + end + + def input form_helper + @input ||= begin + klass_name = field_type && "CustomField::Instance::#{field_type.classify}::Input" + klass = klass_name.safe_constantize || CustomField::Instance::Base::Input + klass.new self, form_helper + end + end + + def errors_key + "custom_fields.#{code}" + end + + def to_hash + HashWithIndifferentAccess[*%w(code name field_type options value).map{|k| [k, send(k)]}.flatten(1)] + end + + def display_value + value + end + + def initialize_custom_field + end + + def preprocess_value_for_assignment val + val + end + + def render_partial + ActionView::Base.new(Rails.configuration.paths["app/views"].first).render( + :partial => "shared/custom_fields/#{field_type}", + :locals => { field: self} + ) + end + + class Input + def initialize instance, form_helper + @instance = instance + @form_helper = form_helper + end + + def custom_field + @instance.custom_field + end + + delegate :custom_field, :value, :options, to: :@instance + delegate :code, :name, :field_type, to: :custom_field + + def to_s + out = form_input + out.html_safe + end + + protected + + def form_input_id + "custom_field_#{code}" + end + + def form_input_name + "#{@form_helper.object_name}[custom_field_values][#{code}]" + end + + def form_input_options + { + input_html: {value: value, name: form_input_name}, + label: name + } + end + + def form_input + @form_helper.input form_input_id, form_input_options + end + end + end + + class Integer < Base + def value + @raw_value&.to_i + end + + def validate + @valid = true + return if @raw_value.is_a?(Fixnum) || @raw_value.is_a?(Float) + unless @raw_value.to_s =~ /\A\d*\Z/ + @owner.errors.add errors_key, "'#{@raw_value}' is not a valid integer" + @valid = false + end + end + end + + class List < Integer + def validate + super + return unless value.present? + unless value >= 0 && value < options["list_values"].size + @owner.errors.add errors_key, "'#{@raw_value}' is not a valid value" + @valid = false + end + end + + def display_value + return unless value + k = options["list_values"].is_a?(Hash) ? value.to_s : value.to_i + options["list_values"][k] + end + + class Input < Base::Input + def form_input_options + collection = options["list_values"] + collection = collection.each_with_index.to_a if collection.is_a?(Array) + collection = collection.map(&:reverse) if collection.is_a?(Hash) + super.update({ + selected: value, + collection: collection + }) + end + end + end + + class Attachment < Base + def initialize_custom_field + custom_field_code = self.code + _attr_name = attr_name + _uploader_name = uploader_name + _digest_name = digest_name + owner.send :define_singleton_method, "read_uploader" do |attr| + if attr.to_s == _attr_name + custom_field_values[custom_field_code] && custom_field_values[custom_field_code]["path"] + else + read_attribute attr + end + end + + owner.send :define_singleton_method, "write_uploader" do |attr, val| + if attr.to_s == _attr_name + self.custom_field_values[custom_field_code] ||= {} + self.custom_field_values[custom_field_code]["path"] = val + self.custom_field_values[custom_field_code]["digest"] = self.send _digest_name + else + write_attribute attr, val + end + end + + owner.send :define_singleton_method, "#{_attr_name}_will_change!" do + self.send "#{_digest_name}=", nil + custom_field_values_will_change! + end + + owner.send :define_singleton_method, _digest_name do + val = instance_variable_get "@#{_digest_name}" + if val.nil? && (file = send(_uploader_name)).present? + val = CustomField::Instance::Attachment.digest(file) + instance_variable_set "@#{_digest_name}", val + end + val + end + + _extension_whitelist = options["extension_whitelist"] + + owner.send :define_singleton_method, "#{_uploader_name}_extension_whitelist" do + _extension_whitelist + end + + unless owner.class.uploaders.has_key? _uploader_name.to_sym + owner.class.mount_uploader _uploader_name, CustomFieldAttachmentUploader, mount_on: "custom_field_#{code}_raw_value" + owner.class.send :attr_accessor, _digest_name + end + + digest = @raw_value && @raw_value["digest"] + owner.send "#{_digest_name}=", digest + end + + def self.digest file + Digest::SHA256.file(file.path).hexdigest + end + + def preprocess_value_for_assignment val + if val.present? + owner.send "#{uploader_name}=", val + else + @raw_value + end + end + + def checksum + owner.send digest_name + end + + def value + owner.send "custom_field_#{code}" + end + + def raw_value + @raw_value + end + + def attr_name + "custom_field_#{code}_raw_value" + end + + def uploader_name + "custom_field_#{code}" + end + + def digest_name + "#{uploader_name}_digest" + end + + def display_value + render_partial + end + + class Input < Base::Input + def form_input_options + super.update({ + as: :file, + wrapper: :horizontal_file_input + }) + end + end + end + + class String < Base + def value + "#{@raw_value}" + end + end + end end diff --git a/app/models/dashboard.rb b/app/models/dashboard.rb index 46c621266..e0857dca3 100644 --- a/app/models/dashboard.rb +++ b/app/models/dashboard.rb @@ -27,4 +27,5 @@ class Dashboard def current_organisation context.send(:current_organisation) end + end diff --git a/app/models/export.rb b/app/models/export.rb deleted file mode 100644 index 8c38d6684..000000000 --- a/app/models/export.rb +++ /dev/null @@ -1,53 +0,0 @@ -class Export - include JobConcern - - def initialize( response ) - @datas = response - end - - def report? - links["action_report"].present? - end - - def report - Rails.cache.fetch("#{cache_key}/action_report", expires_in: cache_expiration) do - report_path = links["action_report"] - if report_path - response = Ievkit.get(report_path) - ExportReport.new(response) - else - nil - end - end - end - - def destroy - delete_path = links["delete"] - cancel_path = links["cancel"] - - if delete_path - Ievkit.delete(delete_path) - elsif cancel_path - Ievkit.delete(cancel_path) - else - nil - end - end - - def file_path? - links["data"].present? - end - - def file_path - links["data"] - end - - def filename - File.basename(file_path) if file_path - end - - def filename_extension - File.extname(filename).gsub(".", "") if filename - end - -end diff --git a/app/models/export/base.rb b/app/models/export/base.rb new file mode 100644 index 000000000..c65539635 --- /dev/null +++ b/app/models/export/base.rb @@ -0,0 +1,135 @@ +require 'net/http/post/multipart' + +class Export::Base < ActiveRecord::Base + include Rails.application.routes.url_helpers + + self.table_name = "exports" + + belongs_to :referential + + validates :type, :referential_id, presence: true + + def self.messages_class_name + "Export::Message" + end + + def self.resources_class_name + "Export::Resource" + end + + def self.human_name + I18n.t("export.#{self.name.demodulize.underscore}") + end + + def self.file_extension_whitelist + %w(zip csv json) + end + + def upload_file file + url = URI.parse upload_workbench_export_url(self.workbench_id, self.id, host: Rails.application.config.rails_host) + res = nil + filename = File.basename(file.path) + content_type = MIME::Types.type_for(filename).first&.content_type + File.open(file.path) do |file_content| + req = Net::HTTP::Post::Multipart.new url.path, + file: UploadIO.new(file_content, content_type, filename), + token: self.token_upload + res = Net::HTTP.start(url.host, url.port) do |http| + http.request(req) + end + end + res + end + + if Rails.env.development? + def self.force_load_descendants + path = Rails.root.join 'app/models/export' + Dir.chdir path do + Dir['**/*.rb'].each do |src| + next if src =~ /^base/ + klass_name = "Export::#{src[0..-4].camelize}" + Rails.logger.info "Loading #{klass_name}" + begin + klass_name.constantize + rescue => e + Rails.logger.info "Failed: #{e.message}" + nil + end + end + end + end + end + + def self.user_visible? + false + end + + def self.inherited child + super child + child.instance_eval do + def self.user_visible? + true + end + end + end + + def self.option name, opts={} + store_accessor :options, name + + if opts[:serialize] + define_method name do + JSON.parse(options[name.to_s]) rescue opts[:serialize].new + end + end + + if !!opts[:required] + validates name, presence: true + end + @options ||= {} + @options[name] = opts + end + + def self.options + @options ||= {} + end + + def self.options= options + @options = options + end + + include IevInterfaces::Task + + def self.model_name + ActiveModel::Name.new Export::Base, Export::Base, "Export" + end + + def self.user_visible_descendants + descendants.select &:user_visible? + end + + def self.user_visible? + true + end + + def visible_options + options.select{|k, v| ! k.match /^_/} + end + + def display_option_value option_name, context + option = self.class.options[option_name.to_sym] + val = self.options[option_name.to_s] + if option[:display] + context.instance_exec(val, &option[:display]) + else + val + end + end + + private + + def initialize_fields + super + self.token_upload = SecureRandom.urlsafe_base64 + end + +end diff --git a/app/models/export/message.rb b/app/models/export/message.rb new file mode 100644 index 000000000..223429900 --- /dev/null +++ b/app/models/export/message.rb @@ -0,0 +1,8 @@ +class Export::Message < ApplicationModel + self.table_name = :export_messages + + include IevInterfaces::Message + + belongs_to :export, class_name: Export::Base + belongs_to :resource, class_name: Export::Resource +end diff --git a/app/models/export/netex.rb b/app/models/export/netex.rb new file mode 100644 index 000000000..069ec2209 --- /dev/null +++ b/app/models/export/netex.rb @@ -0,0 +1,22 @@ +class Export::Netex < Export::Base + after_commit :call_iev_callback, on: :create + option :export_type, collection: %w(line full), required: true + option :duration, type: :integer, default_value: 90, required: true + option :line_code + + private + + def iev_callback_url + URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/exporter/new?id=#{id}") + end + + # def self.user_visible? + # false + # end + + def destroy_non_ready_referential + if referential && !referential.ready + referential.destroy + end + end +end diff --git a/app/models/export/resource.rb b/app/models/export/resource.rb new file mode 100644 index 000000000..2a63c14a8 --- /dev/null +++ b/app/models/export/resource.rb @@ -0,0 +1,8 @@ +class Export::Resource < ApplicationModel + self.table_name = :export_resources + + include IevInterfaces::Resource + + belongs_to :export, class_name: Export::Base + has_many :messages, class_name: "ExportMessage", foreign_key: :resource_id +end diff --git a/app/models/export/simple_exporter/base.rb b/app/models/export/simple_exporter/base.rb new file mode 100644 index 000000000..e77e23468 --- /dev/null +++ b/app/models/export/simple_exporter/base.rb @@ -0,0 +1,94 @@ +class Export::SimpleExporter::Base < Export::Base + after_commit :call_exporter_async, on: :create + + def self.user_visible? + false + end + + def self.inherited child + super child + child.options = @options + child.instance_eval do + def self.user_visible? + true + end + end + end + + def call_exporter_async + SimpleExportWorker.perform_async(id) + end + + def simple_exporter_configuration_name + + end + + def exporter + @exporter ||= begin + if options[:_exporter_id] + exporter = SimpleJsonExporter.find options[:_exporter_id] + else + exporter = SimpleJsonExporter.create configuration_name: simple_exporter_configuration_name + options[:_exporter_id] = exporter.id + end + exporter + end + end + + def configure_exporter config + end + + def call_exporter + tmp = Tempfile.new [simple_exporter_configuration_name.to_s, ".json"] + referential.switch + exporter.configure do |config| + configure_exporter config + end + exporter.filepath = tmp.path + exporter.export + set_status_from_exporter + convert_exporter_journal_to_messages + self.save! + upload_file tmp + end + + def set_status_from_exporter + if exporter.status.to_s == "error" + self.status = :failed + elsif exporter.status.to_s == "success" + self.status = :successful + else + self.status = :warning + end + end + + def convert_exporter_journal_to_messages + self.messages.destroy_all + exporter.journal.each do |journal_item| + journal_item.symbolize_keys! + vals = {} + + if journal_item[:kind].to_s == "warning" + vals[:criticity] = :warning + elsif journal_item[:kind].to_s == "error" + vals[:criticity] = :error + else + vals[:criticity] = :info + if journal_item[:event].to_s == "success" + vals[:message_key] = :success + end + end + vals[:resource_attributes] = journal_item[:row] + + if journal_item[:message].present? + vals[:message_key] = :full_text + vals[:message_attributes] = { + text: journal_item[:message] + } + end + vals[:message_attributes] ||= {} + vals[:message_attributes][:line] = journal_item[:line] + self.messages.build vals + end + end +end diff --git a/app/models/export/workgroup.rb b/app/models/export/workgroup.rb new file mode 100644 index 000000000..3430596c7 --- /dev/null +++ b/app/models/export/workgroup.rb @@ -0,0 +1,9 @@ +class Export::Workgroup < Export::Base + after_commit :launch_worker, :on => :create + + option :duration, required: true, type: :integer, default_value: 90 + + def launch_worker + WorkgroupExportWorker.perform_async(id) + end +end diff --git a/app/models/export_log_message.rb b/app/models/export_log_message.rb deleted file mode 100644 index 4bb9d3cc7..000000000 --- a/app/models/export_log_message.rb +++ /dev/null @@ -1,42 +0,0 @@ -class ExportLogMessage < ActiveRecord::Base - belongs_to :export - - acts_as_list :scope => :export - - validates_presence_of :key - validates_inclusion_of :severity, :in => %w{info warning error ok uncheck fatal} - - def arguments=(arguments) - write_attribute :arguments, (arguments.to_json if arguments.present?) - end - - def arguments - @decoded_arguments ||= - begin - if (stored_arguments = raw_attributes).present? - ActiveSupport::JSON.decode stored_arguments - else - {} - end - end - end - - def raw_attributes - read_attribute(:arguments) - end - - before_validation :define_default_attributes, :on => :create - def define_default_attributes - self.severity ||= "info" - end - - def full_message - last_key=key.rpartition("|").last - begin - I18n.translate last_key, arguments.symbolize_keys.merge(:scope => "export_log_messages.messages").merge(:default => :undefined).merge(:key => last_key) - rescue => e - Rails.logger.error "missing arguments for message "+last_key - I18n.translate "WRONG_DATA",{"0"=>last_key}.symbolize_keys.merge(:scope => "export_log_messages.messages").merge(:default => :undefined).merge(:key => "WRONG_DATA") - end - end -end diff --git a/app/models/export_report.rb b/app/models/export_report.rb deleted file mode 100644 index 3c0788106..000000000 --- a/app/models/export_report.rb +++ /dev/null @@ -1,8 +0,0 @@ -class ExportReport - #include ReportConcern - - def initialize( response ) - @datas = response.action_report - end - -end diff --git a/app/models/export_service.rb b/app/models/export_service.rb deleted file mode 100644 index 2dbe0d7b3..000000000 --- a/app/models/export_service.rb +++ /dev/null @@ -1,23 +0,0 @@ -class ExportService - - attr_reader :referential - - def initialize(referential) - @referential = referential - end - - # Find an export whith his id - def find(id) - Export.new( Ievkit.scheduled_job(referential.slug, id, { :action => "exporter" }) ) - end - - # Find all exports - def all - [].tap do |jobs| - Ievkit.jobs(referential.slug, { :action => "exporter" }).each do |job| - jobs << Export.new( job ) - end - end - end - -end diff --git a/app/models/export_task.rb b/app/models/export_task.rb deleted file mode 100644 index f02cb914e..000000000 --- a/app/models/export_task.rb +++ /dev/null @@ -1,119 +0,0 @@ -class ExportTask - extend Enumerize - extend ActiveModel::Naming - extend ActiveModel::Translation - extend ActiveModel::Callbacks - include ActiveModel::Validations - include ActiveModel::Conversion - - attr_accessor :start_date, :end_date - - define_model_callbacks :initialize, only: :after - - enumerize :data_format, in: %w( neptune netex gtfs hub kml ) - attr_accessor :referential_id, :user_id, :user_name, :references_type, :data_format, :name, :projection_type, :reference_ids - - validates_presence_of :referential_id - validates_presence_of :user_id - validates_presence_of :user_name - validates_presence_of :name - validates_presence_of :data_format - - validate :period_validation - - after_initialize :init_period - - def initialize( params = {} ) - run_callbacks :initialize do - params.each {|k,v| send("#{k}=",v)} - end - end - - def period_validation - st_date = start_date.is_a?(String) ? Date.parse(start_date) : start_date - ed_date = end_date.is_a?(String) ? Date.parse(end_date) : end_date - - unless Chouette::TimeTable.start_validity_period.nil? || st_date.nil? - tt_st_date = Chouette::TimeTable.start_validity_period - errors.add(:start_date, ExportTask.human_attribute_name("start_date_greater_than" , {:tt_st_date => tt_st_date})) unless tt_st_date <= st_date - end - unless st_date.nil? || ed_date.nil? - errors.add(:end_date, ExportTask.human_attribute_name("end_date_greater_than_start_date")) unless st_date <= ed_date - end - unless ed_date.nil? || Chouette::TimeTable.end_validity_period.nil? - tt_ed_date = Chouette::TimeTable.end_validity_period - errors.add(:end_date, ExportTask.human_attribute_name("end_date_less_than", {:tt_ed_date => tt_ed_date})) unless ed_date <= tt_ed_date - end - end - - def init_period - unless Chouette::TimeTable.start_validity_period.nil? - if start_date.nil? - self.start_date = Chouette::TimeTable.start_validity_period - end - if end_date.nil? - self.end_date = Chouette::TimeTable.end_validity_period - end - end - end - - def referential - Referential.find(referential_id) - end - - def organisation - referential.organisation - end - - def save - if self.valid? - # Call Iev Server - begin - Ievkit.create_job( referential.slug, "exporter", data_format, { - :file1 => params_io, - } ) - rescue Exception => exception - raise exception - end - true - else - false - end - end - - def self.data_formats - self.data_format.values - end - - def self.references_types - self.references_type.values - end - - def params - {}.tap do |h| - h["parameters"] = action_params - end - end - - def action_params - {} - end - - def params_io - file = StringIO.new( params.to_json ) - Faraday::UploadIO.new(file, "application/json", "parameters.json") - end - - def self.optional_attributes(references_type) - [] - end - - def optional_attributes - self.class.optional_attributes(references_type.to_s) - end - - def optional_attribute?(attribute) - optional_attributes.include? attribute.to_sym - end - -end diff --git a/app/models/generic_attribute_control/min_max.rb b/app/models/generic_attribute_control/min_max.rb index 18873b683..bab900f0e 100644 --- a/app/models/generic_attribute_control/min_max.rb +++ b/app/models/generic_attribute_control/min_max.rb @@ -1,9 +1,7 @@ module GenericAttributeControl class MinMax < ComplianceControl store_accessor :control_attributes, :minimum, :maximum, :target - - validates_numericality_of :minimum, allow_nil: true, greater_than_or_equal_to: 0 - validates_numericality_of :maximum, allow_nil: true, greater_than_or_equal_to: 0 + validates :target, presence: true include MinMaxValuesValidation diff --git a/app/models/gtfs_export.rb b/app/models/gtfs_export.rb deleted file mode 100644 index d0b9fc4f9..000000000 --- a/app/models/gtfs_export.rb +++ /dev/null @@ -1,47 +0,0 @@ -class GtfsExport < ExportTask - - validates_presence_of :time_zone, unless: Proc.new { |e| e.optional_attribute? :time_zone } - attr_accessor :object_id_prefix, :time_zone - - enumerize :references_type, in: %w( network line company group_of_line stop_area ) - - after_initialize :init_params - - def init_params - if time_zone.nil? - self.time_zone = "Paris" - end - end - - def real_time_zone - ActiveSupport::TimeZone.find_tzinfo(time_zone).name - end - - def action_params - { - "gtfs-export" => { - "name" => name, - "references_type" => references_type, - "reference_ids" => reference_ids, - "user_name" => user_name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "time_zone" => real_time_zone, - "object_id_prefix" => object_id_prefix, - "start_date" => start_date, - "end_date" => end_date - } - } - end - - def self.optional_attributes(references_type) - super.tap do |optional_attributes| - optional_attributes.push :time_zone, :start_date, :end_date if references_type == "stop_area" - end - end - - def data_format - "gtfs" - end - -end diff --git a/app/models/gtfs_import.rb b/app/models/gtfs_import.rb deleted file mode 100644 index 1d7b5c6f5..000000000 --- a/app/models/gtfs_import.rb +++ /dev/null @@ -1,36 +0,0 @@ -class GtfsImport < ImportTask - - enumerize :references_type, in: %w( stop_area ) - - attr_accessor :object_id_prefix, :max_distance_for_commercial, :ignore_last_word, :ignore_end_chars, :max_distance_for_connection_link, :references_type - - validates_presence_of :object_id_prefix - - def references_types - self.references_type.values - end - - def action_params - { - "gtfs-import" => { - "no_save" => no_save, - "user_name" => user_name, - "name" => name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "object_id_prefix" => object_id_prefix, - "max_distance_for_commercial" => max_distance_for_commercial, - "ignore_last_word" => ignore_last_word, - "ignore_end_chars" => ignore_end_chars, - "max_distance_for_connection_link" => max_distance_for_connection_link, - "references_type" => references_type - } - } - end - - - def data_format - "gtfs" - end - -end diff --git a/app/models/hub_export.rb b/app/models/hub_export.rb deleted file mode 100644 index 802600692..000000000 --- a/app/models/hub_export.rb +++ /dev/null @@ -1,24 +0,0 @@ -class HubExport < ExportTask - - enumerize :references_type, in: %w( network line company group_of_line ) - - def action_params - { - "hub-export" => { - "name" => name, - "references_type" => references_type, - "reference_ids" => reference_ids, - "user_name" => user_name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "start_date" => start_date, - "end_date" => end_date - } - } - end - - def data_format - "hub" - end - -end diff --git a/app/models/import.rb b/app/models/import.rb deleted file mode 100644 index 29aadcd56..000000000 --- a/app/models/import.rb +++ /dev/null @@ -1,103 +0,0 @@ -class Import < ActiveRecord::Base - mount_uploader :file, ImportUploader - belongs_to :workbench - belongs_to :referential - - belongs_to :parent, polymorphic: true - - has_many :messages, class_name: "ImportMessage", dependent: :destroy - has_many :resources, class_name: "ImportResource", dependent: :destroy - has_many :children, foreign_key: :parent_id, class_name: "Import", dependent: :destroy - - scope :where_started_at_in, ->(period_range) do - where('started_at BETWEEN :begin AND :end', begin: period_range.begin, end: period_range.end) - end - - scope :blocked, -> { where('created_at < ? AND status = ?', 4.hours.ago, 'running') } - - extend Enumerize - enumerize :status, in: %w(new pending successful warning failed running aborted canceled), scope: true, default: :new - - validates :name, presence: true - validates :file, presence: true - validates_presence_of :workbench, :creator - - before_create :initialize_fields - - def self.model_name - ActiveModel::Name.new Import, Import, "Import" - end - - def children_succeedeed - children.with_status(:successful, :warning).count - end - - def self.launched_statuses - %w(new pending) - end - - def self.failed_statuses - %w(failed aborted canceled) - end - - def self.finished_statuses - %w(successful failed warning aborted canceled) - end - - def self.abort_old - where( - 'created_at < ? AND status NOT IN (?)', - 4.hours.ago, - finished_statuses - ).update_all(status: 'aborted') - end - - def notify_parent - parent.child_change - update(notified_parent_at: DateTime.now) - end - - def child_change - return if self.class.finished_statuses.include?(status) - - update_status - update_referentials - end - - def update_status - status = - if children.where(status: self.class.failed_statuses).count > 0 - 'failed' - elsif children.where(status: "warning").count > 0 - 'warning' - elsif children.where(status: "successful").count == children.count - 'successful' - end - - attributes = { - current_step: children.count, - status: status - } - - if self.class.finished_statuses.include?(status) - attributes[:ended_at] = Time.now - end - - update attributes - end - - def update_referentials - return unless self.class.finished_statuses.include?(status) - - children.each do |import| - import.referential.update(ready: true) if import.referential - end - end - - private - - def initialize_fields - self.token_download = SecureRandom.urlsafe_base64 - end - -end diff --git a/app/models/import/base.rb b/app/models/import/base.rb new file mode 100644 index 000000000..f98e359d4 --- /dev/null +++ b/app/models/import/base.rb @@ -0,0 +1,47 @@ +class Import::Base < ApplicationModel + self.table_name = "imports" + validates :file, presence: true + + def self.messages_class_name + "Import::Message" + end + + def self.resources_class_name + "Import::Resource" + end + + def self.file_extension_whitelist + %w(zip) + end + + include IevInterfaces::Task + + def self.model_name + ActiveModel::Name.new Import::Base, Import::Base, "Import" + end + + def child_change + Rails.logger.info "child_change for #{inspect}" + return if self.class.finished_statuses.include?(status) + + super + update_referentials + end + + def update_referentials + Rails.logger.info "update_referentials for #{inspect}" + return unless self.class.finished_statuses.include?(status) + + children.each do |import| + import.referential.update(ready: true) if import.referential + end + end + + private + + def initialize_fields + super + self.token_download ||= SecureRandom.urlsafe_base64 + end + +end diff --git a/app/models/import/gtfs.rb b/app/models/import/gtfs.rb new file mode 100644 index 000000000..a20c468c1 --- /dev/null +++ b/app/models/import/gtfs.rb @@ -0,0 +1,309 @@ +class Import::Gtfs < Import::Base + after_commit :launch_worker, :on => :create + + def launch_worker + GtfsImportWorker.perform_async id + end + + def import + update status: 'running', started_at: Time.now + + import_without_status + update status: 'successful', ended_at: Time.now + rescue Exception => e + update status: 'failed', ended_at: Time.now + Rails.logger.error "Error in GTFS import: #{e} #{e.backtrace.join('\n')}" + ensure + notify_parent + referential&.update ready: true + end + + def self.accept_file?(file) + Zip::File.open(file) do |zip_file| + zip_file.glob('agency.txt').size == 1 + end + rescue Exception => e + Rails.logger.debug "Error in testing GTFS file: #{e}" + return false + end + + def create_referential + self.referential ||= Referential.create!( + name: "GTFS Import", + organisation_id: workbench.organisation_id, + workbench_id: workbench.id, + metadatas: [referential_metadata] + ) + end + + def referential_metadata + registration_numbers = source.routes.map(&:id) + line_ids = line_referential.lines.where(registration_number: registration_numbers).pluck(:id) + + start_dates, end_dates = source.calendars.map { |c| [c.start_date, c.end_date ] }.transpose + excluded_dates = source.calendar_dates.select { |d| d.exception_type == "2" }.map(&:date) + + min_date = Date.parse (start_dates + [excluded_dates.min]).compact.min + max_date = Date.parse (end_dates + [excluded_dates.max]).compact.max + + ReferentialMetadata.new line_ids: line_ids, periodes: [min_date..max_date] + end + + attr_accessor :local_file + def local_file + @local_file ||= download_local_file + end + + attr_accessor :download_host + def download_host + @download_host ||= Rails.application.config.rails_host + end + + def local_temp_directory + @local_temp_directory ||= + begin + directory = Rails.application.config.try(:import_temporary_directory) || Rails.root.join('tmp', 'imports') + FileUtils.mkdir_p directory + directory + end + end + + def local_temp_file(&block) + Tempfile.open("chouette-import", local_temp_directory) do |file| + file.binmode + yield file + end + end + + def download_path + Rails.application.routes.url_helpers.download_workbench_import_path(workbench, id, token: token_download) + end + + def download_uri + @download_uri ||= + begin + host = download_host + host = "http://#{host}" unless host =~ %r{https?://} + URI.join(host, download_path) + end + end + + def download_local_file + local_temp_file do |file| + begin + Net::HTTP.start(download_uri.host, download_uri.port) do |http| + http.request_get(download_uri.request_uri) do |response| + response.read_body do |segment| + file.write segment + end + end + end + ensure + file.close + end + + file.path + end + end + + def source + @source ||= ::GTFS::Source.build local_file + end + + delegate :line_referential, :stop_area_referential, to: :workbench + + def prepare_referential + import_agencies + import_stops + import_routes + + create_referential + referential.switch + end + + def import_without_status + prepare_referential + + import_calendars + import_trips + import_stop_times + end + + def import_agencies + Chouette::Company.transaction do + source.agencies.each do |agency| + company = line_referential.companies.find_or_initialize_by(registration_number: agency.id) + company.attributes = { name: agency.name } + + save_model company + end + end + end + + def import_stops + Chouette::StopArea.transaction do + source.stops.each do |stop| + stop_area = stop_area_referential.stop_areas.find_or_initialize_by(registration_number: stop.id) + + stop_area.name = stop.name + stop_area.area_type = stop.location_type == "1" ? "zdlp" : "zdep" + stop_area.parent = stop_area_referential.stop_areas.find_by!(registration_number: stop.parent_station) if stop.parent_station.present? + stop_area.latitude, stop_area.longitude = stop.lat, stop.lon + stop_area.kind = "commercial" + + # TODO correct default timezone + + save_model stop_area + end + end + end + + def import_routes + Chouette::Line.transaction do + source.routes.each do |route| + line = line_referential.lines.find_or_initialize_by(registration_number: route.id) + line.name = route.long_name.presence || route.short_name + line.number = route.short_name + line.published_name = route.long_name + + line.company = line_referential.companies.find_by(registration_number: route.agency_id) if route.agency_id.present? + + # TODO transport mode + + line.comment = route.desc + + # TODO colors + + line.url = route.url + + save_model line + end + end + end + + def vehicle_journey_by_trip_id + @vehicle_journey_by_trip_id ||= {} + end + + def import_trips + source.trips.each_slice(100) do |slice| + slice.each do |trip| + Chouette::Route.transaction do + line = line_referential.lines.find_by registration_number: trip.route_id + + route = referential.routes.build line: line + route.wayback = (trip.direction_id == "0" ? :outbound : :inbound) + # TODO better name ? + name = route.published_name = trip.short_name.presence || trip.headsign.presence || route.wayback.to_s.capitalize + route.name = name + save_model route + + journey_pattern = route.journey_patterns.build name: name + save_model journey_pattern + + vehicle_journey = journey_pattern.vehicle_journeys.build route: route + vehicle_journey.published_journey_name = trip.headsign.presence || trip.id + save_model vehicle_journey + + time_table = referential.time_tables.find_by(id: time_tables_by_service_id[trip.service_id]) if time_tables_by_service_id[trip.service_id] + if time_table + vehicle_journey.time_tables << time_table + else + messages.create! criticity: "warning", message_key: "gtfs.trips.unkown_service_id", message_attributes: {service_id: trip.service_id} + end + + vehicle_journey_by_trip_id[trip.id] = vehicle_journey.id + end + end + end + end + + def import_stop_times + source.stop_times.group_by(&:trip_id).each_slice(50) do |slice| + slice.each do |trip_id, stop_times| + Chouette::VehicleJourneyAtStop.transaction do + vehicle_journey = referential.vehicle_journeys.find vehicle_journey_by_trip_id[trip_id] + journey_pattern = vehicle_journey.journey_pattern + route = journey_pattern.route + + stop_times.sort_by! { |s| s.stop_sequence.to_i } + + stop_times.each do |stop_time| + stop_area = stop_area_referential.stop_areas.find_by(registration_number: stop_time.stop_id) + + stop_point = route.stop_points.build stop_area: stop_area + save_model stop_point + + journey_pattern.stop_points << stop_point + + # JourneyPattern#vjas_add creates automaticaly VehicleJourneyAtStop + vehicle_journey_at_stop = journey_pattern.vehicle_journey_at_stops.find_by(stop_point_id: stop_point.id) + + departure_time = GTFS::Time.parse(stop_time.departure_time) + arrival_time = GTFS::Time.parse(stop_time.arrival_time) + + vehicle_journey_at_stop.departure_time = departure_time.time + vehicle_journey_at_stop.arrival_time = arrival_time.time + vehicle_journey_at_stop.departure_day_offset = departure_time.day_offset + vehicle_journey_at_stop.arrival_day_offset = arrival_time.day_offset + + # TODO offset + + save_model vehicle_journey_at_stop + end + end + end + end + end + + def time_tables_by_service_id + @time_tables_by_service_id ||= {} + end + + def import_calendars + source.calendars.each_slice(500) do |slice| + Chouette::TimeTable.transaction do + slice.each do |calendar| + time_table = referential.time_tables.build comment: "Calendar #{calendar.service_id}" + Chouette::TimeTable.all_days.each do |day| + time_table.send("#{day}=", calendar.send(day)) + end + time_table.periods.build period_start: calendar.start_date, period_end: calendar.end_date + + save_model time_table + + time_tables_by_service_id[calendar.service_id] = time_table.id + end + end + end + end + + def import_calendar_dates + source.calendar_dates.each_slice(500) do |slice| + Chouette::TimeTable.transaction do + slice.each do |calendar_date| + time_table = referential.time_tables.find time_tables_by_service_id[calendar_date.service_id] + date = time_table.dates.build date: Date.parse(calendar_date.date), in_out: calendar_date.exception_type == "1" + + save_model date + end + end + end + end + + def save_model(model) + unless model.save + Rails.logger.info "Can't save #{model.class.name} : #{model.errors.inspect}" + raise ActiveRecord::RecordNotSaved.new("Invalid #{model.class.name} : #{model.errors.inspect}") + end + Rails.logger.debug "Created #{model.inspect}" + end + + def notify_parent + return unless parent.present? + return if notified_parent_at + parent.child_change + update_column :notified_parent_at, Time.now + end + +end diff --git a/app/models/import/message.rb b/app/models/import/message.rb new file mode 100644 index 000000000..30b76ec5c --- /dev/null +++ b/app/models/import/message.rb @@ -0,0 +1,8 @@ +class Import::Message < ApplicationModel + self.table_name = :import_messages + + include IevInterfaces::Message + + belongs_to :import, class_name: Import::Base + belongs_to :resource, class_name: Import::Resource +end diff --git a/app/models/import_message_export.rb b/app/models/import/message_export.rb index 05f8a2cc7..7d03783ed 100644 --- a/app/models/import_message_export.rb +++ b/app/models/import/message_export.rb @@ -2,7 +2,7 @@ require "csv" require "zip" -class ImportMessageExport +class Import::MessageExport include ActiveModel::Validations include ActiveModel::Conversion extend ActiveModel::Naming @@ -22,16 +22,18 @@ class ImportMessageExport end def column_names - ["criticity", "message key", "message", "file name", "line", "column"] + ["criticity", "message_key", "message", "filename", "line", "column"].map {|c| Import::Message.tmf(c)} end def to_csv(options = {}) - CSV.generate(options) do |csv| + csv_string = CSV.generate(options) do |csv| csv << column_names import_messages.each do |import_message| - csv << [import_message.criticity, import_message.message_key, I18n.t("import_messages.#{import_message.message_key}", import_message.message_attributes.deep_symbolize_keys), *import_message.resource_attributes.values_at("filename", "line_number", "column_number") ] + csv << [import_message.criticity, import_message.message_attributes['test_id'], I18n.t("import_messages.#{import_message.message_key}", import_message.message_attributes.deep_symbolize_keys), *import_message.resource_attributes.values_at("filename", "line_number", "column_number") ] end end + # We add a BOM to indicate we use UTF-8 + "\uFEFF" + csv_string end def to_zip(temp_file,options = {}) diff --git a/app/models/import/netex.rb b/app/models/import/netex.rb new file mode 100644 index 000000000..f19fde435 --- /dev/null +++ b/app/models/import/netex.rb @@ -0,0 +1,63 @@ +require 'net/http' +class Import::Netex < Import::Base + before_destroy :destroy_non_ready_referential + + after_commit :call_iev_callback, on: :create + + before_save def abort_unless_referential + self.status = 'aborted' unless referential + end + + validates_presence_of :parent + + def create_with_referential! + self.referential = + Referential.new( + name: self.name, + organisation_id: workbench.organisation_id, + workbench_id: workbench.id, + metadatas: [referential_metadata] + ) + self.referential.save + if self.referential.invalid? + Rails.logger.info "Can't create referential for import #{self.id}: #{referential.inspect} #{referential.metadatas.inspect} #{referential.errors.messages}" + if referential.metadatas.all?{|m| m.line_ids.present? && m.line_ids.empty?} + parent.messages.create criticity: :error, message_key: "referential_creation_missing_lines", message_attributes: {referential_name: referential.name} + else + parent.messages.create criticity: :error, message_key: "referential_creation", message_attributes: {referential_name: referential.name} + end + else + save! + end + end + + private + + def iev_callback_url + URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}") + end + + def destroy_non_ready_referential + if referential && !referential.ready + referential.destroy + end + end + + def referential_metadata + metadata = ReferentialMetadata.new + + if self.file && self.file.path + netex_file = STIF::NetexFile.new(self.file.path) + frame = netex_file.frames.first + + if frame + metadata.periodes = frame.periods + + line_objectids = frame.line_refs.map { |ref| "STIF:CODIFLIGNE:Line:#{ref}" } + metadata.line_ids = workbench.lines.where(objectid: line_objectids).pluck(:id) + end + end + + metadata + end +end diff --git a/app/models/import/resource.rb b/app/models/import/resource.rb new file mode 100644 index 000000000..1951daacd --- /dev/null +++ b/app/models/import/resource.rb @@ -0,0 +1,8 @@ +class Import::Resource < ApplicationModel + self.table_name = :import_resources + + include IevInterfaces::Resource + + belongs_to :import, class_name: Import::Base + has_many :messages, class_name: "Import::Message", foreign_key: :resource_id +end diff --git a/app/models/import/workbench.rb b/app/models/import/workbench.rb new file mode 100644 index 000000000..124b9b0d8 --- /dev/null +++ b/app/models/import/workbench.rb @@ -0,0 +1,26 @@ +class Import::Workbench < Import::Base + after_commit :launch_worker, :on => :create + + def launch_worker + unless Import::Gtfs.accept_file?(file.path) + WorkbenchImportWorker.perform_async(id) + else + import_gtfs + end + end + + def import_gtfs + update_column :status, 'running' + update_column :started_at, Time.now + + Import::Gtfs.create! parent_id: self.id, workbench: workbench, file: File.new(file.path), name: "Import GTFS", creator: "Web service" + + update_column :status, 'successful' + update_column :ended_at, Time.now + rescue Exception => e + Rails.logger.error "Error while processing GTFS file: #{e}" + + update_column :status, 'failed' + update_column :ended_at, Time.now + end +end diff --git a/app/models/import_message.rb b/app/models/import_message.rb deleted file mode 100644 index de70c35d1..000000000 --- a/app/models/import_message.rb +++ /dev/null @@ -1,8 +0,0 @@ -class ImportMessage < ActiveRecord::Base - extend Enumerize - belongs_to :import - belongs_to :resource, class_name: ImportResource - enumerize :criticity, in: %i(info warning error) - - validates :criticity, presence: true -end diff --git a/app/models/import_report.rb b/app/models/import_report.rb deleted file mode 100644 index ba13f0118..000000000 --- a/app/models/import_report.rb +++ /dev/null @@ -1,8 +0,0 @@ -class ImportReport - #include ReportConcern - - def initialize( response ) - @datas = response.action_report - end - -end diff --git a/app/models/import_resource.rb b/app/models/import_resource.rb deleted file mode 100644 index 55e752e74..000000000 --- a/app/models/import_resource.rb +++ /dev/null @@ -1,11 +0,0 @@ -class ImportResource < ActiveRecord::Base - belongs_to :import - - extend Enumerize - enumerize :status, in: %i(OK ERROR WARNING IGNORED), scope: true - - validates_presence_of :name, :resource_type, :reference - - has_many :messages, class_name: "ImportMessage", foreign_key: :resource_id - -end diff --git a/app/models/import_service.rb b/app/models/import_service.rb deleted file mode 100644 index 2e3c1012b..000000000 --- a/app/models/import_service.rb +++ /dev/null @@ -1,23 +0,0 @@ -class ImportService - - attr_reader :referential - - def initialize( referential ) - @referential = referential - end - - # Find an import whith his id - def find(id) - Import.new( Ievkit.scheduled_job(referential.slug, id, { :action => "importer" }) ) - end - - # Find all imports - def all - [].tap do |jobs| - Ievkit.jobs(referential.slug, { :action => "importer" }).each do |job| - jobs << Import.new( job ) - end - end - end - -end diff --git a/app/models/import_task.rb b/app/models/import_task.rb deleted file mode 100644 index 7dfa2c644..000000000 --- a/app/models/import_task.rb +++ /dev/null @@ -1,141 +0,0 @@ -require "zip" - -class ImportTask - extend Enumerize - extend ActiveModel::Naming - extend ActiveModel::Translation - include ActiveModel::Validations - include ActiveModel::Conversion - - # TODO : Move in configuration - @@root = "#{Rails.root}/tmp/imports" - cattr_accessor :root - - enumerize :data_format, in: %w( neptune netex gtfs ) - attr_accessor :referential_id, :user_id, :user_name, :data_format, :resources, :name, :no_save - - validates_presence_of :referential_id - validates_presence_of :resources - validates_presence_of :user_id - validates_presence_of :user_name - validates_presence_of :name - - validate :validate_file_size, :validate_file_content - - def initialize( params = {} ) - params.each {|k,v| send("#{k}=",v)} - end - - def referential - Referential.find(referential_id) - end - - def organisation - referential.organisation - end - - def save - if valid? - # Save resources - save_resources - - # Call Iev Server - begin - Ievkit.create_job(referential.slug, "importer", data_format, { - :file1 => params_io, - :file2 => transport_data_io - } - - ) - - # Delete resources - delete_resources - rescue Exception => exception - # If iev server has an error must delete resources before - delete_resources - - raise exception - end - true - else - false - end - end - - def params - {}.tap do |h| - h["parameters"] = {} - end - end - - def self.data_formats - self.data_format.values - end - - def params_io - file = StringIO.new( params.to_json ) - Faraday::UploadIO.new(file, "application/json", "parameters.json") - end - - def transport_data_io - file = File.new(saved_resources_path, "r") - if file_extname == ".zip" - Faraday::UploadIO.new(file, "application/zip", original_filename ) - elsif file_extname == ".xml" - Faraday::UploadIO.new(file, "application/xml", original_filename ) - end - end - - def save_resources - FileUtils.mkdir_p root - FileUtils.cp resources.path, saved_resources_path - end - - def delete_resources - FileUtils.rm saved_resources_path if File.exists? saved_resources_path - end - - def original_filename - resources.original_filename - end - - def file_extname - File.extname(original_filename) if original_filename - end - - def saved_resources_path - @saved_resources_path ||= "#{root}/#{Time.now.to_i}#{file_extname}" - end - - @@maximum_file_size = 80.megabytes - cattr_accessor :maximum_file_size - - def validate_file_size - return unless resources.present? and resources.path.present? and File.exists? resources.path - - if File.size(resources.path) > maximum_file_size - message = I18n.t("activemodel.errors.models.import_task.attributes.resources.maximum_file_size", file_size: ActionController::Base.helpers.number_to_human_size(File.size(resources.path)), maximum_file_size: ActionController::Base.helpers.number_to_human_size(maximum_file_size)) - errors.add(:resources, message) - end - end - - @@valid_mime_types = { - neptune: %w{application/zip application/xml}, - netex: %w{application/zip}, - gtfs: %w{application/zip text/plain} - } - cattr_accessor :valid_mime_types - - def validate_file_content - return unless resources.present? and resources.path.present? and File.exists? resources.path - - mime_type = (File.open(resources.path) { |f| MimeMagic.by_magic f }).try :type - expected_mime_types = valid_mime_types[data_format.to_sym] - - unless expected_mime_types.include? mime_type - message = I18n.t("activemodel.errors.models.import_task.attributes.resources.invalid_mime_type", mime_type: mime_type) - errors.add(:resources, message) - end - end - -end diff --git a/app/models/kml_export.rb b/app/models/kml_export.rb deleted file mode 100644 index f6db77172..000000000 --- a/app/models/kml_export.rb +++ /dev/null @@ -1,24 +0,0 @@ -class KmlExport < ExportTask - - enumerize :references_type, in: %w( network line company group_of_line ) - - def action_params - { - "kml-export" => { - "name" => name, - "references_type" => references_type, - "reference_ids" => reference_ids, - "user_name" => user_name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "start_date" => start_date, - "end_date" => end_date - } - } - end - - def data_format - "kml" - end - -end diff --git a/app/models/line_control/lines_scope.rb b/app/models/line_control/lines_scope.rb new file mode 100644 index 000000000..4210a10dd --- /dev/null +++ b/app/models/line_control/lines_scope.rb @@ -0,0 +1,8 @@ +module LineControl + class LinesScope < ComplianceControl + + def self.default_code; "3-Line-2" end + + def prerequisite; I18n.t("compliance_controls.#{self.class.name.underscore}.prerequisite") end + end +end diff --git a/app/models/line_referential.rb b/app/models/line_referential.rb index 15b2f6276..08193c960 100644 --- a/app/models/line_referential.rb +++ b/app/models/line_referential.rb @@ -1,7 +1,7 @@ -class LineReferential < ActiveRecord::Base +class LineReferential < ApplicationModel include ObjectidFormatterSupport extend StifTransportModeEnumerations - + has_many :line_referential_memberships has_many :organisations, through: :line_referential_memberships has_many :lines, class_name: 'Chouette::Line' @@ -10,10 +10,11 @@ class LineReferential < ActiveRecord::Base has_many :networks, class_name: 'Chouette::Network' has_many :line_referential_syncs, -> { order created_at: :desc } has_many :workbenches + has_one :workgroup def add_member(organisation, options = {}) attributes = options.merge organisation: organisation - line_referential_memberships.build attributes + line_referential_memberships.build attributes unless organisations.include?(organisation) end validates :name, presence: true diff --git a/app/models/line_referential_membership.rb b/app/models/line_referential_membership.rb index b49d1b5b1..8371bdc32 100644 --- a/app/models/line_referential_membership.rb +++ b/app/models/line_referential_membership.rb @@ -1,4 +1,6 @@ -class LineReferentialMembership < ActiveRecord::Base +class LineReferentialMembership < ApplicationModel belongs_to :organisation belongs_to :line_referential + + validates :organisation_id, presence: true, uniqueness: { scope: :line_referential } end diff --git a/app/models/line_referential_sync.rb b/app/models/line_referential_sync.rb index 75c1e48a2..39e3846f0 100644 --- a/app/models/line_referential_sync.rb +++ b/app/models/line_referential_sync.rb @@ -1,4 +1,4 @@ -class LineReferentialSync < ActiveRecord::Base +class LineReferentialSync < ApplicationModel include AASM belongs_to :line_referential has_many :line_referential_sync_messages, :dependent => :destroy diff --git a/app/models/line_referential_sync_message.rb b/app/models/line_referential_sync_message.rb index 3b6cf3367..00a2b58a3 100644 --- a/app/models/line_referential_sync_message.rb +++ b/app/models/line_referential_sync_message.rb @@ -1,4 +1,4 @@ -class LineReferentialSyncMessage < ActiveRecord::Base +class LineReferentialSyncMessage < ApplicationModel belongs_to :line_referential_sync enum criticity: [:info, :warning, :error] diff --git a/app/models/merge.rb b/app/models/merge.rb index 62bf581d6..7df7aa590 100644 --- a/app/models/merge.rb +++ b/app/models/merge.rb @@ -1,4 +1,4 @@ -class Merge < ActiveRecord::Base +class Merge < ApplicationModel extend Enumerize belongs_to :workbench @@ -50,7 +50,7 @@ class Merge < ActiveRecord::Base new = if workbench.output.current Rails.logger.debug "Clone current output" - Referential.new_from(workbench.output.current, fixme_functional_scope).tap do |clone| + Referential.new_from(workbench.output.current, workbench.organisation).tap do |clone| clone.inline_clone = true end else @@ -135,10 +135,18 @@ class Merge < ActiveRecord::Base referential_stop_points_by_route = referential_stop_points.group_by(&:route_id) + referential_routes_constraint_zones = referential.switch do + referential.routing_constraint_zones.each_with_object(Hash.new { |h,k| h[k] = [] }) do |routing_constraint_zone, hash| + hash[routing_constraint_zone.route_id] << routing_constraint_zone + end + end + new.switch do referential_routes.each do |route| existing_route = new.routes.find_by line_id: route.line_id, checksum: route.checksum - unless existing_route + if existing_route + existing_route.merge_metadata_from route + else objectid = Chouette::Route.where(objectid: route.objectid).exists? ? nil : route.objectid attributes = route.attributes.merge( id: nil, @@ -152,21 +160,50 @@ class Merge < ActiveRecord::Base route_stop_points = referential_stop_points_by_route[route.id] # Stop Points - route_stop_points.each do |stop_point| + route_stop_points.sort_by(&:position).each do |stop_point| objectid = Chouette::StopPoint.where(objectid: stop_point.objectid).exists? ? nil : stop_point.objectid attributes = stop_point.attributes.merge( id: nil, route_id: nil, objectid: objectid, ) - new_route.stop_points.build attributes end + # We need to create StopPoints to known new primary keys + new_route.save! + + old_stop_point_ids = route_stop_points.sort_by(&:position).map(&:id) + new_stop_point_ids = new_route.stop_points.sort_by(&:position).map(&:id) + + stop_point_ids_mapping = Hash[[old_stop_point_ids, new_stop_point_ids].transpose] + + # RoutingConstraintZones + routes_constraint_zones = referential_routes_constraint_zones[route.id] + + routes_constraint_zones.each do |routing_constraint_zone| + objectid = new.routing_constraint_zones.where(objectid: routing_constraint_zone.objectid).exists? ? nil : routing_constraint_zone.objectid + stop_point_ids = routing_constraint_zone.stop_point_ids.map { |id| stop_point_ids_mapping[id] }.compact + + if stop_point_ids.size != routing_constraint_zone.stop_point_ids.size + raise "Can't find all required StopPoints for RoutingConstraintZone #{routing_constraint_zone.inspect}" + end + + attributes = routing_constraint_zone.attributes.merge( + id: nil, + route_id: nil, + objectid: objectid, + stop_point_ids: stop_point_ids, + ) + new_route.routing_constraint_zones.build attributes + + # No checksum check. RoutingConstraintZones are always recreated + end + new_route.save! if new_route.checksum != route.checksum - raise "Checksum has changed: #{route.inspect} #{new_route.inspect}" + raise "Checksum has changed: \"#{route.checksum}\", \"#{route.checksum_source}\" -> \"#{new_route.checksum}\", \"#{new_route.checksum_source}\"" end end end @@ -197,7 +234,9 @@ class Merge < ActiveRecord::Base existing_journey_pattern = new.journey_patterns.find_by route_id: existing_associated_route.id, checksum: journey_pattern.checksum - unless existing_journey_pattern + if existing_journey_pattern + existing_journey_pattern.merge_metadata_from journey_pattern + else objectid = Chouette::JourneyPattern.where(objectid: journey_pattern.objectid).exists? ? nil : journey_pattern.objectid attributes = journey_pattern.attributes.merge( id: nil, @@ -221,7 +260,7 @@ class Merge < ActiveRecord::Base new_journey_pattern = new.journey_patterns.create! attributes if new_journey_pattern.checksum != journey_pattern.checksum - raise "Checksum has changed for #{journey_pattern.inspect}: #{journey_pattern.checksum_source} #{new_journey_pattern.checksum_source} " + raise "Checksum has changed for #{journey_pattern.inspect} (to #{new_journey_pattern.inspect}): \"#{journey_pattern.checksum_source}\" -> \"#{new_journey_pattern.checksum_source}\"" end end end @@ -233,16 +272,37 @@ class Merge < ActiveRecord::Base referential.vehicle_journeys.includes(:vehicle_journey_at_stops).all.to_a end + referential_purchase_windows_by_checksum, referential_vehicle_journey_purchase_window_checksums = referential.switch do + purchase_windows_by_checksum = referential.purchase_windows.each_with_object({}) do |purchase_window, hash| + hash[purchase_window.checksum] = purchase_window + end + + vehicle_journey_purchase_window_checksums = Hash.new { |h,k| h[k] = [] } + referential.purchase_windows.joins(:vehicle_journeys).pluck("vehicle_journeys.id", :checksum).each do |vehicle_journey_id, checksum| + vehicle_journey_purchase_window_checksums[vehicle_journey_id] << checksum + end + + [purchase_windows_by_checksum, vehicle_journey_purchase_window_checksums] + end + + new_vehicle_journey_ids = {} + new.switch do referential_vehicle_journeys.each do |vehicle_journey| # find parent journey pattern by checksum # TODO add line_id for security + associated_route_checksum = referential_routes_checksums[vehicle_journey.route_id] associated_journey_pattern_checksum = referential_journey_patterns_checksums[vehicle_journey.journey_pattern_id] - existing_associated_journey_pattern = new.journey_patterns.find_by checksum: associated_journey_pattern_checksum + + existing_associated_route = new.routes.find_by checksum: associated_route_checksum + existing_associated_journey_pattern = existing_associated_route.journey_patterns.find_by checksum: associated_journey_pattern_checksum existing_vehicle_journey = new.vehicle_journeys.find_by journey_pattern_id: existing_associated_journey_pattern.id, checksum: vehicle_journey.checksum - unless existing_vehicle_journey + if existing_vehicle_journey + existing_vehicle_journey.merge_metadata_from vehicle_journey + new_vehicle_journey_ids[vehicle_journey.id] = existing_vehicle_journey.id + else objectid = Chouette::VehicleJourney.where(objectid: vehicle_journey.objectid).exists? ? nil : vehicle_journey.objectid attributes = vehicle_journey.attributes.merge( id: nil, @@ -264,11 +324,39 @@ class Merge < ActiveRecord::Base new_vehicle_journey.vehicle_journey_at_stops.build at_stop_attributes end + # Associate (and create if needed) PurchaseWindows + + referential_vehicle_journey_purchase_window_checksums[vehicle_journey.id].each do |purchase_window_checksum| + associated_purchase_window = new.purchase_windows.find_by(checksum: purchase_window_checksum) + + unless associated_purchase_window + purchase_window = referential_purchase_windows_by_checksum[purchase_window_checksum] + + objectid = new.purchase_windows.where(objectid: purchase_window.objectid).exists? ? nil : purchase_window.objectid + attributes = purchase_window.attributes.merge( + id: nil, + objectid: objectid + ) + new_purchase_window = new.purchase_windows.build attributes + new_purchase_window.save! + + if new_purchase_window.checksum != purchase_window.checksum + raise "Checksum has changed: #{purchase_window.checksum_source} #{new_purchase_window.checksum_source}" + end + + associated_purchase_window = new_purchase_window + end + + new_vehicle_journey.purchase_windows << associated_purchase_window + end + new_vehicle_journey.save! if new_vehicle_journey.checksum != vehicle_journey.checksum raise "Checksum has changed: #{vehicle_journey.checksum_source} #{new_vehicle_journey.checksum_source}" end + + new_vehicle_journey_ids[vehicle_journey.id] = new_vehicle_journey.id end end @@ -280,14 +368,14 @@ class Merge < ActiveRecord::Base time_tables_by_id = Hash[referential.time_tables.includes(:dates, :periods).all.to_a.map { |t| [t.id, t] }] time_tables_with_associated_lines = - referential.time_tables.joins(vehicle_journeys: {route: :line}).pluck("lines.id", :id, "vehicle_journeys.checksum") + referential.time_tables.joins(vehicle_journeys: {route: :line}).pluck("lines.id", :id, "vehicle_journeys.id") # Because TimeTables will be modified according metadata periods # we're loading timetables per line (line is associated to a period list) # # line_id: [ { time_table.id, vehicle_journey.checksum } ] time_tables_by_lines = time_tables_with_associated_lines.inject(Hash.new { |h,k| h[k] = [] }) do |hash, row| - hash[row.shift] << {id: row.first, vehicle_journey_checksum: row.second} + hash[row.shift] << {id: row.first, vehicle_journey_id: row.second} hash end @@ -339,7 +427,9 @@ class Merge < ActiveRecord::Base existing_time_table = line.time_tables.find_by checksum: candidate_time_table.checksum - unless existing_time_table + if existing_time_table + existing_time_table.merge_metadata_from candidate_time_table + else objectid = Chouette::TimeTable.where(objectid: time_table.objectid).exists? ? nil : time_table.objectid candidate_time_table.objectid = objectid @@ -355,7 +445,12 @@ class Merge < ActiveRecord::Base # associate VehicleJourney - associated_vehicle_journey = line.vehicle_journeys.find_by!(checksum: properties[:vehicle_journey_checksum]) + new_vehicle_journey_id = new_vehicle_journey_ids[properties[:vehicle_journey_id]] + unless new_vehicle_journey_id + raise "TimeTable #{existing_time_table.inspect} associated to a not-merged VehicleJourney: #{properties[:vehicle_journey_id]}" + end + + associated_vehicle_journey = line.vehicle_journeys.find(new_vehicle_journey_id) associated_vehicle_journey.time_tables << existing_time_table end end @@ -364,7 +459,7 @@ class Merge < ActiveRecord::Base def save_current output.update current: new, new: nil - output.current.update referential_suite: output + output.current.update referential_suite: output, ready: true referentials.update_all merged_at: created_at, archived_at: created_at end diff --git a/app/models/neptune_export.rb b/app/models/neptune_export.rb deleted file mode 100644 index f25db69c0..000000000 --- a/app/models/neptune_export.rb +++ /dev/null @@ -1,27 +0,0 @@ -class NeptuneExport < ExportTask - - attr_accessor :extensions, :export_type - enumerize :references_type, in: %w( network line company group_of_line ) - - def action_params - { - "neptune-export" => { - "name" => name, - "references_type" => references_type, - "reference_ids" => reference_ids, - "user_name" => user_name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "projection_type" => projection_type || "", - "add_extension" => extensions, - "start_date" => start_date, - "end_date" => end_date - } - } - end - - def data_format - "neptune" - end - -end diff --git a/app/models/neptune_import.rb b/app/models/neptune_import.rb deleted file mode 100644 index 1f0bdaa13..000000000 --- a/app/models/neptune_import.rb +++ /dev/null @@ -1,19 +0,0 @@ -class NeptuneImport < ImportTask - - def action_params - { - "neptune-import" => { - "no_save" => no_save, - "user_name" => user_name, - "name" => name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - } - } - end - - def data_format - "neptune" - end - -end diff --git a/app/models/netex_export.rb b/app/models/netex_export.rb deleted file mode 100644 index a4c3e2454..000000000 --- a/app/models/netex_export.rb +++ /dev/null @@ -1,24 +0,0 @@ -class NetexExport < ExportTask - - enumerize :references_type, in: %w( network line company group_of_line ) - - def action_params - { - "netex-export" => { - "name" => name, - "references_type" => references_type, - "reference_ids" => reference_ids, - "user_name" => user_name, - "organisation_name" => organisation.name, - "referential_name" => referential.name, - "start_date" => start_date, - "end_date" => end_date - } - } - end - - def data_format - "netex" - end - -end diff --git a/app/models/netex_import.rb b/app/models/netex_import.rb deleted file mode 100644 index b21af3408..000000000 --- a/app/models/netex_import.rb +++ /dev/null @@ -1,38 +0,0 @@ -require 'net/http' -class NetexImport < Import - before_destroy :destroy_non_ready_referential - - after_commit :launch_java_import, on: :create - before_save def abort_unless_referential - self.status = 'aborted' unless referential - end - - validates_presence_of :parent - - def launch_java_import - return if self.class.finished_statuses.include?(status) - threaded_call_boiv_iev - end - - private - - def destroy_non_ready_referential - if referential && !referential.ready - referential.destroy - end - end - - def threaded_call_boiv_iev - Thread.new(&method(:call_boiv_iev)) - end - - def call_boiv_iev - Rails.logger.error("Begin IEV call for import") - Net::HTTP.get(URI("#{Rails.configuration.iev_url}/boiv_iev/referentials/importer/new?id=#{id}")) - Rails.logger.error("End IEV call for import") - rescue Exception => e - logger.error "IEV server error : #{e.message}" - logger.error e.backtrace.inspect - end - -end diff --git a/app/models/organisation.rb b/app/models/organisation.rb index e8fb4e060..5742c81e8 100644 --- a/app/models/organisation.rb +++ b/app/models/organisation.rb @@ -1,5 +1,5 @@ # coding: utf-8 -class Organisation < ActiveRecord::Base +class Organisation < ApplicationModel include DataFormatEnumerations has_many :users, :dependent => :destroy @@ -13,6 +13,8 @@ class Organisation < ActiveRecord::Base has_many :line_referentials, through: :line_referential_memberships has_many :workbenches + has_many :workgroups, through: :workbenches + has_many :calendars has_many :api_keys, class_name: 'Api::V1::ApiKey' @@ -84,4 +86,8 @@ class Organisation < ActiveRecord::Base workbenches.default end + def lines_scope + functional_scope = sso_attributes.try(:[], "functional_scope") + JSON.parse(functional_scope) if functional_scope + end end diff --git a/app/models/public_version.rb b/app/models/public_version.rb deleted file mode 100644 index 4dbf6ce27..000000000 --- a/app/models/public_version.rb +++ /dev/null @@ -1,4 +0,0 @@ -class PublicVersion < PaperTrail::Version - # custom behaviour, e.g: - self.table_name = :'public.versions' -end diff --git a/app/models/referential.rb b/app/models/referential.rb index 91a88d02d..78b719fab 100644 --- a/app/models/referential.rb +++ b/app/models/referential.rb @@ -1,5 +1,5 @@ # coding: utf-8 -class Referential < ActiveRecord::Base +class Referential < ApplicationModel include DataFormatEnumerations include ObjectidFormatterSupport @@ -78,31 +78,29 @@ class Referential < ActiveRecord::Base alias_method_chain :save, :table_lock_timeout - if Rails.env.development? - def self.force_register_models_with_checksum - paths = Rails.application.paths['app/models'].to_a - Rails.application.railties.each do |tie| - next unless tie.respond_to? :paths - paths += tie.paths['app/models'].to_a - end + def self.force_register_models_with_checksum + paths = Rails.application.paths['app/models'].to_a + Rails.application.railties.each do |tie| + next unless tie.respond_to? :paths + paths += tie.paths['app/models'].to_a + end - paths.each do |path| - next unless File.directory?(path) - Dir.chdir path do - Dir['**/*.rb'].each do |src| - next if src =~ /^concerns/ - # thanks for inconsistent naming ... - if src == "route_control/zdl_stop_area.rb" - RouteControl::ZDLStopArea - next - end - Rails.logger.info "Loading #{src}" - begin - src[0..-4].classify.safe_constantize - rescue => e - Rails.logger.info "Failed: #{e.message}" - nil - end + paths.each do |path| + next unless File.directory?(path) + Dir.chdir path do + Dir['**/*.rb'].each do |src| + next if src =~ /^concerns/ + # thanks for inconsistent naming ... + if src == "route_control/zdl_stop_area.rb" + RouteControl::ZDLStopArea + next + end + Rails.logger.info "Loading #{src}" + begin + src[0..-4].classify.safe_constantize + rescue => e + Rails.logger.info "Failed: #{e.message}" + nil end end end @@ -168,6 +166,10 @@ class Referential < ActiveRecord::Base Chouette::TimeTable.all end + def time_table_dates + Chouette::TimeTableDate.all + end + def timebands Chouette::Timeband.all end @@ -184,6 +186,10 @@ class Referential < ActiveRecord::Base Chouette::VehicleJourneyFrequency.all end + def vehicle_journey_at_stops + Chouette::VehicleJourneyAtStop.all + end + def routing_constraint_zones Chouette::RoutingConstraintZone.all end @@ -233,7 +239,7 @@ class Referential < ActiveRecord::Base end end - def self.new_from(from, functional_scope) + def self.new_from(from, organisation) Referential.new( name: I18n.t("activerecord.copy", name: from.name), slug: "#{from.slug}_clone", @@ -244,7 +250,7 @@ class Referential < ActiveRecord::Base stop_area_referential: from.stop_area_referential, created_from: from, objectid_format: from.objectid_format, - metadatas: from.metadatas.map { |m| ReferentialMetadata.new_from(m, functional_scope) } + metadatas: from.metadatas.map { |m| ReferentialMetadata.new_from(m, organisation) } ) end diff --git a/app/models/referential_cloning.rb b/app/models/referential_cloning.rb index d4b74bd52..f2c81009a 100644 --- a/app/models/referential_cloning.rb +++ b/app/models/referential_cloning.rb @@ -1,4 +1,4 @@ -class ReferentialCloning < ActiveRecord::Base +class ReferentialCloning < ApplicationModel include AASM belongs_to :source_referential, class_name: 'Referential' belongs_to :target_referential, class_name: 'Referential' diff --git a/app/models/referential_metadata.rb b/app/models/referential_metadata.rb index 393dc70d3..7a8a01774 100644 --- a/app/models/referential_metadata.rb +++ b/app/models/referential_metadata.rb @@ -1,7 +1,7 @@ require 'activeattr_ext.rb' require 'range_ext' -class ReferentialMetadata < ActiveRecord::Base +class ReferentialMetadata < ApplicationModel belongs_to :referential, touch: true belongs_to :referential_source, class_name: 'Referential' has_array_of :lines, class_name: 'Chouette::Line' @@ -155,10 +155,10 @@ class ReferentialMetadata < ActiveRecord::Base end private :clear_periods - def self.new_from(from, functional_scope) + def self.new_from(from, organisation) from.dup.tap do |metadata| metadata.referential_source_id = from.referential_id - metadata.line_ids = from.referential.lines.where(id: metadata.line_ids, objectid: functional_scope).collect(&:id) + metadata.line_ids = from.referential.lines.where(id: metadata.line_ids).for_organisation(organisation).pluck(:id) metadata.referential_id = nil end end diff --git a/app/models/referential_suite.rb b/app/models/referential_suite.rb index 4f825628c..f4a72f22c 100644 --- a/app/models/referential_suite.rb +++ b/app/models/referential_suite.rb @@ -1,4 +1,4 @@ -class ReferentialSuite < ActiveRecord::Base +class ReferentialSuite < ApplicationModel belongs_to :new, class_name: 'Referential' validate def validate_consistent_new return true if new_id.nil? || new.nil? diff --git a/app/models/simple_exporter.rb b/app/models/simple_exporter.rb new file mode 100644 index 000000000..1fcb76a29 --- /dev/null +++ b/app/models/simple_exporter.rb @@ -0,0 +1,182 @@ +# coding: utf-8 +class SimpleExporter < SimpleInterface + def export opts={} + configuration.validate! + + init_env opts + + @csv = nil + fail_with_error "Unable to write in file: #{self.filepath}" do + dir = Pathname.new(self.filepath).dirname + FileUtils.mkdir_p dir + @csv = CSV.open(self.filepath, 'w', self.configuration.csv_options) + end + + self.configuration.before_actions(:parsing).each do |action| action.call self end + + @statuses = "" + + process_collection + + self.status ||= :success + rescue SimpleInterface::FailedOperation + self.status = :failed + ensure + @csv&.close + task_finished + end + + def collection + @collection ||= begin + coll = configuration.collection + coll = coll.call() if coll.is_a?(Proc) + coll + end + end + + def encode_string s + s.encode("utf-8").force_encoding("utf-8") + end + + protected + def init_env opts + @number_of_lines = collection.size + super opts + end + + def process_collection + self.configuration.before_actions(:all).each do |action| action.call self end + log "Starting export ..." + log "Export will be written in #{filepath}" + @csv << self.configuration.columns.map(&:name) + if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id") + ids = collection.pluck :id + ids.in_groups_of(configuration.batch_size).each do |batch_ids| + collection.where(id: batch_ids).each do |item| + handle_item item + end + end + else + collection.each{|item| handle_item item } + end + print_state + end + + def map_item_to_rows item + return [item] unless configuration.item_to_rows_mapping + instance_exec(item, &configuration.item_to_rows_mapping).map {|row| row.is_a?(ActiveRecord::Base) ? row : CustomRow.new(row) } + end + + def resolve_value item, col + scoped_item = col.scope.inject(item){|tmp, scope| tmp.send(scope)} + val = col[:value] + if val.nil? || val.is_a?(Proc) + if val.is_a?(Proc) + val = instance_exec(scoped_item, &val) + else + attributes = [col.attribute].flatten + val = attributes.inject(scoped_item){|tmp, attr| tmp.send(attr) if tmp } + end + end + if val.nil? && !col.omit_nil? + push_in_journal({event: :attribute_not_found, message: "Value missing for: #{[col.scope, col.attribute].flatten.join('.')}", kind: :warning}) + self.status ||= :success_with_warnings + @new_status ||= colorize("✓", :orange) + end + + if val.nil? && col.required? + @new_status = colorize("x", :red) + raise "MISSING VALUE FOR COLUMN #{col.name}" + end + + val = encode_string(val) if val.is_a?(String) + val + end + + def handle_item item + number_of_lines = @number_of_lines + @current_item = item + map_item_to_rows(item).each_with_index do |item, i| + @number_of_lines = number_of_lines + i + @current_row = item.attributes + @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present? + row = [] + @new_status = nil + self.configuration.columns.each do |col| + val = resolve_value item, col + @new_status ||= colorize("✓", :green) + row << val + end + push_in_journal({event: :success, kind: :log}) + @statuses += @new_status + print_state if @current_line % 20 == 0 || i > 0 + @current_line += 1 + @csv << row + end + end + + class CustomRow < OpenStruct + def initialize data + super data + @data = data + end + + def attributes + flatten_hash @data + end + + protected + def flatten_hash h + h.each_with_object({}) do |(k, v), h| + if v.is_a? Hash + flatten_hash(v).map do |h_k, h_v| + h["#{k}.#{h_k}".to_sym] = h_v + end + elsif v.is_a? ActiveRecord::Base + flatten_hash(v.attributes).map do |h_k, h_v| + h["#{k}.#{h_k}".to_sym] = h_v + end + else + h[k] = v + end + end + end + end + + class Configuration < SimpleInterface::Configuration + attr_accessor :collection + attr_accessor :batch_size + attr_accessor :logged_attributes + attr_accessor :item_to_rows_mapping + + def initialize import_name, opts={} + super import_name, opts + @collection = opts[:collection] + @batch_size = opts[:batch_size] || 1000 + @logged_attributes = opts[:logged_attributes] + @item_to_rows_mapping = opts[:item_to_rows_mapping] + end + + def options + super.update({ + collection: collection, + batch_size: batch_size, + logged_attributes: logged_attributes, + item_to_rows_mapping: item_to_rows_mapping, + }) + end + + def map_item_to_rows &block + @item_to_rows_mapping = block + end + + def add_column name, opts={} + raise "Column already defined: #{name}" if @columns.any?{|c| c.name == name.to_s} + super name, opts + end + + def validate! + raise "Incomplete configuration, missing collection for #{@import_name}" if collection.nil? + end + end +end diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb index d6ba64494..4cfe90cff 100644 --- a/app/models/simple_importer.rb +++ b/app/models/simple_importer.rb @@ -1,38 +1,5 @@ -class SimpleImporter < ActiveRecord::Base - attr_accessor :configuration - - def self.define name - @importers ||= {} - configuration = Configuration.new name - yield configuration - configuration.validate! - @importers[name.to_sym] = configuration - end - - def self.find_configuration name - @importers ||= {} - configuration = @importers[name.to_sym] - raise "Importer not found: #{name}" unless configuration - configuration - end - - def initialize *args - super *args - self.configuration = self.class.find_configuration self.configuration_name - self.journal ||= [] - end - - def configure - new_config = configuration.duplicate - yield new_config - new_config.validate! - self.configuration = new_config - end - - def context - self.configuration.context - end - +# coding: utf-8 +class SimpleImporter < SimpleInterface def resolve col_name, value, &block val = block.call(value) return val if val.present? @@ -41,20 +8,15 @@ class SimpleImporter < ActiveRecord::Base end def import opts={} - @verbose = opts.delete :verbose - + configuration.validate! - @resolution_queue = Hash.new{|h,k| h[k] = []} - @errors = [] - @messages = [] - @number_of_lines = 0 - @padding = 1 - @current_line = 0 fail_with_error "File not found: #{self.filepath}" do @number_of_lines = CSV.read(self.filepath, self.configuration.csv_options).length - @padding = [1, Math.log(@number_of_lines, 10).ceil()].max end + init_env opts + + @resolution_queue = Hash.new{|h,k| h[k] = []} self.configuration.before_actions(:parsing).each do |action| action.call self end @@ -68,34 +30,19 @@ class SimpleImporter < ActiveRecord::Base end end self.status ||= :success - rescue FailedImport + rescue SimpleInterface::FailedOperation self.status = :failed ensure - self.save! - end - - def fail_with_error msg=nil, opts={} - begin - yield - rescue => e - msg = msg.call if msg.is_a?(Proc) - custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures - push_in_journal({message: msg, error: e.message, event: :error, kind: :error}) - @new_status = colorize("x", :red) - if self.configuration.ignore_failures - raise FailedRow if opts[:abort_row] - else - raise FailedImport - end - end + task_finished end def encode_string s + return if s.nil? s.encode("utf-8").force_encoding("utf-8") end def dump_csv_from_context - dir = context[:output_dir] || "log/importers" + dir = context[:logs_output_dir] || "log/importers" filepath = File.join dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv" # for some reason, context[:csv].to_csv does not work CSV.open(filepath, 'w') do |csv| @@ -109,16 +56,6 @@ class SimpleImporter < ActiveRecord::Base log "CSV file dumped in #{filepath}" end - def log msg, opts={} - msg = colorize msg, opts[:color] if opts[:color] - if opts[:append] - @messages[-1] = (@messages[-1] || "") + msg - else - @messages << msg - end - print_state - end - protected def process_csv_file @@ -144,6 +81,7 @@ class SimpleImporter < ActiveRecord::Base if self.configuration.ignore_failures unless @current_record.save @new_status = colorize("x", :red) + self.status = :success_with_errors push_in_journal({message: "errors: #{@current_record.errors.messages}", error: "invalid record", event: :error, kind: :error}) end else @@ -154,23 +92,23 @@ class SimpleImporter < ActiveRecord::Base action.call self, @current_record end end - rescue FailedRow + rescue SimpleInterface::FailedRow @new_status = colorize("x", :red) end push_in_journal({event: @event, kind: :log}) if @current_record&.valid? @statuses += @new_status self.configuration.columns.each do |col| - if @current_record && col.name && @resolution_queue.any? - val = @current_record.send col[:attribute] - (@resolution_queue.delete([col.name, val]) || []).each do |res| - record = res[:record] - attribute = res[:attribute] - value = res[:block].call(val, record) - record.send "#{attribute}=", value - record.save! + if @current_record && col.name && @resolution_queue.any? + val = @current_record.send col[:attribute] + (@resolution_queue.delete([col.name, val]) || []).each do |res| + record = res[:record] + attribute = res[:attribute] + value = res[:block].call(val, record) + record.send "#{attribute}=", value + record.save! + end end end - end print_state @current_line += 1 end @@ -179,7 +117,7 @@ class SimpleImporter < ActiveRecord::Base self.configuration.after_actions(:all).each do |action| action.call self end - rescue FailedRow + rescue SimpleInterface::FailedRow end end @@ -215,211 +153,20 @@ class SimpleImporter < ActiveRecord::Base end end - def push_in_journal data - line = @current_line + 1 - line += 1 if configuration.headers - self.journal.push data.update(line: line, row: @current_row) - if data[:kind] == :error || data[:kind] == :warning - @errors.push data - end - end - - def colorize txt, color - color = { - red: "31", - green: "32", - orange: "33", - }[color] || "33" - "\e[#{color}m#{txt}\e[0m" - end - - def print_state - return unless @verbose - - @status_width ||= begin - term_width = %x(tput cols).to_i - term_width - @padding - 10 - rescue - 100 - end - - @status_height ||= begin - term_height = %x(tput lines).to_i - term_height - 3 - rescue - 50 - end - - full_status = @statuses || "" - full_status = full_status.last(@status_width*10) || "" - padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min - full_status = "#{full_status}#{"."*[padding_size, 0].max}" - - msg = "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}" - - lines_count = [(@status_height / 2) - 3, 1].max - - if @messages.any? - msg += "\n\n" - msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green - msg += "[...]\n" if @messages.count > lines_count - msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n") - msg += "\n"*[lines_count-@messages.count, 0].max - end - - if @errors.any? - msg += "\n\n" - msg += colorize "=== ERRORS (#{@errors.count}) ===\n", :red - msg += "[...]\n" if @errors.count > lines_count - msg += @errors.last(lines_count).map do |j| - kind = j[:kind] - kind = colorize(kind, kind == :error ? :red : :orange) - kind = "[#{kind}]" - kind += " "*(25 - kind.size) - encode_string("#{kind}L#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}").truncate(@status_width) - end.join("\n") - end - custom_print msg, clear: true - end - - def custom_print msg, opts={} - return unless @verbose - out = "" - msg = colorize(msg, opts[:color]) if opts[:color] - puts "\e[H\e[2J" if opts[:clear] - out += msg - print out - end - - class FailedImport < RuntimeError - end - - class FailedRow < RuntimeError - end - - class Configuration - attr_accessor :model, :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope - attr_reader :columns + class Configuration < SimpleInterface::Configuration + attr_accessor :model def initialize import_name, opts={} - @import_name = import_name - @key = opts[:key] || "id" - @headers = opts.has_key?(:headers) ? opts[:headers] : true - @separator = opts[:separator] || "," - @encoding = opts[:encoding] - @columns = opts[:columns] || [] + super import_name, opts @model = opts[:model] - @custom_handler = opts[:custom_handler] - @before = opts[:before] - @after = opts[:after] - @ignore_failures = opts[:ignore_failures] - @context = opts[:context] || {} - @scope = opts[:scope] - end - - def duplicate - Configuration.new @import_name, self.options end def options - { - key: @key, - headers: @headers, - separator: @separator, - encoding: @encoding, - columns: @columns.map(&:duplicate), - model: model, - custom_handler: @custom_handler, - before: @before, - after: @after, - ignore_failures: @ignore_failures, - context: @context, - scope: @scope - } + super.update({model: model}) end def validate! raise "Incomplete configuration, missing model for #{@import_name}" unless model.present? end - - def attribute_for_col col_name - column = self.columns.find{|c| c.name == col_name} - column && column[:attribute] || col_name - end - - def record_scope - _scope = @scope - _scope = instance_exec(&_scope) if _scope.is_a?(Proc) - _scope || model - end - - def find_record attrs - record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s]) - end - - def csv_options - { - headers: self.headers, - col_sep: self.separator, - encoding: self.encoding - } - end - - def add_column name, opts={} - @columns.push Column.new({name: name.to_s}.update(opts)) - end - - def add_value attribute, value - @columns.push Column.new({attribute: attribute, value: value}) - end - - def before group=:all, &block - @before ||= Hash.new{|h, k| h[k] = []} - @before[group].push block - end - - def after group=:all, &block - @after ||= Hash.new{|h, k| h[k] = []} - @after[group].push block - end - - def before_actions group=:all - @before ||= Hash.new{|h, k| h[k] = []} - @before[group] - end - - def after_actions group=:all - @after ||= Hash.new{|h, k| h[k] = []} - @after[group] - end - - def custom_handler &block - @custom_handler = block - end - - def get_custom_handler - @custom_handler - end - - class Column - attr_accessor :name - def initialize opts={} - @name = opts[:name] - @options = opts - @options[:attribute] ||= @name - end - - def duplicate - Column.new @options.dup - end - - def required? - !!@options[:required] - end - - def [](key) - @options[key] - end - end end end diff --git a/app/models/simple_interface.rb b/app/models/simple_interface.rb new file mode 100644 index 000000000..7b04a07df --- /dev/null +++ b/app/models/simple_interface.rb @@ -0,0 +1,372 @@ +class SimpleInterface < ApplicationModel + attr_accessor :configuration, :interfaces_group + + class << self + def configuration_class + "#{self.name}::Configuration".constantize + end + + def define name + @importers ||= {} + configuration = configuration_class.new name + yield configuration if block_given? + @importers[name.to_sym] = configuration + end + + def find_configuration name + @importers ||= {} + configuration = @importers[name.to_sym] + raise "#{self.name} not found: #{name}" unless configuration + configuration + end + end + + def initialize *args + super *args + self.configuration = self.class.find_configuration self.configuration_name + self.journal ||= [] + end + + def configuration + @configuration ||= self.class.find_configuration self.configuration_name + end + + def init_env opts + @verbose = opts.delete :verbose + + @_errors = [] + @messages = [] + @padding = 1 + @current_line = -1 + @number_of_lines ||= 1 + @padding = [1, Math.log([@number_of_lines, 1].max, 10).ceil()].max + @output_dir = opts[:output_dir] || Rails.root.join('tmp', self.class.name.tableize) + @start_time = Time.now + end + + def configure + new_config = configuration.duplicate + yield new_config + self.configuration = new_config + end + + def context + self.configuration.context + end + + def fail_with_error msg=nil, opts={} + begin + yield + rescue => e + msg = msg.call if msg.is_a?(Proc) + custom_print "\nFAILED: \n errors: #{msg}\n exception: #{e.message}\n#{e.backtrace.join("\n")}", color: :red unless self.configuration.ignore_failures + push_in_journal({message: msg, error: e.message, event: :error, kind: :error}) + @new_status = colorize("x", :red) + self.status = :success_with_errors + if self.configuration.ignore_failures + raise SimpleInterface::FailedRow if opts[:abort_row] + else + raise FailedOperation + end + end + end + + def log msg, opts={} + msg = msg.to_s + msg = colorize msg, opts[:color] if opts[:color] + @start_time ||= Time.now + time = Time.now - @start_time + @messages ||= [] + if opts[:append] + _time, _msg = @messages.pop || [] + _time ||= time + _msg ||= "" + @messages.push [_time, _msg+msg] + elsif opts[:replace] + @messages.pop + @messages << [time, msg] + else + @messages << [time, msg] + end + print_state true + end + + def output_filepath + @output_filepath ||= File.join @output_dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}_out.csv" + end + + def write_output_to_csv + cols = %i(line kind event message error) + if self.journal.size > 0 && self.journal.first[:row].present? + log "Writing output log" + FileUtils.mkdir_p @output_dir + keys = self.journal.first[:row].map(&:first) + CSV.open(output_filepath, "w") do |csv| + csv << cols + keys + self.journal.each do |j| + csv << cols.map{|c| j[c]} + j[:row].map(&:last) + end + end + log "Output written in #{output_filepath}", replace: true + end + end + + protected + + def task_finished + log "Saving..." + self.save! + log "Saved", replace: true + write_output_to_csv + log "FINISHED, status: " + log status, color: SimpleInterface.status_color(status), append: true + print_state true + end + + def push_in_journal data + line = (@current_line || 0) + 1 + line += 1 if configuration.headers + @_errors ||= [] + self.journal.push data.update(line: line, row: @current_row) + if data[:kind] == :error || data[:kind] == :warning + @_errors.push data + end + end + + def self.colorize txt, color + color = { + red: "31", + green: "32", + orange: "33", + }[color] || "33" + "\e[#{color}m#{txt}\e[0m" + end + + def self.status_color status + color = :green + color = :orange if status.to_s == "success_with_warnings" + color = :red if status.to_s == "success_with_errors" + color = :red if status.to_s == "error" + color + end + + def colorize txt, color + SimpleInterface.colorize txt, color + end + + def print_state force=false + return unless @verbose + return if !@last_repaint.nil? && (Time.now - @last_repaint < 0.1) && !force + + @status_width ||= begin + @term_width = %x(tput cols).to_i + @term_width - @padding - 10 + rescue + 100 + end + + @status_height ||= begin + term_height = %x(tput lines).to_i + term_height - 3 + rescue + 50 + end + + msg = "" + + if @banner.nil? && interfaces_group.present? + @banner = interfaces_group.banner @status_width + @status_height -= @banner.lines.count + 2 + end + + if @banner.present? + msg += @banner + msg += "\n" + "-"*@term_width + "\n" + end + + full_status = @statuses || "" + full_status = full_status.last(@status_width*10) || "" + padding_size = [(@number_of_lines - @current_line - 1), (@status_width - full_status.size/10)].min + full_status = "#{full_status}#{"."*[padding_size, 0].max}" + + msg += "#{"%#{@padding}d" % (@current_line + 1)}/#{@number_of_lines}: #{full_status}" + + lines_count = [(@status_height / 2) - 3, 1].max + + if @messages.any? + msg += "\n\n" + msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green + msg += "[...]\n" if @messages.count > lines_count + msg += @messages.last(lines_count).map do |m| + "[#{"%.5f" % m[0]}]\t" + m[1].truncate(@status_width - 10) + end.join("\n") + msg += "\n"*[lines_count-@messages.count, 0].max + end + + if @_errors.any? + msg += "\n\n" + msg += colorize "=== ERRORS (#{@_errors.count}) ===\n", :red + msg += "[...]\n" if @_errors.count > lines_count + msg += @_errors.last(lines_count).map do |j| + kind = j[:kind] + kind = colorize(kind, kind == :error ? :red : :orange) + kind = "[#{kind}]" + kind += " "*(25 - kind.size) + encode_string("#{kind}L#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}").truncate(@status_width) + end.join("\n") + end + custom_print msg, clear: true + @last_repaint = Time.now + end + + def custom_print msg, opts={} + return unless @verbose + out = "" + msg = colorize(msg, opts[:color]) if opts[:color] + puts "\e[H\e[2J" if opts[:clear] + out += msg + print out + end + + class FailedRow < RuntimeError + end + + class FailedOperation < RuntimeError + end + + class Configuration + attr_accessor :headers, :separator, :key, :context, :encoding, :ignore_failures, :scope + attr_reader :columns + + def initialize import_name, opts={} + @import_name = import_name + @key = opts[:key] || "id" + @headers = opts.has_key?(:headers) ? opts[:headers] : true + @separator = opts[:separator] || "," + @encoding = opts[:encoding] + @columns = opts[:columns] || [] + @custom_handler = opts[:custom_handler] + @before = opts[:before] + @after = opts[:after] + @ignore_failures = opts[:ignore_failures] + @context = opts[:context] || {} + @scope = opts[:scope] + end + + def on_relation relation_name + @current_scope ||= [] + @current_scope.push relation_name + yield + @current_scope.pop + end + + def duplicate + self.class.new @import_name, self.options + end + + def options + { + key: @key, + headers: @headers, + separator: @separator, + encoding: @encoding, + columns: @columns.map(&:duplicate), + custom_handler: @custom_handler, + before: @before, + after: @after, + ignore_failures: @ignore_failures, + context: @context, + scope: @scope + } + end + + def attribute_for_col col_name + column = self.columns.find{|c| c.name == col_name} + column && column[:attribute] || col_name + end + + def record_scope + _scope = @scope + _scope = instance_exec(&_scope) if _scope.is_a?(Proc) + _scope || model + end + + def find_record attrs + record_scope.find_or_initialize_by(attribute_for_col(@key) => attrs[@key.to_s]) + end + + def csv_options + { + headers: self.headers, + col_sep: self.separator, + encoding: self.encoding + } + end + + def add_column name, opts={} + @current_scope ||= [] + @columns.push Column.new({name: name.to_s, scope: @current_scope.dup}.update(opts)) + end + + def add_value attribute, value + @columns.push Column.new({attribute: attribute, value: value}) + end + + def before group=:all, &block + @before ||= Hash.new{|h, k| h[k] = []} + @before[group].push block + end + + def after group=:all, &block + @after ||= Hash.new{|h, k| h[k] = []} + @after[group].push block + end + + def before_actions group=:all + @before ||= Hash.new{|h, k| h[k] = []} + @before[group] + end + + def after_actions group=:all + @after ||= Hash.new{|h, k| h[k] = []} + @after[group] + end + + def custom_handler &block + @custom_handler = block + end + + def get_custom_handler + @custom_handler + end + + class Column + attr_accessor :name, :attribute + def initialize opts={} + @name = opts[:name] + @options = opts + @attribute = @options[:attribute] ||= @name + end + + def duplicate + Column.new @options.dup + end + + def required? + !!@options[:required] + end + + def omit_nil? + !!@options[:omit_nil] + end + + def scope + @options[:scope] || [] + end + + def [](key) + @options[key] + end + end + end +end diff --git a/app/models/simple_interfaces_group.rb b/app/models/simple_interfaces_group.rb new file mode 100644 index 000000000..808be6570 --- /dev/null +++ b/app/models/simple_interfaces_group.rb @@ -0,0 +1,76 @@ +class SimpleInterfacesGroup + attr_accessor :name, :shared_options + + def initialize name + @name = name + @interfaces = [] + @current_step = 0 + end + + def add_interface interface, name, action, opts={} + @interfaces.push({interface: interface, name: name, action: action, opts: opts}) + end + + def run + @interfaces.each do |interface_def| + interface = interface_def[:interface] + interface.interfaces_group = self + interface.send interface_def[:action], interface_def[:opts].reverse_update(shared_options || {}) + return if interface.status == :error + @current_step += 1 + end + + print_summary + end + + def banner width=nil + width ||= @width + width ||= 128 + @width = width + + name = "### #{self.name} ###" + centered_name = " " * ([width - name.size, 0].max / 2) + name + banner = [centered_name, ""] + banner << @interfaces.each_with_index.map do |interface, i| + if interface[:interface].status.present? + SimpleInterface.colorize interface[:name], SimpleInterface.status_color(interface[:interface].status) + elsif i == @current_step + "☕︎ #{interface[:name]}" + else + interface[:name] + end + end.join(' > ') + banner.join("\n") + end + + def print_summary + puts "\e[H\e[2J" + out = [banner] + out << "-" * @width + out << "" + out << SimpleInterface.colorize("=== STATUSES ===", :green) + out << "" + @interfaces.each do |i| + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{SimpleInterface.colorize i[:interface].status, SimpleInterface.status_color(i[:interface].status)}" + end + out << "" + out << SimpleInterface.colorize("=== OUTPUTS ===", :green) + out << "" + @interfaces.each do |i| + if i[:interface].is_a? SimpleExporter + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].filepath}" + end + end + out << "" + out << "" + out << SimpleInterface.colorize("=== DEBUG OUTPUTS ===", :green) + out << "" + @interfaces.each do |i| + out << "#{i[:name].rjust(@interfaces.map{|i| i[:name].size}.max)}:\t#{i[:interface].output_filepath}" + end + out << "" + out << "" + + print out.join("\n") + end +end diff --git a/app/models/simple_json_exporter.rb b/app/models/simple_json_exporter.rb new file mode 100644 index 000000000..024d75c97 --- /dev/null +++ b/app/models/simple_json_exporter.rb @@ -0,0 +1,183 @@ +class SimpleJsonExporter < SimpleExporter + + def export opts={} + configuration.validate! + + init_env opts + + if self.configuration.root + @out = {self.configuration.root => []} + else + @out = [] + end + + fail_with_error "Unable to write in file: #{self.filepath}" do + dir = Pathname.new(self.filepath).dirname + FileUtils.mkdir_p dir + @file = File.open(self.filepath, 'w', self.configuration.file_options) + end + + self.configuration.before_actions(:parsing).each do |action| action.call self end + + @statuses = "" + + process_collection + self.status ||= :success + rescue SimpleInterface::FailedOperation + self.status = :failed + ensure + if @file + log "Writing to JSON file..." + @file.write @out.to_json + log "JSON file written", replace: true + @file.close + end + task_finished + end + + protected + def root + if self.configuration.root + @out[self.configuration.root] + else + @out + end + end + + def process_collection + self.configuration.before_actions(:all).each do |action| action.call self end + log "Starting export ..." + log "Export will be written in #{filepath}" + + if collection.is_a?(ActiveRecord::Relation) && collection.model.column_names.include?("id") + log "Using paginated collection", color: :green + ids = collection.pluck :id + ids.in_groups_of(configuration.batch_size).each do |batch_ids| + collection.where(id: batch_ids.compact).each do |item| + handle_item item + end + end + else + log "Using non-paginated collection", color: :orange + collection.each{|item| handle_item item } + end + print_state true + end + + def resolve_node item, node + vals = [] + scoped_item = node.scope.inject(item){|tmp, scope| tmp.send(scope)} + + [scoped_item.send(node.attribute)].flatten.each do |node_item| + item_val = {} + apply_configuration node_item, node.configuration, item_val + vals.push item_val + end + node.multiple ? vals : vals.first + end + + def apply_configuration item, configuration, output + configuration.columns.each do |col| + val = resolve_value item, col + output[col.name] = val unless val.nil? && col.omit_nil? + end + + configuration.nodes.each do |node| + val = resolve_node item, node + output[node.name] = val + end + end + + def handle_item item + number_of_lines = @number_of_lines + @current_item = item + map_item_to_rows(item).each_with_index do |item, i| + @number_of_lines = number_of_lines + i + serialized_item = {} + @current_row = item.attributes.symbolize_keys + @current_row = @current_row.slice(*configuration.logged_attributes) if configuration.logged_attributes.present? + @new_status = nil + + apply_configuration item, self.configuration, serialized_item + + @new_status ||= colorize("✓", :green) + + push_in_journal({event: :success, kind: :log}) + @statuses += @new_status + print_state if @current_line % 20 == 0 || i > 0 + @current_line += 1 + append_item serialized_item + end + end + + def append_item serialized_item + root.push serialized_item + end + + class Configuration < SimpleExporter::Configuration + attr_reader :nodes + attr_accessor :root + + alias_method :add_field, :add_column + + def initialize import_name, opts={} + super import_name, opts + @collection = opts[:collection] + @nodes = opts[:nodes] || [] + @root = opts[:root] + end + + def options + super.update({ + nodes: @nodes, + root: @root, + }) + end + + def add_node name, opts={} + @nodes ||= [] + @current_scope ||= [] + node = Node.new({name: name.to_s, scope: @current_scope.dup}.update(opts)) + yield node.configuration + @nodes.push node + end + + def add_nodes name, opts={}, &block + self.add_node name, opts.update({multiple: true}), &block + end + + def file_options + { + encoding: self.encoding + } + end + end + + class NodeConfiguration < Configuration + def initialize node + super + end + end + + class Node + attr_accessor :name, :configuration + + def initialize opts={} + @name = opts[:name] + @options = opts + @configuration = NodeConfiguration.new self + end + + def attribute + @options[:attribute] || name + end + + def multiple + !!@options[:multiple] + end + + def scope + @options[:scope] || [] + end + end +end diff --git a/app/models/stop_area_referential.rb b/app/models/stop_area_referential.rb index 54e895cd0..6c339547c 100644 --- a/app/models/stop_area_referential.rb +++ b/app/models/stop_area_referential.rb @@ -1,4 +1,6 @@ -class StopAreaReferential < ActiveRecord::Base +class StopAreaReferential < ApplicationModel + validates :registration_number_format, format: { with: /\AX*\z/ } + include ObjectidFormatterSupport has_many :stop_area_referential_memberships has_many :organisations, through: :stop_area_referential_memberships @@ -6,13 +8,40 @@ class StopAreaReferential < ActiveRecord::Base has_many :stop_areas, class_name: 'Chouette::StopArea' has_many :stop_area_referential_syncs, -> {order created_at: :desc} has_many :workbenches + has_one :workgroup def add_member(organisation, options = {}) attributes = options.merge organisation: organisation - stop_area_referential_memberships.build attributes + stop_area_referential_memberships.build attributes unless organisations.include?(organisation) end def last_sync stop_area_referential_syncs.last end + + def generate_registration_number + return "" unless registration_number_format.present? + last = self.stop_areas.order("registration_number DESC NULLS LAST").limit(1).first&.registration_number + if self.stop_areas.count == 26**self.registration_number_format.size + raise "NO MORE AVAILABLE VALUES FOR registration_number in referential #{self.name}" + end + + return "A" * self.registration_number_format.size unless last + + if last == "Z" * self.registration_number_format.size + val = "A" * self.registration_number_format.size + while self.stop_areas.where(registration_number: val).exists? + val = val.next + end + val + else + last.next + end + end + + def validates_registration_number value + return false unless value.size == registration_number_format.size + return false unless value =~ /^[A-Z]*$/ + true + end end diff --git a/app/models/stop_area_referential_membership.rb b/app/models/stop_area_referential_membership.rb index 435970961..d507bc50e 100644 --- a/app/models/stop_area_referential_membership.rb +++ b/app/models/stop_area_referential_membership.rb @@ -1,4 +1,6 @@ -class StopAreaReferentialMembership < ActiveRecord::Base +class StopAreaReferentialMembership < ApplicationModel belongs_to :organisation belongs_to :stop_area_referential + + validates :organisation_id, presence: true, uniqueness: { scope: :stop_area_referential } end diff --git a/app/models/stop_area_referential_sync.rb b/app/models/stop_area_referential_sync.rb index e6cf2ecbc..8b48d35e6 100644 --- a/app/models/stop_area_referential_sync.rb +++ b/app/models/stop_area_referential_sync.rb @@ -1,4 +1,4 @@ -class StopAreaReferentialSync < ActiveRecord::Base +class StopAreaReferentialSync < ApplicationModel include AASM belongs_to :stop_area_referential has_many :stop_area_referential_sync_messages, :dependent => :destroy diff --git a/app/models/stop_area_referential_sync_message.rb b/app/models/stop_area_referential_sync_message.rb index cd2e62405..642ccfc38 100644 --- a/app/models/stop_area_referential_sync_message.rb +++ b/app/models/stop_area_referential_sync_message.rb @@ -1,4 +1,4 @@ -class StopAreaReferentialSyncMessage < ActiveRecord::Base +class StopAreaReferentialSyncMessage < ApplicationModel belongs_to :stop_area_referential_sync enum criticity: [:info, :warning, :error] diff --git a/app/models/user.rb b/app/models/user.rb index 31e634415..ba166b06f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,4 +1,4 @@ -class User < ActiveRecord::Base +class User < ApplicationModel # Include default devise modules. Others available are: # :token_authenticatable, :encryptable, :confirmable, :lockable, :timeoutable and :omniauthable, :database_authenticatable @@ -9,7 +9,7 @@ class User < ActiveRecord::Base :recoverable, :rememberable, :trackable, :async, authentication_type # FIXME https://github.com/nbudin/devise_cas_authenticatable/issues/53 - # Work around :validatable, when database_authenticatable is diabled. + # Work around :validatable, when database_authenticatable is disabled. attr_accessor :password unless authentication_type == :database_authenticatable # Setup accessible (or protected) attributes for your model @@ -30,6 +30,8 @@ class User < ActiveRecord::Base scope :with_organisation, -> { where.not(organisation_id: nil) } + scope :from_workgroup, ->(workgroup_id) { joins(:workbenches).where(workbenches: {workgroup_id: workgroup_id}) } + # Callback invoked by DeviseCasAuthenticable::Model#authernticate_with_cas_ticket def cas_extra_attributes=(extra_attributes) @@ -67,6 +69,10 @@ class User < ActiveRecord::Base permissions && permissions.include?(permission) end + def can_monitor_sidekiq? + has_permission?("sidekiq.monitor") + end + private # remove organisation and referentials if last user of it diff --git a/app/models/vehicle_journey_control/speed.rb b/app/models/vehicle_journey_control/speed.rb index e5e331b50..c9775e7a3 100644 --- a/app/models/vehicle_journey_control/speed.rb +++ b/app/models/vehicle_journey_control/speed.rb @@ -2,8 +2,6 @@ module VehicleJourneyControl class Speed < ComplianceControl store_accessor :control_attributes, :minimum, :maximum - validates_numericality_of :minimum, allow_nil: true, greater_than_or_equal_to: 0 - validates_numericality_of :maximum, allow_nil: true, greater_than_or_equal_to: 0 include MinMaxValuesValidation def self.default_code; "3-VehicleJourney-2" end diff --git a/app/models/vehicle_journey_import.rb b/app/models/vehicle_journey_import.rb index 250f3a9e9..2eb723c29 100644 --- a/app/models/vehicle_journey_import.rb +++ b/app/models/vehicle_journey_import.rb @@ -1,7 +1,7 @@ class VehicleJourneyImport - include ActiveModel::Validations - include ActiveModel::Conversion - extend ActiveModel::Naming + include ActiveModel::Model + + extend EnhancedModelI18n attr_accessor :file, :route attr_accessor :created_vehicle_journey_count,:updated_vehicle_journey_count,:deleted_vehicle_journey_count diff --git a/app/models/workbench.rb b/app/models/workbench.rb index eb53af7aa..ef0b2eaa4 100644 --- a/app/models/workbench.rb +++ b/app/models/workbench.rb @@ -1,4 +1,4 @@ -class Workbench < ActiveRecord::Base +class Workbench < ApplicationModel DEFAULT_WORKBENCH_NAME = "Gestion de l'offre" include ObjectidFormatterSupport @@ -13,8 +13,9 @@ class Workbench < ActiveRecord::Base has_many :companies, through: :line_referential has_many :group_of_lines, through: :line_referential has_many :stop_areas, through: :stop_area_referential - has_many :imports - has_many :workbench_imports + has_many :imports, class_name: Import::Base + has_many :exports, class_name: Export::Base + has_many :workbench_imports, class_name: Import::Workbench has_many :compliance_check_sets has_many :compliance_control_sets has_many :merges @@ -42,6 +43,10 @@ class Workbench < ActiveRecord::Base end end + def calendars + workgroup.calendars.where('(organisation_id = ? OR shared = ?)', organisation.id, true) + end + def self.default self.last if self.count == 1 where(name: DEFAULT_WORKBENCH_NAME).last diff --git a/app/models/workbench_import.rb b/app/models/workbench_import.rb deleted file mode 100644 index 27f53a44f..000000000 --- a/app/models/workbench_import.rb +++ /dev/null @@ -1,7 +0,0 @@ -class WorkbenchImport < Import - after_commit :launch_worker, :on => :create - - def launch_worker - WorkbenchImportWorker.perform_async(id) - end -end diff --git a/app/models/workgroup.rb b/app/models/workgroup.rb index 3af20ae23..3e8409634 100644 --- a/app/models/workgroup.rb +++ b/app/models/workgroup.rb @@ -1,4 +1,4 @@ -class Workgroup < ActiveRecord::Base +class Workgroup < ApplicationModel belongs_to :line_referential belongs_to :stop_area_referential @@ -11,10 +11,16 @@ class Workgroup < ActiveRecord::Base validates_presence_of :line_referential_id validates_presence_of :stop_area_referential_id + validates_uniqueness_of :stop_area_referential_id + validates_uniqueness_of :line_referential_id has_many :custom_fields def custom_fields_definitions Hash[*custom_fields.map{|cf| [cf.code, cf]}.flatten] end + + def has_export? export_name + export_types.include? export_name + end end |