13 files changed, 218 insertions, 53 deletions
diff --git a/app/models/chouette/time_table.rb b/app/models/chouette/time_table.rb
index b76de852a..20d6e69ac 100644
--- a/app/models/chouette/time_table.rb
+++ b/app/models/chouette/time_table.rb
@@ -17,6 +17,10 @@ module Chouette
       (column_names + ['tag_search']) + _ransackers.keys
     end
 
+    ransacker :unaccented_comment, formatter: ->(val){ val.parameterize } do
+      Arel.sql('unaccent(comment)')
+    end
+
     has_and_belongs_to_many :vehicle_journeys, :class_name => 'Chouette::VehicleJourney'
 
     has_many :dates, -> {order(:date)}, inverse_of: :time_table, :validate => :true, :class_name => "Chouette::TimeTableDate", :dependent => :destroy
diff --git a/app/models/chouette/time_table_date.rb b/app/models/chouette/time_table_date.rb
index b3b2fd561..98d8fa765 100644
--- a/app/models/chouette/time_table_date.rb
+++ b/app/models/chouette/time_table_date.rb
@@ -20,4 +20,4 @@ module Chouette
       self.slice(*attrs).values
     end
   end
-end
-\ No newline at end of file
+end
diff --git a/app/models/chouette/vehicle_journey.rb b/app/models/chouette/vehicle_journey.rb
index 1a79db823..c5a6901d7 100644
--- a/app/models/chouette/vehicle_journey.rb
+++ b/app/models/chouette/vehicle_journey.rb
@@ -3,6 +3,7 @@ module Chouette
   class VehicleJourney < Chouette::TridentActiveRecord
     has_paper_trail
     include ChecksumSupport
+    include CustomFieldsSupport
     include VehicleJourneyRestrictions
     include ObjectidSupport
     include StifTransportModeEnumerations
@@ -52,10 +53,48 @@ module Chouette
       end
     }
 
+    scope :with_stop_area_id, ->(id){
+      if id.present?
+        joins(journey_pattern: :stop_points).where('stop_points.stop_area_id = ?', id)
+      else
+        all
+      end
+    }
+
+    scope :with_ordered_stop_area_ids, ->(first, second){
+      if first.present? && second.present?
+        joins(journey_pattern: :stop_points).
+          joins('INNER JOIN "journey_patterns" ON "journey_patterns"."id" = "vehicle_journeys"."journey_pattern_id" INNER JOIN "journey_patterns_stop_points" ON "journey_patterns_stop_points"."journey_pattern_id" = "journey_patterns"."id" INNER JOIN "stop_points" as "second_stop_points" ON "stop_points"."id" = "journey_patterns_stop_points"."stop_point_id"').
+          where('stop_points.stop_area_id = ?', first).
+          where('second_stop_points.stop_area_id = ? and stop_points.position < second_stop_points.position', second)
+      else
+        all
+      end
+    }
+
+    scope :starting_with, ->(id){
+      if id.present?
+        joins(journey_pattern: :stop_points).where('stop_points.position = 0 AND stop_points.stop_area_id = ?', id)
+      else
+        all
+      end
+    }
+
+    scope :ending_with, ->(id){
+      if id.present?
+        pattern_ids = all.select(:journey_pattern_id).uniq.map(&:journey_pattern_id)
+        pattern_ids = Chouette::JourneyPattern.where(id: pattern_ids).to_a.select{|jp| p "ici: #{jp.stop_points.order(:position).last.stop_area_id}" ; jp.stop_points.order(:position).last.stop_area_id == id.to_i}.map &:id
+        where(journey_pattern_id: pattern_ids)
+      else
+        all
+      end
+    }
+
+
     scope :in_purchase_window, ->(range){
       purchase_windows = Chouette::PurchaseWindow.overlap_dates(range)
       sql = purchase_windows.joins(:vehicle_journeys).select('vehicle_journeys.id').uniq.to_sql
-      where("id IN (#{sql})")
+      where("vehicle_journeys.id IN (#{sql})")
     }
 
     # We need this for the ransack object in the filters
@@ -302,21 +341,6 @@ module Chouette
       end
     end
 
-    def self.custom_fields
-      CustomField.where(resource_type: self.name.split("::").last)
-    end
-
-
-    def custom_fields
-      Hash[*self.class.custom_fields.map do |v|
-        [v.code, v.slice(:code, :name, :field_type, :options).update(value: custom_field_value(v.code))]
-      end.flatten]
-    end
-
-    def custom_field_value key
-      (custom_field_values || {})[key.to_s]
-    end
-
     def self.matrix(vehicle_journeys)
       Hash[*VehicleJourneyAtStop.where(vehicle_journey_id: vehicle_journeys.pluck(:id)).map do |vjas|
         [ "#{vjas.vehicle_journey_id}-#{vjas.stop_point_id}", vjas]
diff --git a/app/models/compliance_check.rb b/app/models/compliance_check.rb
index 55f2ae228..9d817e146 100644
--- a/app/models/compliance_check.rb
+++ b/app/models/compliance_check.rb
@@ -1,14 +1,23 @@
 class ComplianceCheck < ActiveRecord::Base
+  include ComplianceItemSupport
 
   self.inheritance_column = nil
 
   extend Enumerize
   belongs_to :compliance_check_set
   belongs_to :compliance_check_block
-  
+
   enumerize :criticity, in: %i(warning error), scope: true, default: :warning
   validates :criticity, presence: true
   validates :name, presence: true
   validates :code, presence: true
   validates :origin_code, presence: true
+
+  def control_class
+    compliance_control_name.present? ? compliance_control_name.constantize : nil
+  end
+
+  delegate :predicate, to: :control_class, allow_nil: true
+  delegate :prerequisite, to: :control_class, allow_nil: true
+
 end
diff --git a/app/models/compliance_check_message_export.rb b/app/models/compliance_check_message_export.rb
index 04e1a9caa..bbaaa8e3f 100644
--- a/app/models/compliance_check_message_export.rb
+++ b/app/models/compliance_check_message_export.rb
@@ -26,12 +26,14 @@ class ComplianceCheckMessageExport
   end
 
   def to_csv(options = {})
-    CSV.generate(options.slice(:col_sep, :quote_char, :force_quotes)) do |csv|
+    csv_string = CSV.generate(options.slice(:col_sep, :quote_char, :force_quotes)) do |csv|
       csv << column_names
       compliance_check_messages.each do |compliance_check_message|
         csv << [compliance_check_message.compliance_check.criticity, *compliance_check_message.message_attributes.values_at('test_id', 'source_objectid'), options[:server_url] + compliance_check_message.message_attributes['source_object_path'], I18n.t("compliance_check_messages.#{compliance_check_message.message_key}", compliance_check_message.message_attributes.deep_symbolize_keys)]
       end
     end
+    # We add a BOM to indicate we use UTF-8
+    "\uFEFF" + csv_string
   end
 
   def to_zip(temp_file,options = {})
diff --git a/app/models/compliance_control.rb b/app/models/compliance_control.rb
index 298a63ab9..537343005 100644
--- a/app/models/compliance_control.rb
+++ b/app/models/compliance_control.rb
@@ -1,18 +1,16 @@
 class ComplianceControl < ActiveRecord::Base
+  include ComplianceItemSupport
 
   class << self
     def criticities; %i(warning error) end
     def default_code; "" end
-    def dynamic_attributes
-      stored_attributes[:control_attributes] || []
-    end
 
     def policy_class
       ComplianceControlPolicy
     end
 
     def subclass_patterns
-      { 
+      {
         generic: 'Generic',
         journey_pattern: 'JourneyPattern',
         line: 'Line',
@@ -30,6 +28,9 @@ class ComplianceControl < ActiveRecord::Base
       end
       super
     end
+
+    def predicate; I18n.t("compliance_controls.#{self.name.underscore}.description") end
+    def prerequisite; I18n.t("compliance_controls.#{self.name.underscore}.prerequisite") end
   end
 
   extend Enumerize
@@ -45,26 +46,25 @@ class ComplianceControl < ActiveRecord::Base
   validates :compliance_control_set, presence: true
 
   validate def coherent_control_set
-  return true if compliance_control_block_id.nil?
-  ids = [compliance_control_block.compliance_control_set_id, compliance_control_set_id]
-  return true if ids.first == ids.last
-  names = ids.map{|id| ComplianceControlSet.find(id).name}
-  errors.add(:coherent_control_set,
-             I18n.t('compliance_controls.errors.incoherent_control_sets',
-                    indirect_set_name: names.first,
-                    direct_set_name: names.last))
-end
-
+    return true if compliance_control_block_id.nil?
+    ids = [compliance_control_block.compliance_control_set_id, compliance_control_set_id]
+    return true if ids.first == ids.last
+    names = ids.map{|id| ComplianceControlSet.find(id).name}
+    errors.add(:coherent_control_set,
+               I18n.t('compliance_controls.errors.incoherent_control_sets',
+                      indirect_set_name: names.first,
+                      direct_set_name: names.last))
+  end
 
-def initialize(attributes = {})
-  super
-  self.name ||= I18n.t("activerecord.models.#{self.class.name.underscore}.one")
-  self.code ||= self.class.default_code
-  self.origin_code ||= self.class.default_code
-end
+  def initialize(attributes = {})
+    super
+    self.name ||= I18n.t("activerecord.models.#{self.class.name.underscore}.one")
+    self.code ||= self.class.default_code
+    self.origin_code ||= self.class.default_code
+  end
 
-def predicate; I18n.t("compliance_controls.#{self.class.name.underscore}.description") end
-def prerequisite; I18n.t('compliance_controls.metas.no_prerequisite'); end
+  def predicate; self.class.predicate end
+  def prerequisite; self.class.prerequisite end
 
 end
 
diff --git a/app/models/concerns/compliance_item_support.rb b/app/models/concerns/compliance_item_support.rb
new file mode 100644
index 000000000..f44f5719f
--- /dev/null
+++ b/app/models/concerns/compliance_item_support.rb
@@ -0,0 +1,13 @@
+module ComplianceItemSupport
+  extend ActiveSupport::Concern
+  included do
+
+  end
+
+  module ClassMethods
+    def dynamic_attributes
+      stored_attributes[:control_attributes] || []
+    end
+  end
+
+end
diff --git a/app/models/concerns/custom_fields_support.rb b/app/models/concerns/custom_fields_support.rb
new file mode 100644
index 000000000..6c76bd653
--- /dev/null
+++ b/app/models/concerns/custom_fields_support.rb
@@ -0,0 +1,24 @@
+module CustomFieldsSupport
+  extend ActiveSupport::Concern
+
+  included do
+    validate :custom_fields_values_are_valid
+
+    def self.custom_fields
+      CustomField.where(resource_type: self.name.split("::").last)
+    end
+
+    def custom_fields
+      CustomField::Collection.new self
+    end
+
+    def custom_field_value key
+      (custom_field_values || {})[key.to_s]
+    end
+
+    private
+    def custom_fields_values_are_valid
+      custom_fields.values.all?{|cf| cf.valid?}
+    end
+  end
+end
diff --git a/app/models/custom_field.rb b/app/models/custom_field.rb
index 774c8b0f6..4a840744e 100644
--- a/app/models/custom_field.rb
+++ b/app/models/custom_field.rb
@@ -6,4 +6,80 @@ class CustomField < ActiveRecord::Base
 
   validates :name, uniqueness: {scope: [:resource_type, :workgroup_id]}
   validates :code, uniqueness: {scope: [:resource_type, :workgroup_id], case_sensitive: false}
+
+  class Collection < HashWithIndifferentAccess
+    def initialize object
+      vals = object.class.custom_fields.map do |v|
+        [v.code, CustomField::Value.new(object, v, object.custom_field_value(v.code))]
+      end
+      super Hash[*vals.flatten]
+    end
+
+    def to_hash
+      HashWithIndifferentAccess[*self.map{|k, v| [k, v.to_hash]}.flatten(1)]
+    end
+  end
+
+  class Value
+    def self.new owner, custom_field, value
+      field_type = custom_field.options["field_type"]
+      klass_name = field_type && "CustomField::Value::#{field_type.classify}"
+      klass = klass_name && const_defined?(klass_name) ? klass_name.constantize : CustomField::Value::Base
+      klass.new owner, custom_field, value
+    end
+
+    class Base
+      def initialize owner, custom_field, value
+        @custom_field = custom_field
+        @raw_value = value
+        @owner = owner
+        @errors = []
+        @validated = false
+        @valid = false
+      end
+
+      delegate :code, :name, :field_type, :options, to: :@custom_field
+
+      def validate
+        @valid = true
+      end
+
+      def valid?
+        validate unless @validated
+        @valid
+      end
+
+      def value
+        @raw_value
+      end
+
+      def errors_key
+        "custom_fields.#{code}"
+      end
+
+      def to_hash
+        HashWithIndifferentAccess[*%w(code name field_type options value).map{|k| [k, send(k)]}.flatten(1)]
+      end
+    end
+
+    class Integer < Base
+      def value
+        @raw_value.to_i
+      end
+
+      def validate
+        @valid = true
+        unless @raw_value =~ /\A\d*\Z/
+          @owner.errors.add errors_key, "'#{@raw_value}' is not a valid integer"
+          @valid = false
+        end
+      end
+    end
+
+    class String < Base
+      def value
+        "#{@raw_value}"
+      end
+    end
+  end
 end
diff --git a/app/models/import_message_export.rb b/app/models/import_message_export.rb
index 05f8a2cc7..991eb0f61 100644
--- a/app/models/import_message_export.rb
+++ b/app/models/import_message_export.rb
@@ -26,12 +26,14 @@ class ImportMessageExport
   end
 
   def to_csv(options = {})
-    CSV.generate(options) do |csv|
+    csv_string = CSV.generate(options) do |csv|
       csv << column_names
       import_messages.each do |import_message|
         csv << [import_message.criticity, import_message.message_key, I18n.t("import_messages.#{import_message.message_key}", import_message.message_attributes.deep_symbolize_keys), *import_message.resource_attributes.values_at("filename", "line_number", "column_number")  ]
       end
     end
+    # We add a BOM to indicate we use UTF-8
+    "\uFEFF" + csv_string
   end
 
   def to_zip(temp_file,options = {})
diff --git a/app/models/merge.rb b/app/models/merge.rb
index 62bf581d6..d42d882ac 100644
--- a/app/models/merge.rb
+++ b/app/models/merge.rb
@@ -152,7 +152,7 @@ class Merge < ActiveRecord::Base
           route_stop_points = referential_stop_points_by_route[route.id]
 
           # Stop Points
-          route_stop_points.each do |stop_point|
+          route_stop_points.sort_by(&:position).each do |stop_point|
             objectid = Chouette::StopPoint.where(objectid: stop_point.objectid).exists? ? nil : stop_point.objectid
             attributes = stop_point.attributes.merge(
               id: nil,
@@ -166,7 +166,7 @@ class Merge < ActiveRecord::Base
           new_route.save!
 
           if new_route.checksum != route.checksum
-            raise "Checksum has changed: #{route.inspect} #{new_route.inspect}"
+            raise "Checksum has changed: \"#{route.checksum}\", \"#{route.checksum_source}\" -> \"#{new_route.checksum}\", \"#{new_route.checksum_source}\""
           end
         end
       end
@@ -221,7 +221,7 @@ class Merge < ActiveRecord::Base
 
           new_journey_pattern = new.journey_patterns.create! attributes
           if new_journey_pattern.checksum != journey_pattern.checksum
-            raise "Checksum has changed for #{journey_pattern.inspect}: #{journey_pattern.checksum_source} #{new_journey_pattern.checksum_source} "
+            raise "Checksum has changed for #{journey_pattern.inspect}: \"#{journey_pattern.checksum_source}\" -> \"#{new_journey_pattern.checksum_source}\""
           end
         end
       end
diff --git a/app/models/referential.rb b/app/models/referential.rb
index 09c2e7d34..91a88d02d 100644
--- a/app/models/referential.rb
+++ b/app/models/referential.rb
@@ -408,6 +408,7 @@ class Referential < ActiveRecord::Base
       end
 
       check_migration_count(report)
+      # raise "Wrong migration count: #{migration_count}" if migration_count < 300
     end
   end
 
@@ -417,13 +418,20 @@ class Referential < ActiveRecord::Base
   end
 
   def migration_count
-    if self.class.connection.table_exists?("#{slug}.schema_migrations")
-      self.class.connection.select_value("select count(*) from #{slug}.schema_migrations;")
-    end
+    raw_value =
+      if self.class.connection.table_exists?("#{slug}.schema_migrations")
+        self.class.connection.select_value("select count(*) from #{slug}.schema_migrations;")
+      end
+
+    raw_value.to_i
   end
 
-  def assign_slug
-    self.slug ||= "#{name.parameterize.gsub('-', '_')}_#{Time.now.to_i}" if name
+  def assign_slug(time_reference = Time)
+    self.slug ||= begin
+      prefix = name.parameterize.gsub('-','_').gsub(/[^a-zA-Z_]/,'').gsub(/^_/,'')
+      prefix = "referential" if prefix.blank?
+      "#{prefix}_#{time_reference.now.to_i}"
+    end if name
   end
 
   def assign_prefix
diff --git a/app/models/simple_importer.rb b/app/models/simple_importer.rb
index b824d596d..d6ba64494 100644
--- a/app/models/simple_importer.rb
+++ b/app/models/simple_importer.rb
@@ -95,7 +95,8 @@ class SimpleImporter < ActiveRecord::Base
   end
 
   def dump_csv_from_context
-    filepath = "./#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
+    dir = context[:output_dir] || "log/importers"
+    filepath = File.join dir, "#{self.configuration_name}_#{Time.now.strftime "%y%m%d%H%M"}.csv"
     # for some reason, context[:csv].to_csv does not work
     CSV.open(filepath, 'w') do |csv|
       header = true
@@ -262,7 +263,7 @@ class SimpleImporter < ActiveRecord::Base
       msg += "\n\n"
       msg += colorize "=== MESSAGES (#{@messages.count}) ===\n", :green
       msg += "[...]\n" if @messages.count > lines_count
-      msg += @messages.last(lines_count).join("\n")
+      msg += @messages.last(lines_count).map{|m| m.truncate(@status_width)}.join("\n")
       msg += "\n"*[lines_count-@messages.count, 0].max
     end
 
@@ -273,7 +274,9 @@ class SimpleImporter < ActiveRecord::Base
       msg += @errors.last(lines_count).map do |j|
         kind = j[:kind]
         kind = colorize(kind, kind == :error ? :red : :orange)
-        encode_string "[#{kind}]\t\tL#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}"
+        kind = "[#{kind}]"
+        kind += " "*(25 - kind.size)
+        encode_string("#{kind}L#{j[:line]}\t#{j[:error]}\t\t#{j[:message]}").truncate(@status_width)
       end.join("\n")
     end
     custom_print msg, clear: true