Merge branch 'master' into staging

10 files changed, 400 insertions, 18 deletions

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
new file mode 100644
index 000000000..01ad29afc
--- /dev/null
+++ b/lib/stif/codif_line_synchronization.rb
@@ -0,0 +1,177 @@
+module Stif
+  module CodifLineSynchronization
+    class << self
+      # Don't check last synchronizations if force_sync
+      def synchronize force_sync = false
+        # Check last synchronization and synchronization interval
+        date = DateTime.now.to_date - LineReferential.first.sync_interval.days
+        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: :ok).last.try(:created_at)
+        return if last_sync.present? && last_sync.to_date > date && !force_sync
+
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        # TODO Check exceptions and status messages
+        begin
+          # Fetch Codifline data
+          client = Codifligne::API.new
+          operators       = client.operators
+          lines           = client.lines
+          networks        = client.networks
+          groups_of_lines = client.groups_of_lines
+
+          Rails.logger.info "Codifligne:sync - Codifligne request processed in #{elapsed_time_since start_time} seconds"
+
+          # Create or update Companies
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          operators.map       { |o| create_or_update_company(o) }
+          log_create_or_update "Companies", operators.count, stime
+
+          # Create or update Lines
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          lines.map           { |l| create_or_update_line(l) }
+          log_create_or_update "Lines", lines.count, stime
+
+          # Create or update Networks
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          networks.map        { |n| create_or_update_network(n) }
+          log_create_or_update "Networks", networks.count, stime
+
+          # Create or update Group of lines
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+          groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
+          log_create_or_update "Group of lines", groups_of_lines.count, stime
+
+          # Delete deprecated Group of lines
+          deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
+          log_deleted "Group of lines", deleted_gr unless deleted_gr == 0
+
+          # Delete deprecated Networks
+          deleted_ne = delete_deprecated(networks, Chouette::Network)
+          log_deleted "Networks", deleted_ne unless deleted_ne == 0
+
+          # Delete deprecated Lines
+          deleted_li = delete_deprecated_lines(lines)
+          log_deleted "Lines", deleted_li unless deleted_li == 0
+
+          # Delete deprecated Operators
+          deleted_op = delete_deprecated(operators, Chouette::Company)
+          log_deleted "Operators", deleted_op unless deleted_op == 0
+
+          # Building log message
+          total_codifligne_elements = operators.count + lines.count + networks.count + groups_of_lines.count
+          total_deleted = deleted_op + deleted_li + deleted_ne + deleted_gr
+          total_time = elapsed_time_since start_time
+
+          LineReferential.first.line_referential_sync.record_status :ok, I18n.t('synchronization.message.success', time: total_time, imported: total_codifligne_elements, deleted: total_deleted)
+        rescue Exception => e
+          total_time = elapsed_time_since start_time
+
+          Rails.logger.error "Codifligne:sync - Error: #{e}, ended after #{total_time} seconds"
+          LineReferential.first.line_referential_sync.record_status :ko, I18n.t('synchronization.message.failure', time: total_time)
+        end
+      end
+
+      def create_or_update_company(api_operator)
+        params = {
+          name: api_operator.name,
+          objectid: api_operator.stif_id,
+          import_xml: api_operator.xml
+        }
+        save_or_update(params, Chouette::Company)
+      end
+
+      def create_or_update_line(api_line)
+        params = {
+          name: api_line.name,
+          objectid: api_line.stif_id,
+          number: api_line.short_name,
+          deactivated: (api_line.status == "inactive" ? true : false),
+          import_xml: api_line.xml
+        }
+
+        # Find Company
+        # TODO Check behavior when operator_codes count is 0 or > 1
+        if api_line.operator_codes.any?
+          company_id = "STIF:CODIFLIGNE:Operator:" + api_line.operator_codes.first
+          params[:company] = Chouette::Company.find_by(objectid: company_id)
+        end
+
+        save_or_update(params, Chouette::Line)
+      end
+
+      def create_or_update_network(api_network)
+        params = {
+          name: api_network.name,
+          objectid: api_network.stif_id,
+          import_xml: api_network.xml
+        }
+
+        # Find Lines
+        params[:lines] = []
+        api_network.line_codes.each do |line|
+          line_id = "STIF:CODIFLIGNE:Line:" + line
+          params[:lines] << Chouette::Line.find_by(objectid: line_id)
+        end
+
+        save_or_update(params, Chouette::Network)
+      end
+
+      def create_or_update_group_of_lines(api_group_of_lines)
+        params = {
+          name: api_group_of_lines.name,
+          objectid: api_group_of_lines.stif_id,
+          import_xml: api_group_of_lines.xml
+        }
+
+        # Find Lines
+        params[:lines] = []
+        api_group_of_lines.line_codes.each do |line|
+          line_id = "STIF:CODIFLIGNE:Line:" + line
+          # TODO : handle when lines doesn't exist
+          chouette_line = Chouette::Line.find_by(objectid: line_id)
+          params[:lines] << chouette_line if chouette_line.present?
+        end
+
+        save_or_update(params, Chouette::GroupOfLine)
+      end
+
+      def delete_deprecated(objects, klass)
+        ids = objects.map{ |o| o.stif_id }.to_a
+        deprecated = klass.where.not(objectid: ids)
+        deprecated.destroy_all.length
+      end
+
+      def delete_deprecated_lines(lines)
+        ids = lines.map{ |l| l.stif_id }.to_a
+        deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false)
+        deprecated.update_all deactivated: true
+      end
+
+      def save_or_update(params, klass)
+        params[:line_referential] = LineReferential.first
+        object = klass.where(objectid: params[:objectid]).first
+        if object
+          object.assign_attributes(params)
+          object.save if object.changed?
+        else
+          object = klass.new(params)
+          object.save if object.valid?
+        end
+        object
+      end
+
+      def elapsed_time_since start_time = 0
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+      end
+
+      def log_create_or_update name, count, start_time
+        time = elapsed_time_since start_time
+        Rails.logger.info "Codifligne:sync - #{count} #{name} retrieved"
+        Rails.logger.info "Codifligne:sync - Create or update #{name} done in #{time} seconds"
+      end
+
+      def log_deleted name, count
+        Rails.logger.info "Codifligne:sync - #{count} #{name} deleted"
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
new file mode 100644
index 000000000..55fe3d404
--- /dev/null
+++ b/lib/stif/reflex_synchronization.rb
@@ -0,0 +1,116 @@
+module Stif
+  module ReflexSynchronization
+    class << self
+      def defaut_referential
+        StopAreaReferential.find_by(name: "Reflex")
+      end
+
+      # Todo remove dummy objectid
+      def find_by_object_id objectid
+        Chouette::StopArea.find_by(objectid: "dummy:StopArea:#{objectid.tr(':', '')}")
+      end
+
+      def synchronize
+        start      = Time.now
+        client     = Reflex::API.new
+        processed  = []
+
+        ['getOR', 'getOP'].each do |method|
+          results = client.process method
+          Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds"
+          results.each do |type, entries|
+            Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
+          end
+
+          # Create or update stop_area for every quay, stop_place
+          stop_areas = results[:Quay].merge(results[:StopPlace])
+          start = Time.now
+          stop_areas.each do |id, entry|
+            processed << self.create_or_update_stop_area(entry).objectid
+          end
+          Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
+
+          # Walk through every entry and set parent stop_area
+          start = Time.now
+          stop_areas.each do |id, entry|
+            self.stop_area_set_parent entry
+          end
+          Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
+        end
+        # Purge deleted stop_area
+        self.set_deleted_stop_area processed.uniq
+      end
+
+      def set_deleted_stop_area processed
+        Rails.logger.info "Reflex:sync - StopArea start deleted_stop_area"
+        start   = Time.now
+        deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - processed
+        deleted.each_slice(50) do |object_ids|
+          Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now)
+        end
+        Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync"
+        Rails.logger.info "Reflex:sync - StopArea purge deleted in #{Time.now - start} seconds"
+      end
+
+      def stop_area_set_parent entry
+        return false unless entry.try(:parent_site_ref) || entry.try(:quays)
+        stop = self.find_by_object_id entry.id
+        return false unless stop
+
+        if entry.try(:parent_site_ref)
+          stop.parent = self.find_by_object_id entry.parent_site_ref
+          stop.save if stop.changed
+        end
+
+        if entry.try(:quays)
+          entry.quays.each do |quay|
+            children = self.find_by_object_id(quay[:ref])
+            next unless children
+            children.parent = stop
+            children.save if children.changed?
+          end
+        end
+      end
+
+      def create_or_update_access_point entry, stop_area
+        access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
+        entry.version = entry.version.to_i + 1  if access.persisted?
+        access.stop_area = stop_area
+        {
+          :name           => :name,
+          :access_type    => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| access[k] = entry.try(v) end
+        access.save if access.changed?
+      end
+
+      def create_or_update_stop_area entry
+        stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
+        # Hack, on save object_version will be incremented by 1
+        entry.version = entry.version.to_i + 1  if stop.persisted?
+        stop.deleted_at            = nil
+        stop.stop_area_referential = self.defaut_referential
+        {
+          :name           => :name,
+          :creation_time  => :created,
+          :area_type      => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| stop[k] = entry.try(v) end
+        stop.save if stop.changed?
+        # Create AccessPoint from StopPlaceEntrance
+        if entry.try(:entrances)
+          entry.entrances.each do |entrance|
+            self.create_or_update_access_point entrance, stop
+          end
+        end
+        stop
+      end
+    end
+  end
+end
diff --git a/lib/tasks/ci.rake b/lib/tasks/ci.rake
index beb3b19d1..7fca878b6 100644
--- a/lib/tasks/ci.rake
+++ b/lib/tasks/ci.rake
@@ -33,4 +33,4 @@ namespace :ci do
 end
 
 desc "Run continuous integration tasks (spec, ...)"
-task :ci => ["ci:setup", "spec", "ci:deploy", "ci:clean"]
+task :ci => ["ci:setup", "spec", "cucumber", "ci:deploy", "ci:clean"]
diff --git a/lib/tasks/codifligne.rake b/lib/tasks/codifligne.rake
new file mode 100644
index 000000000..8b7a4f669
--- /dev/null
+++ b/lib/tasks/codifligne.rake
@@ -0,0 +1,6 @@
+namespace :codifligne do
+  desc "Sync lines, companies, networks, and group of lines from codifligne"
+  task sync: :environment  do
+    Stif::CodifLineSynchronization.synchronize
+  end
+end
diff --git a/lib/tasks/cucumber.rake b/lib/tasks/cucumber.rake
new file mode 100644
index 000000000..23d9b3eb4
--- /dev/null
+++ b/lib/tasks/cucumber.rake
@@ -0,0 +1,76 @@
+# IMPORTANT: This file is generated by cucumber-rails - edit at your own peril.
+# It is recommended to regenerate this file in the future when you upgrade to a
+# newer version of cucumber-rails. Consider adding your own code to a new file
+# instead of editing this one. Cucumber will automatically load all features/**/*.rb
+# files.
+
+
+unless ARGV.any? {|a| a =~ /^gems/} # Don't load anything when running the gems:* tasks
+
+vendored_cucumber_bin = Dir["#{Rails.root}/vendor/{gems,plugins}/cucumber*/bin/cucumber"].first
+$LOAD_PATH.unshift(File.dirname(vendored_cucumber_bin) + '/../lib') unless vendored_cucumber_bin.nil?
+
+begin
+  require 'cucumber/rake/task'
+
+  namespace :cucumber do
+    Cucumber::Rake::Task.new({:ok => 'test:prepare'}, 'Run features that should pass') do |t|
+      t.binary = vendored_cucumber_bin # If nil, the gem's binary is used.
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'default'
+    end
+
+    Cucumber::Rake::Task.new({:wip => 'test:prepare'}, 'Run features that are being worked on') do |t|
+      t.binary = vendored_cucumber_bin
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'wip'
+    end
+
+    Cucumber::Rake::Task.new({:rerun => 'test:prepare'}, 'Record failing features and run only them if any exist') do |t|
+      t.binary = vendored_cucumber_bin
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'rerun'
+    end
+
+    desc 'Run all features'
+    task :all => [:ok, :wip]
+
+    task :statsetup do
+      require 'rails/code_statistics'
+      ::STATS_DIRECTORIES << %w(Cucumber\ features features) if File.exist?('features')
+      ::CodeStatistics::TEST_TYPES << "Cucumber features" if File.exist?('features')
+    end
+
+    task :annotations_setup do
+      Rails.application.configure do
+        if config.respond_to?(:annotations)
+          config.annotations.directories << 'features'
+          config.annotations.register_extensions('feature') { |tag| /#\s*(#{tag}):?\s*(.*)$/ }
+        end
+      end
+    end
+  end
+  desc 'Alias for cucumber:ok'
+  task :cucumber => 'cucumber:ok'
+
+  task :default => :cucumber
+
+  task :features => :cucumber do
+    STDERR.puts "*** The 'features' task is deprecated. See rake -T cucumber ***"
+  end
+
+  # In case we don't have the generic Rails test:prepare hook, append a no-op task that we can depend upon.
+  task 'test:prepare' do
+  end
+
+  task :stats => 'cucumber:statsetup'
+
+  task :notes => 'cucumber:annotations_setup'
+rescue LoadError
+  desc 'cucumber rake task not available (cucumber not installed)'
+  task :cucumber do
+    abort 'Cucumber rake task is not available. Be sure to install cucumber as a gem or plugin'
+  end
+end
+
+end
diff --git a/lib/tasks/demo.rake b/lib/tasks/demo.rake
index fbafa96c1..d7e8c59f1 100644
--- a/lib/tasks/demo.rake
+++ b/lib/tasks/demo.rake
@@ -10,14 +10,13 @@ namespace :demo do
 
     organisation = Organisation.create!(:name => "DemoChouette")
     user = organisation.users.create( :name => "Demo", :email => "demo@chouette.mobi", :password => "chouette", :password_confirmation =>"chouette")
-    user.confirm!
     referential = organisation.referentials.create( :name => "Tatrobus", :slug => "tatrobus", :prefix => "TAT")
 
     #resource = Rack::Test::UploadedFile.new( Rails.application.config.demo_data, 'application/zip', false)
     #import_instance = ImportTask.new( :resources => resource, :referential_id => referential.id, :user_name => user.name, :no_save => false, :user_id => user.id, :name => "initialize demo", :data_format => "neptune")
     #import_instance.save
-	 
-	 File.open("/tmp/parameters_demo.json", "w")  { |file| 
+
+	 File.open("/tmp/parameters_demo.json", "w")  { |file|
 	                                           file.write('{
 "parameters" :  {
     "neptune-import": {
@@ -30,10 +29,10 @@ namespace :demo do
         }
 	 }
 }') }
-	 
+
 	 cmd = 'curl -F "file=@'+Rails.application.config.demo_data+';filename=tatrobus.zip" -F "file=@/tmp/parameters_demo.json;filename=parameters.json" http://localhost:8180/chouette_iev/referentials/tatrobus/importer/neptune'
 	 system(cmd)
-	 
+
     puts "Restore demo environment complete"
   end
 end
diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
new file mode 100644
index 000000000..1b21d7119
--- /dev/null
+++ b/lib/tasks/organisations.rake
@@ -0,0 +1,6 @@
+namespace :organisations do
+  desc "Sync organisations from stif portail"
+  task sync: :environment  do
+    Organisation.portail_sync
+  end
+end
diff --git a/lib/tasks/reflex.rake b/lib/tasks/reflex.rake
new file mode 100644
index 000000000..0020f1b60
--- /dev/null
+++ b/lib/tasks/reflex.rake
@@ -0,0 +1,8 @@
+namespace :reflex do
+  desc "Sync data from Reflex api"
+  task sync: :environment  do
+    start = Time.now
+    Stif::ReflexSynchronization.synchronize
+    Rails.logger.debug "Reflex:sync done in #{Time.now - start} seconds !"
+  end
+end
diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
new file mode 100644
index 000000000..c045639c1
--- /dev/null
+++ b/lib/tasks/users.rake
@@ -0,0 +1,6 @@
+namespace :users do
+  desc "Sync users from stif portail"
+  task sync: :environment  do
+    User.portail_sync
+  end
+end
diff --git a/lib/tasks/warbler.rake b/lib/tasks/warbler.rake
deleted file mode 100644
index e69457583..000000000
--- a/lib/tasks/warbler.rake
+++ /dev/null
@@ -1,12 +0,0 @@
-begin
-  require 'warbler'
-  Warbler::Task.new
-
-  task "war:setup" do
-    ENV['RAILS_RELATIVE_URL_ROOT'] ||= "/chouette2"
-  end
-
-  task :war => ["war:setup", "assets:precompile"]
-rescue LoadError => e
-  puts "Failed to load Warbler. Make sure it's installed."
-end