Merge branch 'master' into staging

5 files changed, 148 insertions, 131 deletions

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 4bbc48946..1be5474c2 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -1,73 +1,56 @@
 module Stif
   module CodifLineSynchronization
     class << self
-      # Don't check last synchronizations if force_sync
-      def synchronize force_sync = false
-        # Check last synchronization and synchronization interval
-        date = DateTime.now.to_date - LineReferential.first.sync_interval.days
-        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: :ok).last.try(:created_at)
-        return if last_sync.present? && last_sync.to_date > date && !force_sync
-
+      def synchronize
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-        # TODO Check exceptions and status messages
-        begin
-          # Fetch Codifline data
-          client = Codifligne::API.new
-          operators       = client.operators
-          lines           = client.lines
-          networks        = client.networks
-          groups_of_lines = client.groups_of_lines
-
-          Rails.logger.info "Codifligne:sync - Codifligne request processed in #{elapsed_time_since start_time} seconds"
-
-          # Create or update Companies
-          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-          operators.map       { |o| create_or_update_company(o) }
-          log_create_or_update "Companies", operators.count, stime
-
-          # Create or update Lines
-          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-          lines.map           { |l| create_or_update_line(l) }
-          log_create_or_update "Lines", lines.count, stime
-
-          # Create or update Networks
-          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-          networks.map        { |n| create_or_update_network(n) }
-          log_create_or_update "Networks", networks.count, stime
-
-          # Create or update Group of lines
-          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
-          groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
-          log_create_or_update "Group of lines", groups_of_lines.count, stime
-
-          # Delete deprecated Group of lines
-          deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
-          log_deleted "Group of lines", deleted_gr unless deleted_gr == 0
-
-          # Delete deprecated Networks
-          deleted_ne = delete_deprecated(networks, Chouette::Network)
-          log_deleted "Networks", deleted_ne unless deleted_ne == 0
-
-          # Delete deprecated Lines
-          deleted_li = delete_deprecated_lines(lines)
-          log_deleted "Lines", deleted_li unless deleted_li == 0
-
-          # Delete deprecated Operators
-          deleted_op = delete_deprecated(operators, Chouette::Company)
-          log_deleted "Operators", deleted_op unless deleted_op == 0
-
-          # Building log message
-          total_codifligne_elements = operators.count + lines.count + networks.count + groups_of_lines.count
-          total_deleted = deleted_op + deleted_li + deleted_ne + deleted_gr
-          total_time = elapsed_time_since start_time
-
-          LineReferential.first.line_referential_sync.record_status :ok, I18n.t('synchronization.codifligne.message.success', time: total_time, imported: total_codifligne_elements, deleted: total_deleted)
-        rescue Exception => e
-          total_time = elapsed_time_since start_time
-
-          Rails.logger.error "Codifligne:sync - Error: #{e}, ended after #{total_time} seconds"
-          LineReferential.first.line_referential_sync.record_status :ko, I18n.t('synchronization.codifligne.message.failure', time: total_time)
-        end
+        # Fetch Codifline data
+        client = Codifligne::API.new
+        operators       = client.operators
+        lines           = client.lines
+        networks        = client.networks
+        # groups_of_lines = client.groups_of_lines
+
+        Rails.logger.info "Codifligne:sync - Codifligne request processed in #{elapsed_time_since start_time} seconds"
+
+        # Create or update Companies
+        stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        operators.map       { |o| create_or_update_company(o) }
+        log_create_or_update "Companies", operators.count, stime
+
+        # Create or update Lines
+        stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        lines.map           { |l| create_or_update_line(l) }
+        log_create_or_update "Lines", lines.count, stime
+
+        # Create or update Networks
+        stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        networks.map        { |n| create_or_update_network(n) }
+        log_create_or_update "Networks", networks.count, stime
+
+        # # Create or update Group of lines
+        # stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
+        # groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
+        # log_create_or_update "Group of lines", groups_of_lines.count, stime
+
+        # # Delete deprecated Group of lines
+        # deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
+        # log_deleted "Group of lines", deleted_gr unless deleted_gr == 0
+
+        # Delete deprecated Networks
+        deleted_ne = delete_deprecated(networks, Chouette::Network)
+        log_deleted "Networks", deleted_ne unless deleted_ne == 0
+
+        # Delete deprecated Lines
+        deleted_li = delete_deprecated_lines(lines)
+        log_deleted "Lines", deleted_li unless deleted_li == 0
+
+        # Delete deprecated Operators
+        deleted_op = delete_deprecated(operators, Chouette::Company)
+        log_deleted "Operators", deleted_op unless deleted_op == 0
+        {
+          imported: operators.count + lines.count + networks.count,
+          deleted: deleted_op + deleted_li + deleted_ne
+        }
       end
 
       def create_or_update_company(api_operator)
diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index b68fd6fc5..822a295c0 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -10,42 +10,47 @@ module Stif
       end
 
       def synchronize
-        tstart     = Time.now
-        begin
-          client     = Reflex::API.new
-          processed  = []
+        tstart           = Time.now
+        client           = Reflex::API.new
+        processed        = []
+        initial_count    = Chouette::StopArea.where(deleted_at: nil).count
 
-          ['getOR', 'getOP'].each do |method|
-            start   = Time.now
-            results = client.process method
-            Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds"
-            results.each do |type, entries|
-              Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
-            end
+        ['getOR', 'getOP'].each do |method|
+          start   = Time.now
+          results = client.process method
+          Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds"
+          results.each do |type, entries|
+            Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
+          end
 
-            # Create or update stop_area for every quay, stop_place
-            stop_areas = results[:Quay].merge(results[:StopPlace])
-            start = Time.now
-            stop_areas.each do |id, entry|
-              processed << self.create_or_update_stop_area(entry).objectid
-            end
-            Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
+          # Create or update stop_area for every quay, stop_place
+          stop_areas = results[:Quay] | results[:StopPlace]
 
-            # Walk through every entry and set parent stop_area
-            start = Time.now
-            stop_areas.each do |id, entry|
-              self.stop_area_set_parent entry
-            end
-            Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
+          start = Time.now
+          stop_areas.each do |entry|
+            next unless is_valid_type_of_place_ref?(method, entry)
+            processed << entry['id']
+            self.create_or_update_stop_area entry
           end
+          Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
 
-          # Purge deleted stop_area
-          deleted = self.set_deleted_stop_area processed.uniq
-          self.defaut_referential.stop_area_referential_sync.record_status :ok, I18n.t('synchronization.reflex.message.success', time: Time.now - tstart, imported: processed.uniq.size, deleted: deleted.size)
-        rescue Exception => e
-          Rails.logger.error "Reflex:sync - Error: #{e}, ended after #{Time.now - tstart} seconds"
-          self.defaut_referential.stop_area_referential_sync.record_status :ko, I18n.t('synchronization.reflex.message.failure', time: Time.now - tstart)
+          # Walk through every entry and set parent stop_area
+          start = Time.now
+          stop_areas.each do |entry|
+            self.stop_area_set_parent entry
+          end
+          Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
         end
+        {
+          imported: Chouette::StopArea.where(deleted_at: nil).count - initial_count,
+          deleted: self.set_deleted_stop_area(processed.uniq).size
+        }
+      end
+
+      def is_valid_type_of_place_ref? method, entry
+        return true if entry["TypeOfPlaceRef"].nil?
+        return true if method == 'getOR' && ['ZDL', 'LDA', 'ZDE'].include?(entry["TypeOfPlaceRef"])
+        return true if method == 'getOP' && ['ZDE'].include?(entry["TypeOfPlaceRef"])
       end
 
       def set_deleted_stop_area processed
@@ -59,58 +64,70 @@ module Stif
       end
 
       def stop_area_set_parent entry
-        return false unless entry.try(:parent_site_ref) || entry.try(:quays)
-        stop = self.find_by_object_id entry.id
+        return false unless entry['parent'] || entry['quays']
+        stop = self.find_by_object_id entry['id']
         return false unless stop
 
-        if entry.try(:parent_site_ref)
-          stop.parent = self.find_by_object_id entry.parent_site_ref
-          stop.save if stop.changed
+        if entry['parent']
+          stop.parent = self.find_by_object_id entry['parent']
+          stop.save! if stop.changed
         end
 
-        if entry.try(:quays)
-          entry.quays.each do |quay|
-            children = self.find_by_object_id(quay[:ref])
+        if entry['quays']
+          entry['quays'].each do |id|
+            children = self.find_by_object_id id
             next unless children
             children.parent = stop
-            children.save if children.changed?
+            children.save! if children.changed?
           end
         end
       end
 
+      def access_point_access_type entry
+        if entry['IsEntry'] ==  'true' && entry['IsExit'] == 'true'
+          'in_out'
+        elsif entry['IsEntry'] == 'true'
+          'in'
+        elsif entry['IsExit'] == 'true'
+          'out'
+        end
+      end
+
       def create_or_update_access_point entry, stop_area
-        access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
+        access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry['id'].tr(':', '')}")
         # Hack, on save object_version will be incremented by 1
-        entry.version = entry.version.to_i + 1  if access.persisted?
+        entry['version']   = entry['version'].to_i + 1  if access.persisted?
+        access.access_type = self.access_point_access_type(entry)
         access.stop_area = stop_area
         {
-          :name           => :name,
-          :access_type    => :area_type,
-          :object_version => :version,
-          :zip_code       => :postal_code,
-          :city_name      => :city,
-          :import_xml     => :xml
-        }.each do |k, v| access[k] = entry.try(v) end
-        access.save if access.changed?
+          :name           => 'Name',
+          :object_version => 'version',
+          :zip_code       => 'PostalRegion',
+          :city_name      => 'Town'
+        }.each do |k, v| access[k] = entry[v] end
+        access.save! if access.changed?
       end
 
       def create_or_update_stop_area entry
-        stop = Chouette::StopArea.find_or_create_by(objectid: entry.id)
+        stop = Chouette::StopArea.find_or_create_by(objectid: entry['id'])
         stop.deleted_at            = nil
         stop.stop_area_referential = self.defaut_referential
         {
-          :name           => :name,
-          :creation_time  => :created,
-          :area_type      => :area_type,
-          :object_version => :version,
-          :zip_code       => :postal_code,
-          :city_name      => :city,
-          :import_xml     => :xml
-        }.each do |k, v| stop[k] = entry.try(v) end
-        stop.save if stop.changed?
+          :name           => 'Name',
+          :area_type      => 'type',
+          :object_version => 'version',
+          :zip_code       => 'PostalRegion',
+          :city_name      => 'Town'
+        }.each do |k, v| stop[k] = entry[v] end
+
+        if stop.changed?
+          stop.creation_time = entry[:created]
+          stop.import_xml = entry[:xml]
+          stop.save!
+        end
         # Create AccessPoint from StopPlaceEntrance
-        if entry.try(:entrances)
-          entry.entrances.each do |entrance|
+        if entry[:stop_place_entrances]
+          entry[:stop_place_entrances].each do |entrance|
             self.create_or_update_access_point entrance, stop
           end
         end
diff --git a/lib/tasks/codifligne.rake b/lib/tasks/codifligne.rake
index 8b7a4f669..dd5e99370 100644
--- a/lib/tasks/codifligne.rake
+++ b/lib/tasks/codifligne.rake
@@ -1,6 +1,8 @@
 namespace :codifligne do
   desc "Sync lines, companies, networks, and group of lines from codifligne"
   task sync: :environment  do
-    Stif::CodifLineSynchronization.synchronize
+    sync = LineReferential.find_by(name: 'CodifLigne').line_referential_syncs.build
+    raise "Codifligne:sync aborted - There is already an synchronisation in progress" unless sync.valid?
+    sync.save if sync.valid?
   end
 end
diff --git a/lib/tasks/extensions.rake b/lib/tasks/extensions.rake
new file mode 100644
index 000000000..6ede62b10
--- /dev/null
+++ b/lib/tasks/extensions.rake
@@ -0,0 +1,15 @@
+namespace :db do
+  desc 'Creates shared_extensions Schema and enables hstore extension'
+  task :extensions => :environment  do
+    ActiveRecord::Base.connection.execute 'CREATE SCHEMA IF NOT EXISTS shared_extensions;'
+    ActiveRecord::Base.connection.execute 'CREATE EXTENSION IF NOT EXISTS hstore SCHEMA shared_extensions;'
+  end
+end
+
+Rake::Task["db:create"].enhance do
+  Rake::Task["db:extensions"].invoke
+end
+
+Rake::Task["db:test:purge"].enhance do
+  Rake::Task["db:extensions"].invoke
+end
diff --git a/lib/tasks/reflex.rake b/lib/tasks/reflex.rake
index 0020f1b60..67496cee0 100644
--- a/lib/tasks/reflex.rake
+++ b/lib/tasks/reflex.rake
@@ -1,8 +1,8 @@
 namespace :reflex do
   desc "Sync data from Reflex api"
   task sync: :environment  do
-    start = Time.now
-    Stif::ReflexSynchronization.synchronize
-    Rails.logger.debug "Reflex:sync done in #{Time.now - start} seconds !"
+    sync = StopAreaReferential.find_by(name: 'Reflex').stop_area_referential_syncs.build
+    raise "reflex:sync aborted - There is already an synchronisation in progress" unless sync.valid?
+    sync.save if sync.valid?
   end
 end