Refactoring reflex & codifligne, add updated count message log

Refs #1981

6 files changed, 87 insertions, 45 deletions

diff --git a/config/locales/line_referential_syncs.en.yml b/config/locales/line_referential_syncs.en.yml
index 82ef48170..e78288299 100644
--- a/config/locales/line_referential_syncs.en.yml
+++ b/config/locales/line_referential_syncs.en.yml
@@ -10,8 +10,8 @@ en:
     message:
       new: "New synchronisation added"
       pending: "Synchronisation en cours"
-      successful: "Synchronisation réussie après %{processing_time} secondes avec %{imported} éléments importés. %{deleted} éléments ont été supprimés."
-      failed: "Synchronisation interrompue après %{processing_time} secondes avec l'erreur : %{error}."
+      successful: "Synchronization successful after %{processing_time} with %{imported} objects created, %{updated} objects updated. %{deleted} objects were deleted.."
+      failed: "Synchronization failed after %{processing_time} with error: %{error}."
 
   notice:
     line_referential_sync:
diff --git a/config/locales/line_referential_syncs.fr.yml b/config/locales/line_referential_syncs.fr.yml
index 48b5df190..4a24fd937 100644
--- a/config/locales/line_referential_syncs.fr.yml
+++ b/config/locales/line_referential_syncs.fr.yml
@@ -10,7 +10,7 @@ fr:
     message:
       new: "Synchronisation en attente"
       pending: "Synchronisation en cours"
-      successful: "Synchronisation réussie après %{processing_time}, avec %{imported} éléments importés. %{deleted} éléments ont été supprimés."
+      successful: "Synchronisation réussie après %{processing_time}, avec %{imported} éléments importés, %{updated} éléments mise à jour. %{deleted} éléments ont été supprimés."
       failed: "Synchronisation interrompue après %{processing_time}, avec l'erreur : %{error}."
   notice:
     line_referential_sync:
diff --git a/config/locales/stop_area_referential_syncs.en.yml b/config/locales/stop_area_referential_syncs.en.yml
index 9dc3173bc..76197d5a6 100644
--- a/config/locales/stop_area_referential_syncs.en.yml
+++ b/config/locales/stop_area_referential_syncs.en.yml
@@ -11,7 +11,7 @@ en:
       message:
         new: "New synchronisation added"
         pending: "Synchronization pending"
-        successful: "Synchronization successful after %{processing_time} with %{imported} objects. %{deleted} objects were deleted.."
+        successful: "Synchronization successful after %{processing_time} with %{imported} objects created, %{updated} objects updated. %{deleted} objects were deleted.."
         failed: "Synchronization failed after %{processing_time} with error: %{error}."
   notice:
     stop_area_referential_sync:
diff --git a/config/locales/stop_area_referential_syncs.fr.yml b/config/locales/stop_area_referential_syncs.fr.yml
index d0b79a8db..bf8d32ac5 100644
--- a/config/locales/stop_area_referential_syncs.fr.yml
+++ b/config/locales/stop_area_referential_syncs.fr.yml
@@ -10,7 +10,7 @@ fr:
     message:
       new: "Synchronisation en attente"
       pending: "Synchronisation en cours"
-      successful: "Synchronisation réussie après %{processing_time}, avec %{imported} éléments importés. %{deleted} éléments ont été supprimés."
+      successful: "Synchronisation réussie après %{processing_time}, avec %{imported} éléments importés, %{updated} éléments mise à jour. %{deleted} éléments ont été supprimés."
       failed: "Synchronisation interrompue après %{processing_time}, avec l'erreur : %{error}."
   notice:
     stop_area_referential_sync:
diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index deadad5ba..600415424 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -1,7 +1,28 @@
 module Stif
   module CodifLineSynchronization
     class << self
+      attr_accessor :imported_count, :updated_count, :deleted_count
+
+      def reset_counts
+        self.imported_count = 0
+        self.updated_count  = 0
+        self.deleted_count  = 0
+      end
+
+      def processed_counts
+        {
+          imported: self.imported_count,
+          updated: self.updated_count,
+          deleted: self.deleted_count
+        }
+      end
+
+      def increment_counts prop_name, value
+        self.send("#{prop_name}=", self.send(prop_name) + value)
+      end
+
       def synchronize
+        self.reset_counts
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
         # Fetch Codifline data
         client = Codifligne::API.new
@@ -47,10 +68,8 @@ module Stif
         # Delete deprecated Operators
         deleted_op = delete_deprecated(operators, Chouette::Company)
         log_deleted "Operators", deleted_op unless deleted_op == 0
-        {
-          imported: operators.count + lines.count + networks.count,
-          deleted: deleted_op + deleted_li + deleted_ne
-        }
+
+        self.processed_counts
       end
 
       def create_or_update_company(api_operator)
@@ -118,13 +137,14 @@ module Stif
       def delete_deprecated(objects, klass)
         ids = objects.map{ |o| o.stif_id }.to_a
         deprecated = klass.where.not(objectid: ids)
-        deprecated.destroy_all.length
+        increment_counts :deleted_count, deprecated.destroy_all.length
       end
 
       def delete_deprecated_lines(lines)
         ids = lines.map{ |l| l.stif_id }.to_a
         deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false)
         deprecated.update_all deactivated: true
+        increment_counts :deleted_count, deprecated.update_all(deactivated: true)
       end
 
       def save_or_update(params, klass)
@@ -132,10 +152,16 @@ module Stif
         object = klass.where(objectid: params[:objectid]).first
         if object
           object.assign_attributes(params)
-          object.save if object.changed?
+          if object.changed?
+            object.save
+            increment_counts :updated_count, 1
+          end
         else
           object = klass.new(params)
-          object.save if object.valid?
+          if object.valid?
+            object.save
+            increment_counts :imported_count, 1
+          end
         end
         object
       end
diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 822a295c0..bc3b9dd2d 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -1,6 +1,31 @@
 module Stif
   module ReflexSynchronization
     class << self
+      attr_accessor :imported_count, :updated_count, :deleted_count, :processed
+
+      def reset_counts
+        self.imported_count = 0
+        self.updated_count  = 0
+        self.deleted_count  = 0
+        self.processed      = []
+      end
+
+      def processed_counts
+        {
+          imported: self.imported_count,
+          updated: self.updated_count,
+          deleted: self.deleted_count
+        }
+      end
+
+      def log_processing_time message, time
+        Rails.logger.info "Reflex:sync - #{message} done in #{time} seconds"
+      end
+
+      def increment_counts prop_name, value
+        self.send("#{prop_name}=", self.send(prop_name) + value)
+      end
+
       def defaut_referential
         StopAreaReferential.find_by(name: "Reflex")
       end
@@ -10,41 +35,32 @@ module Stif
       end
 
       def synchronize
-        tstart           = Time.now
-        client           = Reflex::API.new
-        processed        = []
-        initial_count    = Chouette::StopArea.where(deleted_at: nil).count
-
+        self.reset_counts
         ['getOR', 'getOP'].each do |method|
           start   = Time.now
-          results = client.process method
-          Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds"
-          results.each do |type, entries|
-            Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
-          end
-
-          # Create or update stop_area for every quay, stop_place
+          results = Reflex::API.new().process(method)
+          log_processing_time("Process #{method}", Time.now - start)
           stop_areas = results[:Quay] | results[:StopPlace]
 
-          start = Time.now
-          stop_areas.each do |entry|
-            next unless is_valid_type_of_place_ref?(method, entry)
-            processed << entry['id']
-            self.create_or_update_stop_area entry
+          time = Benchmark.measure do
+            stop_areas.each do |entry|
+              next unless is_valid_type_of_place_ref?(method, entry)
+              self.processed << entry['id']
+              self.create_or_update_stop_area entry
+            end
           end
-          Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
+          log_processing_time("Create or update StopArea", time.real)
 
-          # Walk through every entry and set parent stop_area
-          start = Time.now
-          stop_areas.each do |entry|
-            self.stop_area_set_parent entry
+          time = Benchmark.measure do
+            stop_areas.map{|entry| self.stop_area_set_parent(entry)}
           end
-          Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
+          log_processing_time("StopArea set parent", time.real)
         end
-        {
-          imported: Chouette::StopArea.where(deleted_at: nil).count - initial_count,
-          deleted: self.set_deleted_stop_area(processed.uniq).size
-        }
+
+        # Set deleted_at for item not returned by api since last sync
+        time = Benchmark.measure { self.set_deleted_stop_area }
+        log_processing_time("StopArea #{self.deleted_count} deleted", time.real)
+        self.processed_counts
       end
 
       def is_valid_type_of_place_ref? method, entry
@@ -53,14 +69,12 @@ module Stif
         return true if method == 'getOP' && ['ZDE'].include?(entry["TypeOfPlaceRef"])
       end
 
-      def set_deleted_stop_area processed
-        start   = Time.now
-        deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - processed
+      def set_deleted_stop_area
+        deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - self.processed.uniq
         deleted.each_slice(50) do |object_ids|
           Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now)
         end
-        Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync in #{Time.now - start} seconds"
-        deleted
+        increment_counts :deleted_count, deleted.size
       end
 
       def stop_area_set_parent entry
@@ -122,7 +136,9 @@ module Stif
 
         if stop.changed?
           stop.creation_time = entry[:created]
-          stop.import_xml = entry[:xml]
+          stop.import_xml    = entry[:xml]
+          prop = stop.new_record? ? :imported_count : :updated_count
+          increment_counts prop, 1
           stop.save!
         end
         # Create AccessPoint from StopPlaceEntrance