diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/stif/codif_line_synchronization.rb | 40 | ||||
| -rw-r--r-- | lib/stif/reflex_synchronization.rb | 82 |
2 files changed, 82 insertions, 40 deletions
diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb index deadad5ba..600415424 100644 --- a/lib/stif/codif_line_synchronization.rb +++ b/lib/stif/codif_line_synchronization.rb @@ -1,7 +1,28 @@ module Stif module CodifLineSynchronization class << self + attr_accessor :imported_count, :updated_count, :deleted_count + + def reset_counts + self.imported_count = 0 + self.updated_count = 0 + self.deleted_count = 0 + end + + def processed_counts + { + imported: self.imported_count, + updated: self.updated_count, + deleted: self.deleted_count + } + end + + def increment_counts prop_name, value + self.send("#{prop_name}=", self.send(prop_name) + value) + end + def synchronize + self.reset_counts start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) # Fetch Codifline data client = Codifligne::API.new @@ -47,10 +68,8 @@ module Stif # Delete deprecated Operators deleted_op = delete_deprecated(operators, Chouette::Company) log_deleted "Operators", deleted_op unless deleted_op == 0 - { - imported: operators.count + lines.count + networks.count, - deleted: deleted_op + deleted_li + deleted_ne - } + + self.processed_counts end def create_or_update_company(api_operator) @@ -118,13 +137,14 @@ module Stif def delete_deprecated(objects, klass) ids = objects.map{ |o| o.stif_id }.to_a deprecated = klass.where.not(objectid: ids) - deprecated.destroy_all.length + increment_counts :deleted_count, deprecated.destroy_all.length end def delete_deprecated_lines(lines) ids = lines.map{ |l| l.stif_id }.to_a deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false) deprecated.update_all deactivated: true + increment_counts :deleted_count, deprecated.update_all(deactivated: true) end def save_or_update(params, klass) @@ -132,10 +152,16 @@ module Stif object = klass.where(objectid: params[:objectid]).first if object object.assign_attributes(params) - object.save if object.changed? + if object.changed? + object.save + increment_counts :updated_count, 1 + end else object = klass.new(params) - object.save if object.valid? + if object.valid? + object.save + increment_counts :imported_count, 1 + end end object end diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb index 822a295c0..bc3b9dd2d 100644 --- a/lib/stif/reflex_synchronization.rb +++ b/lib/stif/reflex_synchronization.rb @@ -1,6 +1,31 @@ module Stif module ReflexSynchronization class << self + attr_accessor :imported_count, :updated_count, :deleted_count, :processed + + def reset_counts + self.imported_count = 0 + self.updated_count = 0 + self.deleted_count = 0 + self.processed = [] + end + + def processed_counts + { + imported: self.imported_count, + updated: self.updated_count, + deleted: self.deleted_count + } + end + + def log_processing_time message, time + Rails.logger.info "Reflex:sync - #{message} done in #{time} seconds" + end + + def increment_counts prop_name, value + self.send("#{prop_name}=", self.send(prop_name) + value) + end + def defaut_referential StopAreaReferential.find_by(name: "Reflex") end @@ -10,41 +35,32 @@ module Stif end def synchronize - tstart = Time.now - client = Reflex::API.new - processed = [] - initial_count = Chouette::StopArea.where(deleted_at: nil).count - + self.reset_counts ['getOR', 'getOP'].each do |method| start = Time.now - results = client.process method - Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds" - results.each do |type, entries| - Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved" - end - - # Create or update stop_area for every quay, stop_place + results = Reflex::API.new().process(method) + log_processing_time("Process #{method}", Time.now - start) stop_areas = results[:Quay] | results[:StopPlace] - start = Time.now - stop_areas.each do |entry| - next unless is_valid_type_of_place_ref?(method, entry) - processed << entry['id'] - self.create_or_update_stop_area entry + time = Benchmark.measure do + stop_areas.each do |entry| + next unless is_valid_type_of_place_ref?(method, entry) + self.processed << entry['id'] + self.create_or_update_stop_area entry + end end - Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds" + log_processing_time("Create or update StopArea", time.real) - # Walk through every entry and set parent stop_area - start = Time.now - stop_areas.each do |entry| - self.stop_area_set_parent entry + time = Benchmark.measure do + stop_areas.map{|entry| self.stop_area_set_parent(entry)} end - Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds" + log_processing_time("StopArea set parent", time.real) end - { - imported: Chouette::StopArea.where(deleted_at: nil).count - initial_count, - deleted: self.set_deleted_stop_area(processed.uniq).size - } + + # Set deleted_at for item not returned by api since last sync + time = Benchmark.measure { self.set_deleted_stop_area } + log_processing_time("StopArea #{self.deleted_count} deleted", time.real) + self.processed_counts end def is_valid_type_of_place_ref? method, entry @@ -53,14 +69,12 @@ module Stif return true if method == 'getOP' && ['ZDE'].include?(entry["TypeOfPlaceRef"]) end - def set_deleted_stop_area processed - start = Time.now - deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - processed + def set_deleted_stop_area + deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - self.processed.uniq deleted.each_slice(50) do |object_ids| Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now) end - Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync in #{Time.now - start} seconds" - deleted + increment_counts :deleted_count, deleted.size end def stop_area_set_parent entry @@ -122,7 +136,9 @@ module Stif if stop.changed? stop.creation_time = entry[:created] - stop.import_xml = entry[:xml] + stop.import_xml = entry[:xml] + prop = stop.new_record? ? :imported_count : :updated_count + increment_counts prop, 1 stop.save! end # Create AccessPoint from StopPlaceEntrance |