diff options
| -rw-r--r-- | db/migrate/20160902082958_add_status_to_stop_area.rb | 5 | ||||
| -rw-r--r-- | db/migrate/20160902083036_add_import_xml_to_stop_area.rb | 5 | ||||
| -rw-r--r-- | db/migrate/20160902084601_add_import_xml_to_access_points.rb | 5 | ||||
| -rw-r--r-- | db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb | 5 | ||||
| -rw-r--r-- | db/schema.rb | 10 | ||||
| -rw-r--r-- | lib/stif/reflex_synchronization.rb | 52 | ||||
| -rw-r--r-- | spec/fixtures/reflex.zip | bin | 420184 -> 236609 bytes | |||
| -rw-r--r-- | spec/tasks/reflex_rake_spec.rb | 4 |
8 files changed, 64 insertions, 22 deletions
diff --git a/db/migrate/20160902082958_add_status_to_stop_area.rb b/db/migrate/20160902082958_add_status_to_stop_area.rb new file mode 100644 index 000000000..d05d9449e --- /dev/null +++ b/db/migrate/20160902082958_add_status_to_stop_area.rb @@ -0,0 +1,5 @@ +class AddStatusToStopArea < ActiveRecord::Migration + def change + add_column :stop_areas, :status, :string + end +end diff --git a/db/migrate/20160902083036_add_import_xml_to_stop_area.rb b/db/migrate/20160902083036_add_import_xml_to_stop_area.rb new file mode 100644 index 000000000..26ba99665 --- /dev/null +++ b/db/migrate/20160902083036_add_import_xml_to_stop_area.rb @@ -0,0 +1,5 @@ +class AddImportXmlToStopArea < ActiveRecord::Migration + def change + add_column :stop_areas, :import_xml, :text + end +end diff --git a/db/migrate/20160902084601_add_import_xml_to_access_points.rb b/db/migrate/20160902084601_add_import_xml_to_access_points.rb new file mode 100644 index 000000000..fd75cd783 --- /dev/null +++ b/db/migrate/20160902084601_add_import_xml_to_access_points.rb @@ -0,0 +1,5 @@ +class AddImportXmlToAccessPoints < ActiveRecord::Migration + def change + add_column :access_points, :import_xml, :text + end +end diff --git a/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb b/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb new file mode 100644 index 000000000..036d6a20f --- /dev/null +++ b/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb @@ -0,0 +1,5 @@ +class AddDeletedAtToStopAreas < ActiveRecord::Migration + def change + add_column :stop_areas, :deleted_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index 2aaecbf63..3c6b73b71 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20160831094427) do +ActiveRecord::Schema.define(version: 20160902093127) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -63,6 +63,7 @@ ActiveRecord::Schema.define(version: 20160831094427) do t.integer "stop_area_id", limit: 8 t.string "zip_code" t.string "city_name" + t.text "import_xml" end add_index "access_points", ["objectid"], :name => "access_points_objectid_key", :unique => true @@ -404,10 +405,10 @@ ActiveRecord::Schema.define(version: 20160831094427) do t.integer "object_version" t.datetime "creation_time" t.string "creator_id" - t.spatial "input_geometry", limit: {:srid=>4326, :type=>"line_string"} - t.spatial "processed_geometry", limit: {:srid=>4326, :type=>"line_string"} t.float "distance" t.boolean "no_processing" + t.spatial "input_geometry", limit: {:srid=>4326, :type=>"line_string"} + t.spatial "processed_geometry", limit: {:srid=>4326, :type=>"line_string"} end create_table "routes", force: true do |t| @@ -478,6 +479,9 @@ ActiveRecord::Schema.define(version: 20160831094427) do t.string "url" t.string "time_zone" t.integer "stop_area_referential_id" + t.string "status" + t.text "import_xml" + t.datetime "deleted_at" end add_index "stop_areas", ["objectid"], :name => "stop_areas_objectid_key", :unique => true diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb index 3b43fd5df..580a41c66 100644 --- a/lib/stif/reflex_synchronization.rb +++ b/lib/stif/reflex_synchronization.rb @@ -11,8 +11,9 @@ module Stif end def synchronize - start = Time.now - client = Reflex::API.new + start = Time.now + client = Reflex::API.new + processed = [] ['getOR', 'getOP'].each do |method| results = client.process method @@ -25,7 +26,7 @@ module Stif stop_areas = results[:Quay].merge(results[:StopPlace]) start = Time.now stop_areas.each do |id, entry| - self.create_or_update_stop_area entry + processed << self.create_or_update_stop_area(entry).objectid end Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds" @@ -36,6 +37,19 @@ module Stif end Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds" end + # Purge deleted stop_area + self.set_deleted_stop_area processed.uniq + end + + def set_deleted_stop_area processed + Rails.logger.info "Reflex:sync - StopArea start deleted_stop_area" + start = Time.now + deleted = Chouette::StopArea.pluck(:objectid).uniq - processed + deleted.each_slice(50) do |object_ids| + Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now) + end + Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync" + Rails.logger.info "Reflex:sync - StopArea purge deleted in #{Time.now - start} seconds" end def stop_area_set_parent entry @@ -60,12 +74,15 @@ module Stif def create_or_update_access_point entry, stop_area access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}") entry.version = entry.version.to_i + 1 if access.persisted? - access.name = entry.name - access.stop_area = stop_area - access.object_version = entry.version - access.zip_code = entry.postal_code - access.city_name = entry.city - access.access_type = entry.area_type + access.stop_area = stop_area + { + :name => :name, + :access_type => :area_type, + :object_version => :version, + :zip_code => :postal_code, + :city_name => :city, + :import_xml => :xml + }.each do |k, v| access[k] = entry.try(v) end access.save if access.changed? end @@ -73,16 +90,17 @@ module Stif stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}") # Hack, on save object_version will be incremented by 1 entry.version = entry.version.to_i + 1 if stop.persisted? - stop.stop_area_referential = self.defaut_referential - stop.name = entry.name - stop.creation_time = entry.created - stop.area_type = entry.area_type - stop.object_version = entry.version - stop.zip_code = entry.postal_code - stop.city_name = entry.city + { + :name => :name, + :creation_time => :created, + :area_type => :area_type, + :object_version => :version, + :zip_code => :postal_code, + :city_name => :city, + :import_xml => :xml + }.each do |k, v| stop[k] = entry.try(v) end stop.save if stop.changed? - # Create AccessPoint from StopPlaceEntrance if entry.try(:entrances) entry.entrances.each do |entrance| diff --git a/spec/fixtures/reflex.zip b/spec/fixtures/reflex.zip Binary files differindex 18763576e..831278863 100644 --- a/spec/fixtures/reflex.zip +++ b/spec/fixtures/reflex.zip diff --git a/spec/tasks/reflex_rake_spec.rb b/spec/tasks/reflex_rake_spec.rb index f1dc25bc8..f49eb41ec 100644 --- a/spec/tasks/reflex_rake_spec.rb +++ b/spec/tasks/reflex_rake_spec.rb @@ -12,7 +12,7 @@ describe 'reflex:sync' do # Must have an stop_area_referential create(:stop_area_referential, name: 'Reflex') Stif::ReflexSynchronization.synchronize - expect(Chouette::StopArea.count).to eq 7928 - expect(Chouette::AccessPoint.count).to eq 60 + expect(Chouette::StopArea.count).to eq 4309 + expect(Chouette::AccessPoint.count).to eq 49 end end |