Reflex store xml + purge deleted since last sync

8 files changed, 64 insertions, 22 deletions

diff --git a/db/migrate/20160902082958_add_status_to_stop_area.rb b/db/migrate/20160902082958_add_status_to_stop_area.rb
new file mode 100644
index 000000000..d05d9449e
--- /dev/null
+++ b/db/migrate/20160902082958_add_status_to_stop_area.rb
@@ -0,0 +1,5 @@
+class AddStatusToStopArea < ActiveRecord::Migration
+  def change
+    add_column :stop_areas, :status, :string
+  end
+end
diff --git a/db/migrate/20160902083036_add_import_xml_to_stop_area.rb b/db/migrate/20160902083036_add_import_xml_to_stop_area.rb
new file mode 100644
index 000000000..26ba99665
--- /dev/null
+++ b/db/migrate/20160902083036_add_import_xml_to_stop_area.rb
@@ -0,0 +1,5 @@
+class AddImportXmlToStopArea < ActiveRecord::Migration
+  def change
+    add_column :stop_areas, :import_xml, :text
+  end
+end
diff --git a/db/migrate/20160902084601_add_import_xml_to_access_points.rb b/db/migrate/20160902084601_add_import_xml_to_access_points.rb
new file mode 100644
index 000000000..fd75cd783
--- /dev/null
+++ b/db/migrate/20160902084601_add_import_xml_to_access_points.rb
@@ -0,0 +1,5 @@
+class AddImportXmlToAccessPoints < ActiveRecord::Migration
+  def change
+    add_column :access_points, :import_xml, :text
+  end
+end
diff --git a/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb b/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb
new file mode 100644
index 000000000..036d6a20f
--- /dev/null
+++ b/db/migrate/20160902093127_add_deleted_at_to_stop_areas.rb
@@ -0,0 +1,5 @@
+class AddDeletedAtToStopAreas < ActiveRecord::Migration
+  def change
+    add_column :stop_areas, :deleted_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2aaecbf63..3c6b73b71 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160831094427) do
+ActiveRecord::Schema.define(version: 20160902093127) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -63,6 +63,7 @@ ActiveRecord::Schema.define(version: 20160831094427) do
     t.integer  "stop_area_id",                    limit: 8
     t.string   "zip_code"
     t.string   "city_name"
+    t.text     "import_xml"
   end
 
   add_index "access_points", ["objectid"], :name => "access_points_objectid_key", :unique => true
@@ -404,10 +405,10 @@ ActiveRecord::Schema.define(version: 20160831094427) do
     t.integer  "object_version"
     t.datetime "creation_time"
     t.string   "creator_id"
-    t.spatial  "input_geometry",     limit: {:srid=>4326, :type=>"line_string"}
-    t.spatial  "processed_geometry", limit: {:srid=>4326, :type=>"line_string"}
     t.float    "distance"
     t.boolean  "no_processing"
+    t.spatial  "input_geometry",     limit: {:srid=>4326, :type=>"line_string"}
+    t.spatial  "processed_geometry", limit: {:srid=>4326, :type=>"line_string"}
   end
 
   create_table "routes", force: true do |t|
@@ -478,6 +479,9 @@ ActiveRecord::Schema.define(version: 20160831094427) do
     t.string   "url"
     t.string   "time_zone"
     t.integer  "stop_area_referential_id"
+    t.string   "status"
+    t.text     "import_xml"
+    t.datetime "deleted_at"
   end
 
   add_index "stop_areas", ["objectid"], :name => "stop_areas_objectid_key", :unique => true
diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 3b43fd5df..580a41c66 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -11,8 +11,9 @@ module Stif
       end
 
       def synchronize
-        start  = Time.now
-        client = Reflex::API.new
+        start      = Time.now
+        client     = Reflex::API.new
+        processed  = []
 
         ['getOR', 'getOP'].each do |method|
           results = client.process method
@@ -25,7 +26,7 @@ module Stif
           stop_areas = results[:Quay].merge(results[:StopPlace])
           start = Time.now
           stop_areas.each do |id, entry|
-            self.create_or_update_stop_area entry
+            processed << self.create_or_update_stop_area(entry).objectid
           end
           Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
 
@@ -36,6 +37,19 @@ module Stif
           end
           Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
         end
+        # Purge deleted stop_area
+        self.set_deleted_stop_area processed.uniq
+      end
+
+      def set_deleted_stop_area processed
+        Rails.logger.info "Reflex:sync - StopArea start deleted_stop_area"
+        start   = Time.now
+        deleted = Chouette::StopArea.pluck(:objectid).uniq - processed
+        deleted.each_slice(50) do |object_ids|
+          Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now)
+        end
+        Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync"
+        Rails.logger.info "Reflex:sync - StopArea purge deleted in #{Time.now - start} seconds"
       end
 
       def stop_area_set_parent entry
@@ -60,12 +74,15 @@ module Stif
       def create_or_update_access_point entry, stop_area
         access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
         entry.version = entry.version.to_i + 1  if access.persisted?
-        access.name           = entry.name
-        access.stop_area      = stop_area
-        access.object_version = entry.version
-        access.zip_code       = entry.postal_code
-        access.city_name      = entry.city
-        access.access_type    = entry.area_type
+        access.stop_area = stop_area
+        {
+          :name           => :name,
+          :access_type    => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| access[k] = entry.try(v) end
         access.save if access.changed?
       end
 
@@ -73,16 +90,17 @@ module Stif
         stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
         # Hack, on save object_version will be incremented by 1
         entry.version = entry.version.to_i + 1  if stop.persisted?
-
         stop.stop_area_referential = self.defaut_referential
-        stop.name           = entry.name
-        stop.creation_time  = entry.created
-        stop.area_type      = entry.area_type
-        stop.object_version = entry.version
-        stop.zip_code       = entry.postal_code
-        stop.city_name      = entry.city
+        {
+          :name           => :name,
+          :creation_time  => :created,
+          :area_type      => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| stop[k] = entry.try(v) end
         stop.save if stop.changed?
-
         # Create AccessPoint from StopPlaceEntrance
         if entry.try(:entrances)
           entry.entrances.each do |entrance|
diff --git a/spec/fixtures/reflex.zip b/spec/fixtures/reflex.zip
index 18763576e..831278863 100644
--- a/spec/fixtures/reflex.zip
+++ b/spec/fixtures/reflex.zip
diff --git a/spec/tasks/reflex_rake_spec.rb b/spec/tasks/reflex_rake_spec.rb
index f1dc25bc8..f49eb41ec 100644
--- a/spec/tasks/reflex_rake_spec.rb
+++ b/spec/tasks/reflex_rake_spec.rb
@@ -12,7 +12,7 @@ describe 'reflex:sync' do
     # Must have an stop_area_referential
     create(:stop_area_referential, name: 'Reflex')
     Stif::ReflexSynchronization.synchronize
-    expect(Chouette::StopArea.count).to eq 7928
-    expect(Chouette::AccessPoint.count).to eq 60
+    expect(Chouette::StopArea.count).to eq 4309
+    expect(Chouette::AccessPoint.count).to eq 49
   end
 end