From c98b30a42901e90fa36b83b88574b85b4f998f28 Mon Sep 17 00:00:00 2001
From: jpl
Date: Mon, 11 Jul 2016 17:36:22 +0200
Subject: clean warbler subsisting bones, after removing jruby

---
 lib/tasks/warbler.rake | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100644 lib/tasks/warbler.rake

(limited to 'lib')

diff --git a/lib/tasks/warbler.rake b/lib/tasks/warbler.rake
deleted file mode 100644
index e69457583..000000000
--- a/lib/tasks/warbler.rake
+++ /dev/null
@@ -1,12 +0,0 @@
-begin
-  require 'warbler'
-  Warbler::Task.new
-
-  task "war:setup" do
-    ENV['RAILS_RELATIVE_URL_ROOT'] ||= "/chouette2"
-  end
-
-  task :war => ["war:setup", "assets:precompile"]
-rescue LoadError => e
-  puts "Failed to load Warbler. Make sure it's installed."
-end
-- 
cgit v1.2.3


From 6dc3c7a58d5c0fc0beb8a428616dce52e2361305 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Thu, 28 Jul 2016 15:05:03 +0200
Subject: add codif_line_synchronization #907

---
 lib/stif/codif_line_synchronization.rb | 81 ++++++++++++++++++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 lib/stif/codif_line_synchronization.rb

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
new file mode 100644
index 000000000..0c4c6e91e
--- /dev/null
+++ b/lib/stif/codif_line_synchronization.rb
@@ -0,0 +1,81 @@
+module Stif
+  module CodifLineSynchronization
+    class << self
+      # Don't check last synchronizations date and lines last update if first_sync
+      def synchronize first_sync = false
+        # Check last synchronization and synchronization interval
+        date = DateTime.now.to_date - LineReferential.first.sync_interval.days
+        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.last.try(:created_at)
+        return if (last_sync.nil? || last_sync.to_date > date) && !first_sync
+        
+        # TODO Check exceptions and status messages
+        begin
+          # Fetch Codifline operators and lines
+          client    = Codifligne::API.new
+          operators = client.operators
+          lines = client.lines
+
+          operators.map{ |o| create_or_update_company(o) }
+          lines.map{ |l| create_or_update_line(l) if first_sync || l.instance_variable_get(:@updated_at).to_date > last_sync.to_date}
+          
+          delete_deprecated_companies(operators)
+          deactivate_deprecated_lines(lines)
+
+          LineReferential.first.line_referential_sync.record_status "OK"
+        rescue Exception => e
+          ap e.message
+          LineReferential.first.line_referential_sync.record_status "Error"
+        end
+      end
+
+      def create_or_update_company(api_operator)
+        params = {
+          name: api_operator.name,
+          objectid: api_operator.stif_id
+        }
+        company = save_or_update(params, Chouette::Company)
+      end
+
+      def create_or_update_line(api_line)
+        params = {
+          name: api_line.name,
+          objectid: api_line.stif_id,
+          number: api_line.short_name,
+          deactivated: false
+        }
+        
+        # Find Company
+        # TODO Check behavior when operator_codes count is 0 or > 1
+        if api_line.operator_codes.any?
+          company_id = "STIF:CODIFLIGNE:Operator:" + api_line.operator_codes.first
+          params[:company] = Chouette::Company.find_by(objectid: company_id)
+        end
+
+        save_or_update(params, Chouette::Line)
+      end
+
+      def delete_deprecated_companies(operators)
+        ids = operators.map{ |o| o.stif_id }.to_a
+        Chouette::Company.where.not(objectid: ids).destroy_all
+      end
+
+      def deactivate_deprecated_lines(lines)
+        ids = lines.map{ |l| l.stif_id }.to_a
+        Chouette::Line.where.not(objectid: ids).map { |l| l.deactivated = true ; l.save }
+      end
+
+      def save_or_update(params, klass)
+        params[:line_referential] = LineReferential.first
+        object = klass.where(objectid: params[:objectid]).first
+        if object
+          object.assign_attributes(params)
+          object.save if object.changed?
+        else
+          object = klass.new(params)
+          object.save if object.valid?
+        end
+        object
+      end
+    end
+  end
+end
\ No newline at end of file
-- 
cgit v1.2.3


From 4d0bbfcf7aa6cf8cb313954363d925a823756844 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Mon, 1 Aug 2016 12:11:43 +0200
Subject: fix lines request and deactivated lines

---
 lib/stif/codif_line_synchronization.rb | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 0c4c6e91e..f14dc6f31 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -11,15 +11,16 @@ module Stif
         # TODO Check exceptions and status messages
         begin
           # Fetch Codifline operators and lines
-          client    = Codifligne::API.new
+          client = Codifligne::API.new
           operators = client.operators
-          lines = client.lines
+          # Get last sync date for lines if not first sync
+          date = first_sync ? 0 : last_sync.to_date.strftime("%d%m%Y")
+          lines = client.lines date: date
 
           operators.map{ |o| create_or_update_company(o) }
-          lines.map{ |l| create_or_update_line(l) if first_sync || l.instance_variable_get(:@updated_at).to_date > last_sync.to_date}
+          lines.map{ |l| create_or_update_line(l) }
           
           delete_deprecated_companies(operators)
-          deactivate_deprecated_lines(lines)
 
           LineReferential.first.line_referential_sync.record_status "OK"
         rescue Exception => e
@@ -41,7 +42,7 @@ module Stif
           name: api_line.name,
           objectid: api_line.stif_id,
           number: api_line.short_name,
-          deactivated: false
+          deactivated: (api_line.status == "inactive" ? true : false)
         }
         
         # Find Company
@@ -59,11 +60,6 @@ module Stif
         Chouette::Company.where.not(objectid: ids).destroy_all
       end
 
-      def deactivate_deprecated_lines(lines)
-        ids = lines.map{ |l| l.stif_id }.to_a
-        Chouette::Line.where.not(objectid: ids).map { |l| l.deactivated = true ; l.save }
-      end
-
       def save_or_update(params, klass)
         params[:line_referential] = LineReferential.first
         object = klass.where(objectid: params[:objectid]).first
-- 
cgit v1.2.3


From e270eb3a515d897b2157897e83bc5d4cdb670d62 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Mon, 1 Aug 2016 15:10:28 +0200
Subject: fix line synchronization

---
 lib/stif/codif_line_synchronization.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index f14dc6f31..9a8a92bce 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -13,14 +13,13 @@ module Stif
           # Fetch Codifline operators and lines
           client = Codifligne::API.new
           operators = client.operators
-          # Get last sync date for lines if not first sync
-          date = first_sync ? 0 : last_sync.to_date.strftime("%d%m%Y")
-          lines = client.lines date: date
+          lines = client.lines
 
           operators.map{ |o| create_or_update_company(o) }
           lines.map{ |l| create_or_update_line(l) }
           
           delete_deprecated_companies(operators)
+          delete_deprecated_lines(lines)
 
           LineReferential.first.line_referential_sync.record_status "OK"
         rescue Exception => e
@@ -60,6 +59,11 @@ module Stif
         Chouette::Company.where.not(objectid: ids).destroy_all
       end
 
+      def delete_deprecated_lines(lines)
+        ids = lines.map{ |l| l.stif_id }.to_a
+        Chouette::Line.where.not(objectid: ids).map { |l| l.deactivated = true ; l.save }
+      end
+
       def save_or_update(params, klass)
         params[:line_referential] = LineReferential.first
         object = klass.where(objectid: params[:objectid]).first
-- 
cgit v1.2.3


From fa7154a9fe684f8b42bfc4afcacaa56bc1fe6d45 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 3 Aug 2016 16:07:22 +0200
Subject: Remove device :confirmable

---
 lib/tasks/demo.rake | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/demo.rake b/lib/tasks/demo.rake
index fbafa96c1..d7e8c59f1 100644
--- a/lib/tasks/demo.rake
+++ b/lib/tasks/demo.rake
@@ -10,14 +10,13 @@ namespace :demo do
 
     organisation = Organisation.create!(:name => "DemoChouette")
     user = organisation.users.create( :name => "Demo", :email => "demo@chouette.mobi", :password => "chouette", :password_confirmation =>"chouette")
-    user.confirm!
     referential = organisation.referentials.create( :name => "Tatrobus", :slug => "tatrobus", :prefix => "TAT")
 
     #resource = Rack::Test::UploadedFile.new( Rails.application.config.demo_data, 'application/zip', false)
     #import_instance = ImportTask.new( :resources => resource, :referential_id => referential.id, :user_name => user.name, :no_save => false, :user_id => user.id, :name => "initialize demo", :data_format => "neptune")
     #import_instance.save
-	 
-	 File.open("/tmp/parameters_demo.json", "w")  { |file| 
+
+	 File.open("/tmp/parameters_demo.json", "w")  { |file|
 	                                           file.write('{
 "parameters" :  {
     "neptune-import": {
@@ -30,10 +29,10 @@ namespace :demo do
         }
 	 }
 }') }
-	 
+
 	 cmd = 'curl -F "file=@'+Rails.application.config.demo_data+';filename=tatrobus.zip" -F "file=@/tmp/parameters_demo.json;filename=parameters.json" http://localhost:8180/chouette_iev/referentials/tatrobus/importer/neptune'
 	 system(cmd)
-	 
+
     puts "Restore demo environment complete"
   end
 end
-- 
cgit v1.2.3


From 9a818eeb8991bf0e07f2c9a9837930d153a0492f Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Thu, 4 Aug 2016 16:05:06 +0200
Subject: Rake task organisations:sync

---
 lib/tasks/organisations.rake | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 lib/tasks/organisations.rake

(limited to 'lib')

diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
new file mode 100644
index 000000000..a887cd277
--- /dev/null
+++ b/lib/tasks/organisations.rake
@@ -0,0 +1,32 @@
+namespace :organisations do
+  def api_retrieve_organisation
+    last_sync = User.minimum(:synced_at)
+    conf      = Rails.application.config.stif_portail_api
+
+    conn = Faraday.new(:url => conf[:url]) do |c|
+      c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
+      c.request  :url_encoded
+      c.adapter  Faraday.default_adapter
+    end
+
+    response = conn.get '/api/v1/organizations'
+    JSON.parse response.body if response.status == 200
+  end
+
+  def sync_organisations data
+    data.each do |org|
+      Organisation.sync_or_create(code: org['code'], name: org['name']).tap do |organisation|
+        organisation.name      = org['name']
+        organisation.synced_at = Time.now
+        organisation.save if organisation.changed?
+        puts "Organisation #{organisation.name} has been updated"
+      end
+    end
+  end
+
+  desc "Sync organisations from stif portail"
+  task sync: :environment  do
+    data = api_retrieve_organisation
+    sync_organisations(data) if data
+  end
+end
-- 
cgit v1.2.3


From 7cfd4faefab17e556ac66dd0e52c5942b289ec62 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Fri, 5 Aug 2016 11:23:03 +0200
Subject: Rspec task organisations:sync

---
 lib/tasks/organisations.rake | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
index a887cd277..06e1d3da7 100644
--- a/lib/tasks/organisations.rake
+++ b/lib/tasks/organisations.rake
@@ -1,7 +1,7 @@
 namespace :organisations do
   def api_retrieve_organisation
-    last_sync = User.minimum(:synced_at)
-    conf      = Rails.application.config.stif_portail_api
+    conf = Rails.application.config.try(:stif_portail_api)
+    raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf
 
     conn = Faraday.new(:url => conf[:url]) do |c|
       c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
@@ -9,8 +9,8 @@ namespace :organisations do
       c.adapter  Faraday.default_adapter
     end
 
-    response = conn.get '/api/v1/organizations'
-    JSON.parse response.body if response.status == 200
+    resp = conn.get '/api/v1/organizations'
+    JSON.parse resp.body if resp.status == 200
   end
 
   def sync_organisations data
@@ -19,7 +19,7 @@ namespace :organisations do
         organisation.name      = org['name']
         organisation.synced_at = Time.now
         organisation.save if organisation.changed?
-        puts "Organisation #{organisation.name} has been updated"
+        puts "✓ Organisation #{organisation.name} has been updated" unless Rails.env.test?
       end
     end
   end
-- 
cgit v1.2.3


From 96b1abb62d9ae34c9ee7b334c374476b78f82b9b Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Fri, 5 Aug 2016 15:25:39 +0200
Subject: Refactoring - Rake task users:sync

---
 lib/tasks/organisations.rake | 13 ++++++-------
 lib/tasks/users.rake         | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 7 deletions(-)
 create mode 100644 lib/tasks/users.rake

(limited to 'lib')

diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
index 06e1d3da7..24b02b630 100644
--- a/lib/tasks/organisations.rake
+++ b/lib/tasks/organisations.rake
@@ -5,7 +5,6 @@ namespace :organisations do
 
     conn = Faraday.new(:url => conf[:url]) do |c|
       c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
-      c.request  :url_encoded
       c.adapter  Faraday.default_adapter
     end
 
@@ -14,12 +13,12 @@ namespace :organisations do
   end
 
   def sync_organisations data
-    data.each do |org|
-      Organisation.sync_or_create(code: org['code'], name: org['name']).tap do |organisation|
-        organisation.name      = org['name']
-        organisation.synced_at = Time.now
-        organisation.save if organisation.changed?
-        puts "✓ Organisation #{organisation.name} has been updated" unless Rails.env.test?
+    data.each do |el|
+      Organisation.find_or_create_by(code: el['code']).tap do |org|
+        org.name      = el['name']
+        org.synced_at = Time.now
+        org.save if org.changed?
+        puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test?
       end
     end
   end
diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
new file mode 100644
index 000000000..ca2264381
--- /dev/null
+++ b/lib/tasks/users.rake
@@ -0,0 +1,39 @@
+namespace :users do
+  def api_retrieve_user
+    conf = Rails.application.config.try(:stif_portail_api)
+    raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf
+
+    conn = Faraday.new(:url => conf[:url]) do |c|
+      c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
+      c.adapter  Faraday.default_adapter
+    end
+
+    resp = conn.get '/api/v1/users'
+    JSON.parse resp.body if resp.status == 200
+  end
+
+  def sync_users data
+    data.each do |el|
+      User.find_or_create_by(username: el['username']).tap do |user|
+        user.name         = "#{el['firstname']} #{el['lastname']}"
+        user.email        = el['email']
+        user.synced_at    = Time.now
+
+        # Set organisation
+        user.organisation = Organisation.find_or_create_by(code: el['organization_code']).tap do |org|
+          org.name      = el['organization_name']
+          org.synced_at = Time.now
+        end
+
+        user.save if user.changed?
+        puts "✓ user #{user.name} has been updated" unless Rails.env.test?
+      end
+    end
+  end
+
+  desc "Sync users from stif portail"
+  task sync: :environment  do
+    data = api_retrieve_user
+    sync_users(data) if data
+  end
+end
-- 
cgit v1.2.3


From dd253db7dc07b18b93242a9587019489b0823ea3 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Fri, 5 Aug 2016 16:12:30 +0200
Subject: Refactoring organisations:sync task

---
 lib/tasks/organisations.rake | 6 +++++-
 lib/tasks/users.rake         | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
index 24b02b630..94a9215b7 100644
--- a/lib/tasks/organisations.rake
+++ b/lib/tasks/organisations.rake
@@ -9,7 +9,11 @@ namespace :organisations do
     end
 
     resp = conn.get '/api/v1/organizations'
-    JSON.parse resp.body if resp.status == 200
+    if resp.status == 200
+      JSON.parse resp.body
+    else
+      raise "Error on api request status : #{resp.status} => #{resp.body}"
+    end
   end
 
   def sync_organisations data
diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
index ca2264381..15c76a240 100644
--- a/lib/tasks/users.rake
+++ b/lib/tasks/users.rake
@@ -9,7 +9,11 @@ namespace :users do
     end
 
     resp = conn.get '/api/v1/users'
-    JSON.parse resp.body if resp.status == 200
+    if resp.status == 200
+      JSON.parse resp.body
+    else
+      raise "Error on api request status : #{resp.status} => #{resp.body}"
+    end
   end
 
   def sync_users data
-- 
cgit v1.2.3


From ac6af39ce23b917e62b29b52396f8eb9e68e5e5c Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Mon, 8 Aug 2016 10:26:04 +0200
Subject: Set dev env portail api key

---
 lib/tasks/users.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
index 15c76a240..cc2f17103 100644
--- a/lib/tasks/users.rake
+++ b/lib/tasks/users.rake
@@ -30,7 +30,7 @@ namespace :users do
         end
 
         user.save if user.changed?
-        puts "✓ user #{user.name} has been updated" unless Rails.env.test?
+        puts "✓ user #{user.username} has been updated" unless Rails.env.test?
       end
     end
   end
-- 
cgit v1.2.3


From fabaf9b63e9cb7ecb0f4cd84bb15fda536e8f5b3 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Mon, 8 Aug 2016 11:16:59 +0200
Subject: Set :lockable to model User

---
 lib/tasks/users.rake | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
index cc2f17103..76250f0f0 100644
--- a/lib/tasks/users.rake
+++ b/lib/tasks/users.rake
@@ -21,6 +21,7 @@ namespace :users do
       User.find_or_create_by(username: el['username']).tap do |user|
         user.name         = "#{el['firstname']} #{el['lastname']}"
         user.email        = el['email']
+        user.locked_at    = el['locked_at']
         user.synced_at    = Time.now
 
         # Set organisation
-- 
cgit v1.2.3


From 53526748c12eb790647b656254b009a08278b70f Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 10 Aug 2016 10:43:13 +0200
Subject: Refactoring move portail sync code into models

---
 lib/tasks/organisations.rake | 31 +------------------------------
 lib/tasks/users.rake         | 40 +---------------------------------------
 2 files changed, 2 insertions(+), 69 deletions(-)

(limited to 'lib')

diff --git a/lib/tasks/organisations.rake b/lib/tasks/organisations.rake
index 94a9215b7..1b21d7119 100644
--- a/lib/tasks/organisations.rake
+++ b/lib/tasks/organisations.rake
@@ -1,35 +1,6 @@
 namespace :organisations do
-  def api_retrieve_organisation
-    conf = Rails.application.config.try(:stif_portail_api)
-    raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf
-
-    conn = Faraday.new(:url => conf[:url]) do |c|
-      c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
-      c.adapter  Faraday.default_adapter
-    end
-
-    resp = conn.get '/api/v1/organizations'
-    if resp.status == 200
-      JSON.parse resp.body
-    else
-      raise "Error on api request status : #{resp.status} => #{resp.body}"
-    end
-  end
-
-  def sync_organisations data
-    data.each do |el|
-      Organisation.find_or_create_by(code: el['code']).tap do |org|
-        org.name      = el['name']
-        org.synced_at = Time.now
-        org.save if org.changed?
-        puts "✓ Organisation #{org.name} has been updated" unless Rails.env.test?
-      end
-    end
-  end
-
   desc "Sync organisations from stif portail"
   task sync: :environment  do
-    data = api_retrieve_organisation
-    sync_organisations(data) if data
+    Organisation.portail_sync
   end
 end
diff --git a/lib/tasks/users.rake b/lib/tasks/users.rake
index 76250f0f0..c045639c1 100644
--- a/lib/tasks/users.rake
+++ b/lib/tasks/users.rake
@@ -1,44 +1,6 @@
 namespace :users do
-  def api_retrieve_user
-    conf = Rails.application.config.try(:stif_portail_api)
-    raise 'Rails.application.config.stif_portail_api settings is not defined' unless conf
-
-    conn = Faraday.new(:url => conf[:url]) do |c|
-      c.headers['Authorization'] = "Token token=\"#{conf[:key]}\""
-      c.adapter  Faraday.default_adapter
-    end
-
-    resp = conn.get '/api/v1/users'
-    if resp.status == 200
-      JSON.parse resp.body
-    else
-      raise "Error on api request status : #{resp.status} => #{resp.body}"
-    end
-  end
-
-  def sync_users data
-    data.each do |el|
-      User.find_or_create_by(username: el['username']).tap do |user|
-        user.name         = "#{el['firstname']} #{el['lastname']}"
-        user.email        = el['email']
-        user.locked_at    = el['locked_at']
-        user.synced_at    = Time.now
-
-        # Set organisation
-        user.organisation = Organisation.find_or_create_by(code: el['organization_code']).tap do |org|
-          org.name      = el['organization_name']
-          org.synced_at = Time.now
-        end
-
-        user.save if user.changed?
-        puts "✓ user #{user.username} has been updated" unless Rails.env.test?
-      end
-    end
-  end
-
   desc "Sync users from stif portail"
   task sync: :environment  do
-    data = api_retrieve_user
-    sync_users(data) if data
+    User.portail_sync
   end
 end
-- 
cgit v1.2.3


From b66235384cf777a7195463495b412df1a5f34b80 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Wed, 10 Aug 2016 17:51:42 +0200
Subject: add network and group of lines synchronization and create a rake task

---
 lib/stif/codif_line_synchronization.rb | 72 +++++++++++++++++++++++++++-------
 lib/tasks/codifligne.rake              |  6 +++
 2 files changed, 64 insertions(+), 14 deletions(-)
 create mode 100644 lib/tasks/codifligne.rake

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 9a8a92bce..65d82dc71 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -5,21 +5,27 @@ module Stif
       def synchronize first_sync = false
         # Check last synchronization and synchronization interval
         date = DateTime.now.to_date - LineReferential.first.sync_interval.days
-        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.last.try(:created_at)
+        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: 'ok').last.try(:created_at)
         return if (last_sync.nil? || last_sync.to_date > date) && !first_sync
         
         # TODO Check exceptions and status messages
         begin
-          # Fetch Codifline operators and lines
+          # Fetch Codifline data
           client = Codifligne::API.new
-          operators = client.operators
-          lines = client.lines
+          operators       = client.operators
+          lines           = client.lines
+          networks        = client.networks
+          groups_of_lines = client.groups_of_lines
 
-          operators.map{ |o| create_or_update_company(o) }
-          lines.map{ |l| create_or_update_line(l) }
+          operators.map       { |o| create_or_update_company(o) }
+          lines.map           { |l| create_or_update_line(l) }
+          networks.map        { |n| create_or_update_network(n) }
+          groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
           
-          delete_deprecated_companies(operators)
+          delete_deprecated(operators, Chouette::Company)
           delete_deprecated_lines(lines)
+          delete_deprecated(networks, Chouette::Network)
+          delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
 
           LineReferential.first.line_referential_sync.record_status "OK"
         rescue Exception => e
@@ -31,9 +37,10 @@ module Stif
       def create_or_update_company(api_operator)
         params = {
           name: api_operator.name,
-          objectid: api_operator.stif_id
+          objectid: api_operator.stif_id,
+          import_xml: api_operator.xml
         }
-        company = save_or_update(params, Chouette::Company)
+        save_or_update(params, Chouette::Company)
       end
 
       def create_or_update_line(api_line)
@@ -41,7 +48,8 @@ module Stif
           name: api_line.name,
           objectid: api_line.stif_id,
           number: api_line.short_name,
-          deactivated: (api_line.status == "inactive" ? true : false)
+          deactivated: (api_line.status == "inactive" ? true : false),
+          import_xml: api_line.xml
         }
         
         # Find Company
@@ -54,9 +62,45 @@ module Stif
         save_or_update(params, Chouette::Line)
       end
 
-      def delete_deprecated_companies(operators)
-        ids = operators.map{ |o| o.stif_id }.to_a
-        Chouette::Company.where.not(objectid: ids).destroy_all
+      def create_or_update_network(api_network)
+        params = {
+          name: api_network.name,
+          objectid: api_network.stif_id,
+          import_xml: api_network.xml
+        }
+
+        # Find Lines
+        params[:lines] = []
+        api_network.line_codes.each do |line|
+          line_id = "STIF:CODIFLIGNE:Line:" + line
+          params[:lines] << Chouette::Line.find_by(objectid: line_id)
+        end
+
+        save_or_update(params, Chouette::Network)
+      end
+
+      def create_or_update_group_of_lines(api_group_of_lines)
+        params = {
+          name: api_group_of_lines.name,
+          objectid: api_group_of_lines.stif_id,
+          import_xml: api_group_of_lines.xml
+        }
+
+        # Find Lines
+        params[:lines] = []
+        api_group_of_lines.line_codes.each do |line|
+          line_id = "STIF:CODIFLIGNE:Line:" + line
+          # TODO : handle when lines doesn't exist
+          chouette_line = Chouette::Line.find_by(objectid: line_id)
+          params[:lines] << chouette_line if chouette_line.present?
+        end
+
+        save_or_update(params, Chouette::GroupOfLine)
+      end
+
+      def delete_deprecated(objects, klass)
+        ids = objects.map{ |o| o.stif_id }.to_a
+        klass.where.not(objectid: ids).destroy_all
       end
 
       def delete_deprecated_lines(lines)
@@ -65,7 +109,7 @@ module Stif
       end
 
       def save_or_update(params, klass)
-        params[:line_referential] = LineReferential.first
+        params[:line_referential] = LineReferential.first unless klass == Chouette::Network
         object = klass.where(objectid: params[:objectid]).first
         if object
           object.assign_attributes(params)
diff --git a/lib/tasks/codifligne.rake b/lib/tasks/codifligne.rake
new file mode 100644
index 000000000..8b7a4f669
--- /dev/null
+++ b/lib/tasks/codifligne.rake
@@ -0,0 +1,6 @@
+namespace :codifligne do
+  desc "Sync lines, companies, networks, and group of lines from codifligne"
+  task sync: :environment  do
+    Stif::CodifLineSynchronization.synchronize
+  end
+end
-- 
cgit v1.2.3


From b2eeaa1eb9f8e4788ef1467bd6fbd2dd4aa6300c Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Fri, 19 Aug 2016 11:48:17 +0200
Subject: fix codiflinesynchronization and add better synchronization messages

---
 lib/stif/codif_line_synchronization.rb | 39 ++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 65d82dc71..77b9d1c02 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -1,13 +1,14 @@
 module Stif
   module CodifLineSynchronization
     class << self
-      # Don't check last synchronizations date and lines last update if first_sync
-      def synchronize first_sync = false
+      # Don't check last synchronizations if force_sync
+      def synchronize force_sync = false
         # Check last synchronization and synchronization interval
         date = DateTime.now.to_date - LineReferential.first.sync_interval.days
-        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: 'ok').last.try(:created_at)
-        return if (last_sync.nil? || last_sync.to_date > date) && !first_sync
+        last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: :ok).last.try(:created_at)
+        return if last_sync.present? && last_sync.to_date > date && !force_sync
         
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
         # TODO Check exceptions and status messages
         begin
           # Fetch Codifline data
@@ -22,15 +23,21 @@ module Stif
           networks.map        { |n| create_or_update_network(n) }
           groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
           
-          delete_deprecated(operators, Chouette::Company)
-          delete_deprecated_lines(lines)
-          delete_deprecated(networks, Chouette::Network)
-          delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
+          total_deleted = delete_deprecated(operators, Chouette::Company)
+          total_deleted += delete_deprecated_lines(lines)
+          total_deleted += delete_deprecated(networks, Chouette::Network)
+          total_deleted += delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
 
-          LineReferential.first.line_referential_sync.record_status "OK"
+          # Building log message
+          total_codifligne_elements = operators.count + lines.count + networks.count + groups_of_lines.count
+          total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+
+          LineReferential.first.line_referential_sync.record_status :ok, I18n.t('synchronization.message.success', time: total_time, imported: total_codifligne_elements, deleted: total_deleted)
         rescue Exception => e
-          ap e.message
-          LineReferential.first.line_referential_sync.record_status "Error"
+          Rails.logger.error "#{e} #{e.backtrace}"
+
+          total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+          LineReferential.first.line_referential_sync.record_status :ko, I18n.t('synchronization.message.failure', time: total_time)
         end
       end
 
@@ -100,12 +107,18 @@ module Stif
 
       def delete_deprecated(objects, klass)
         ids = objects.map{ |o| o.stif_id }.to_a
-        klass.where.not(objectid: ids).destroy_all
+        deprecated = klass.where.not(objectid: ids)
+        count = deprecated.count
+        deprecated.destroy_all
+        count
       end
 
       def delete_deprecated_lines(lines)
         ids = lines.map{ |l| l.stif_id }.to_a
-        Chouette::Line.where.not(objectid: ids).map { |l| l.deactivated = true ; l.save }
+        deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false)
+        count = deprecated.count
+        deprecated.map { |l| l.deactivated = true ; l.save }
+        count
       end
 
       def save_or_update(params, klass)
-- 
cgit v1.2.3


From 8b3f3055ec5ee19bdaf9f334c83cfcdd01c7ed0b Mon Sep 17 00:00:00 2001
From: jpl
Date: Tue, 23 Aug 2016 11:28:06 +0200
Subject: Refs #1482: adding cucumber for tests

---
 lib/tasks/ci.rake       |  4 +--
 lib/tasks/cucumber.rake | 76 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 lib/tasks/cucumber.rake

(limited to 'lib')

diff --git a/lib/tasks/ci.rake b/lib/tasks/ci.rake
index beb3b19d1..b6c7d2352 100644
--- a/lib/tasks/ci.rake
+++ b/lib/tasks/ci.rake
@@ -23,7 +23,7 @@ namespace :ci do
 
   desc "Deploy after CI"
   task :deploy do
-    sh "cap #{deploy_env} deploy:migrations deploy:seed"
+    sh "cap #{deploy_env} deploy:migrations deploy:seed deploy:cucumber"
   end
 
   desc "Clean test files"
@@ -33,4 +33,4 @@ namespace :ci do
 end
 
 desc "Run continuous integration tasks (spec, ...)"
-task :ci => ["ci:setup", "spec", "ci:deploy", "ci:clean"]
+task :ci => ["ci:setup", "spec", "cucumber", "ci:deploy", "ci:clean"]
diff --git a/lib/tasks/cucumber.rake b/lib/tasks/cucumber.rake
new file mode 100644
index 000000000..23d9b3eb4
--- /dev/null
+++ b/lib/tasks/cucumber.rake
@@ -0,0 +1,76 @@
+# IMPORTANT: This file is generated by cucumber-rails - edit at your own peril.
+# It is recommended to regenerate this file in the future when you upgrade to a
+# newer version of cucumber-rails. Consider adding your own code to a new file
+# instead of editing this one. Cucumber will automatically load all features/**/*.rb
+# files.
+
+
+unless ARGV.any? {|a| a =~ /^gems/} # Don't load anything when running the gems:* tasks
+
+vendored_cucumber_bin = Dir["#{Rails.root}/vendor/{gems,plugins}/cucumber*/bin/cucumber"].first
+$LOAD_PATH.unshift(File.dirname(vendored_cucumber_bin) + '/../lib') unless vendored_cucumber_bin.nil?
+
+begin
+  require 'cucumber/rake/task'
+
+  namespace :cucumber do
+    Cucumber::Rake::Task.new({:ok => 'test:prepare'}, 'Run features that should pass') do |t|
+      t.binary = vendored_cucumber_bin # If nil, the gem's binary is used.
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'default'
+    end
+
+    Cucumber::Rake::Task.new({:wip => 'test:prepare'}, 'Run features that are being worked on') do |t|
+      t.binary = vendored_cucumber_bin
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'wip'
+    end
+
+    Cucumber::Rake::Task.new({:rerun => 'test:prepare'}, 'Record failing features and run only them if any exist') do |t|
+      t.binary = vendored_cucumber_bin
+      t.fork = true # You may get faster startup if you set this to false
+      t.profile = 'rerun'
+    end
+
+    desc 'Run all features'
+    task :all => [:ok, :wip]
+
+    task :statsetup do
+      require 'rails/code_statistics'
+      ::STATS_DIRECTORIES << %w(Cucumber\ features features) if File.exist?('features')
+      ::CodeStatistics::TEST_TYPES << "Cucumber features" if File.exist?('features')
+    end
+
+    task :annotations_setup do
+      Rails.application.configure do
+        if config.respond_to?(:annotations)
+          config.annotations.directories << 'features'
+          config.annotations.register_extensions('feature') { |tag| /#\s*(#{tag}):?\s*(.*)$/ }
+        end
+      end
+    end
+  end
+  desc 'Alias for cucumber:ok'
+  task :cucumber => 'cucumber:ok'
+
+  task :default => :cucumber
+
+  task :features => :cucumber do
+    STDERR.puts "*** The 'features' task is deprecated. See rake -T cucumber ***"
+  end
+
+  # In case we don't have the generic Rails test:prepare hook, append a no-op task that we can depend upon.
+  task 'test:prepare' do
+  end
+
+  task :stats => 'cucumber:statsetup'
+
+  task :notes => 'cucumber:annotations_setup'
+rescue LoadError
+  desc 'cucumber rake task not available (cucumber not installed)'
+  task :cucumber do
+    abort 'Cucumber rake task is not available. Be sure to install cucumber as a gem or plugin'
+  end
+end
+
+end
-- 
cgit v1.2.3


From 98debbeaac00dbf1aa2c752521d5878776c540dc Mon Sep 17 00:00:00 2001
From: jpl
Date: Tue, 23 Aug 2016 11:41:56 +0200
Subject: Refs #1482: adding cucumber for tests

---
 lib/tasks/ci.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/tasks/ci.rake b/lib/tasks/ci.rake
index b6c7d2352..7fca878b6 100644
--- a/lib/tasks/ci.rake
+++ b/lib/tasks/ci.rake
@@ -23,7 +23,7 @@ namespace :ci do
 
   desc "Deploy after CI"
   task :deploy do
-    sh "cap #{deploy_env} deploy:migrations deploy:seed deploy:cucumber"
+    sh "cap #{deploy_env} deploy:migrations deploy:seed"
   end
 
   desc "Clean test files"
-- 
cgit v1.2.3


From d71549685f124fd895b01884f7d236cf09b96e85 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 31 Aug 2016 11:03:22 +0200
Subject: Wip sync reflex - add gem - task reflex:sync + whenever cfg

---
 lib/stif/reflex_synchronization.rb | 69 ++++++++++++++++++++++++++++++++++++++
 lib/tasks/reflex.rake              |  8 +++++
 2 files changed, 77 insertions(+)
 create mode 100644 lib/stif/reflex_synchronization.rb
 create mode 100644 lib/tasks/reflex.rake

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
new file mode 100644
index 000000000..ce663396b
--- /dev/null
+++ b/lib/stif/reflex_synchronization.rb
@@ -0,0 +1,69 @@
+module Stif
+  module ReflexSynchronization
+    class << self
+      def defaut_referential
+        StopAreaReferential.find_by(name: "Reflex")
+      end
+
+      # Todo remove dummy objectid
+      def find_by_object_id objectid
+        Chouette::StopArea.find_by(objectid: "dummy:StopArea:#{objectid.tr(':', '')}")
+      end
+
+      def synchronize_stop_area
+        client = Reflex::API.new
+        ['getOR', 'getOP'].each do |method|
+          results = client.process method
+          results = results[:Quay].merge(results[:StopPlace])
+
+          processed = []
+          results.each do |id, entry|
+            Rails.logger.debug "Reflex - Processing - #{entry.id}"
+            processed << self.create_or_update_stop_area(entry)
+          end
+          processed.each do |entry|
+            Rails.logger.debug "Reflex - Set parent for - #{entry.id}"
+            self.set_parent entry
+          end
+        end
+      end
+
+      def set_parent entry
+        return false unless entry.try(:parent_site_ref) || entry.try(:quays)
+        stop = self.find_by_object_id entry.id
+
+        if entry.try(:parent_site_ref)
+          stop.parent = self.find_by_object_id entry.parent_site_ref
+          stop.save! if stop.changed
+        end
+
+        if entry.try(:quays)
+          entry.quays.each do |quay|
+            children = self.find_by_object_id(quay[:ref])
+            next unless children
+            children.parent = stop
+            children.save! if children.changed?
+          end
+        end
+      end
+
+      def create_or_update_stop_area entry
+        stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
+        stop.stop_area_referential = self.defaut_referential
+
+        stop.name          = entry.name
+        stop.creation_time = entry.created
+        stop.area_type     = entry.area_type
+        # stop.object_version = entry.version
+        stop.zip_code  = entry.postal_code
+        stop.city_name = entry.city
+
+        if stop.changed?
+          Rails.logger.debug "Reflex - Updating - #{entry.id}"
+          stop.save!
+          stop
+        end
+      end
+    end
+  end
+end
diff --git a/lib/tasks/reflex.rake b/lib/tasks/reflex.rake
new file mode 100644
index 000000000..4ac557aa6
--- /dev/null
+++ b/lib/tasks/reflex.rake
@@ -0,0 +1,8 @@
+namespace :reflex do
+  desc "Sync data from Reflex api"
+  task sync: :environment  do
+    start = Time.now
+    Stif::ReflexSynchronization.synchronize_stop_area
+    Rails.logger.debug "Reflex-api sync done in #{Time.now - start} seconds !"
+  end
+end
-- 
cgit v1.2.3


From ae25f7bdd87a8f90cfede723e87f8291855d3d19 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 31 Aug 2016 12:26:35 +0200
Subject: AccessPoint from reflex StopPlaceEntrance

---
 lib/stif/reflex_synchronization.rb | 42 +++++++++++++++++++++++---------------
 lib/tasks/reflex.rake              |  2 +-
 2 files changed, 26 insertions(+), 18 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index ce663396b..8886fea8e 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -10,31 +10,31 @@ module Stif
         Chouette::StopArea.find_by(objectid: "dummy:StopArea:#{objectid.tr(':', '')}")
       end
 
-      def synchronize_stop_area
+      def synchronize
         client = Reflex::API.new
         ['getOR', 'getOP'].each do |method|
-          results = client.process method
-          results = results[:Quay].merge(results[:StopPlace])
-
+          results   = client.process method
           processed = []
-          results.each do |id, entry|
-            Rails.logger.debug "Reflex - Processing - #{entry.id}"
+
+          results[:StopPlaceEntrance].each do |id, entry|
+            self.create_or_update_access_point entry
+          end
+          results[:Quay].merge(results[:StopPlace]).each do |id, entry|
             processed << self.create_or_update_stop_area(entry)
           end
           processed.each do |entry|
-            Rails.logger.debug "Reflex - Set parent for - #{entry.id}"
-            self.set_parent entry
+            self.stop_area_set_parent entry
           end
         end
       end
 
-      def set_parent entry
+      def stop_area_set_parent entry
         return false unless entry.try(:parent_site_ref) || entry.try(:quays)
         stop = self.find_by_object_id entry.id
 
         if entry.try(:parent_site_ref)
           stop.parent = self.find_by_object_id entry.parent_site_ref
-          stop.save! if stop.changed
+          stop.save if stop.changed
         end
 
         if entry.try(:quays)
@@ -42,11 +42,21 @@ module Stif
             children = self.find_by_object_id(quay[:ref])
             next unless children
             children.parent = stop
-            children.save! if children.changed?
+            children.save if children.changed?
           end
         end
       end
 
+      def create_or_update_access_point entry
+        access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
+        access.name           = entry.name
+        # access.object_version = entry.version
+        access.zip_code       = entry.postal_code
+        access.city_name      = entry.city
+        access.access_type    = entry.area_type
+        access.save if access.changed?
+      end
+
       def create_or_update_stop_area entry
         stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
         stop.stop_area_referential = self.defaut_referential
@@ -54,15 +64,13 @@ module Stif
         stop.name          = entry.name
         stop.creation_time = entry.created
         stop.area_type     = entry.area_type
+        # Todo fixe object_version auto incremented
+        # by DefaultAttributesSupport prepare_auto_columns
         # stop.object_version = entry.version
         stop.zip_code  = entry.postal_code
         stop.city_name = entry.city
-
-        if stop.changed?
-          Rails.logger.debug "Reflex - Updating - #{entry.id}"
-          stop.save!
-          stop
-        end
+        stop.save if stop.changed?
+        stop
       end
     end
   end
diff --git a/lib/tasks/reflex.rake b/lib/tasks/reflex.rake
index 4ac557aa6..0bc3953a8 100644
--- a/lib/tasks/reflex.rake
+++ b/lib/tasks/reflex.rake
@@ -2,7 +2,7 @@ namespace :reflex do
   desc "Sync data from Reflex api"
   task sync: :environment  do
     start = Time.now
-    Stif::ReflexSynchronization.synchronize_stop_area
+    Stif::ReflexSynchronization.synchronize
     Rails.logger.debug "Reflex-api sync done in #{Time.now - start} seconds !"
   end
 end
-- 
cgit v1.2.3


From c3d262e691c3301243be89e498c8d8034c9f4b53 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 31 Aug 2016 14:34:52 +0200
Subject: Fix memory leak reflex:sync task

---
 lib/stif/reflex_synchronization.rb | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 8886fea8e..fd2f497e2 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -13,18 +13,21 @@ module Stif
       def synchronize
         client = Reflex::API.new
         ['getOR', 'getOP'].each do |method|
-          results   = client.process method
-          processed = []
+          results    = client.process method
+          stop_areas = results[:Quay].merge(results[:StopPlace])
 
           results[:StopPlaceEntrance].each do |id, entry|
             self.create_or_update_access_point entry
           end
-          results[:Quay].merge(results[:StopPlace]).each do |id, entry|
-            processed << self.create_or_update_stop_area(entry)
+          Rails.logger.debug "Reflex:sync - StopPlaceEntrance sync done !"
+          stop_areas.each do |id, entry|
+            self.create_or_update_stop_area entry
           end
-          processed.each do |entry|
+          Rails.logger.debug "Reflex:sync - StopAreas sync done !"
+          stop_areas.each do |id, entry|
             self.stop_area_set_parent entry
           end
+          Rails.logger.debug "Reflex:sync - StopAreas : set  parents sync done !"
         end
       end
 
-- 
cgit v1.2.3


From a71d5b8ff5804094bcbe1c945ad8fde02ba7f9f0 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 31 Aug 2016 15:23:31 +0200
Subject: Reflex:sync log processing time

---
 lib/stif/reflex_synchronization.rb | 23 ++++++++++++++++++-----
 lib/tasks/reflex.rake              |  2 +-
 2 files changed, 19 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index fd2f497e2..2da8664a9 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -11,23 +11,36 @@ module Stif
       end
 
       def synchronize
+        start  = Time.now
         client = Reflex::API.new
+
         ['getOR', 'getOP'].each do |method|
-          results    = client.process method
-          stop_areas = results[:Quay].merge(results[:StopPlace])
+          results = client.process method
+          Rails.logger.info "Reflex:sync - Process #{method} done in #{Time.now - start} seconds"
+          results.each do |type, entries|
+            Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
+          end
 
+          start = Time.now
           results[:StopPlaceEntrance].each do |id, entry|
             self.create_or_update_access_point entry
           end
-          Rails.logger.debug "Reflex:sync - StopPlaceEntrance sync done !"
+          Rails.logger.info "Reflex:sync - Create or update AccessPoint done in #{Time.now - start} seconds"
+
+          # Create or update stop_area for every quay, stop_place
+          stop_areas = results[:Quay].merge(results[:StopPlace])
+          start = Time.now
           stop_areas.each do |id, entry|
             self.create_or_update_stop_area entry
           end
-          Rails.logger.debug "Reflex:sync - StopAreas sync done !"
+          Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
+
+          # Walk through every entry and set parent stop_area
+          start = Time.now
           stop_areas.each do |id, entry|
             self.stop_area_set_parent entry
           end
-          Rails.logger.debug "Reflex:sync - StopAreas : set  parents sync done !"
+          Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
         end
       end
 
diff --git a/lib/tasks/reflex.rake b/lib/tasks/reflex.rake
index 0bc3953a8..0020f1b60 100644
--- a/lib/tasks/reflex.rake
+++ b/lib/tasks/reflex.rake
@@ -3,6 +3,6 @@ namespace :reflex do
   task sync: :environment  do
     start = Time.now
     Stif::ReflexSynchronization.synchronize
-    Rails.logger.debug "Reflex-api sync done in #{Time.now - start} seconds !"
+    Rails.logger.debug "Reflex:sync done in #{Time.now - start} seconds !"
   end
 end
-- 
cgit v1.2.3


From 6c4bfff03eba511f6bf4a032473f9441c3f8913c Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Thu, 1 Sep 2016 13:19:12 +0200
Subject: Refactoring StopPlaceEntrance to AcessPoint

---
 lib/stif/reflex_synchronization.rb | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 2da8664a9..3b43fd5df 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -21,12 +21,6 @@ module Stif
             Rails.logger.info "Reflex:sync - #{entries.count} #{type} retrieved"
           end
 
-          start = Time.now
-          results[:StopPlaceEntrance].each do |id, entry|
-            self.create_or_update_access_point entry
-          end
-          Rails.logger.info "Reflex:sync - Create or update AccessPoint done in #{Time.now - start} seconds"
-
           # Create or update stop_area for every quay, stop_place
           stop_areas = results[:Quay].merge(results[:StopPlace])
           start = Time.now
@@ -63,10 +57,12 @@ module Stif
         end
       end
 
-      def create_or_update_access_point entry
+      def create_or_update_access_point entry, stop_area
         access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
+        entry.version = entry.version.to_i + 1  if access.persisted?
         access.name           = entry.name
-        # access.object_version = entry.version
+        access.stop_area      = stop_area
+        access.object_version = entry.version
         access.zip_code       = entry.postal_code
         access.city_name      = entry.city
         access.access_type    = entry.area_type
@@ -75,17 +71,24 @@ module Stif
 
       def create_or_update_stop_area entry
         stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
-        stop.stop_area_referential = self.defaut_referential
+        # Hack, on save object_version will be incremented by 1
+        entry.version = entry.version.to_i + 1  if stop.persisted?
 
-        stop.name          = entry.name
-        stop.creation_time = entry.created
-        stop.area_type     = entry.area_type
-        # Todo fixe object_version auto incremented
-        # by DefaultAttributesSupport prepare_auto_columns
-        # stop.object_version = entry.version
-        stop.zip_code  = entry.postal_code
-        stop.city_name = entry.city
+        stop.stop_area_referential = self.defaut_referential
+        stop.name           = entry.name
+        stop.creation_time  = entry.created
+        stop.area_type      = entry.area_type
+        stop.object_version = entry.version
+        stop.zip_code       = entry.postal_code
+        stop.city_name      = entry.city
         stop.save if stop.changed?
+
+        # Create AccessPoint from StopPlaceEntrance
+        if entry.try(:entrances)
+          entry.entrances.each do |entrance|
+            self.create_or_update_access_point entrance, stop
+          end
+        end
         stop
       end
     end
-- 
cgit v1.2.3


From 94f040eb5812c366ae5916cc59e72183fb8d77ef Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Thu, 1 Sep 2016 16:12:32 +0200
Subject: add logs to codifligne sync

---
 lib/stif/codif_line_synchronization.rb | 58 ++++++++++++++++++++++++++++++----
 1 file changed, 51 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 77b9d1c02..001572af6 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -18,25 +18,55 @@ module Stif
           networks        = client.networks
           groups_of_lines = client.groups_of_lines
 
+          Rails.logger.info "Codifligne:sync - Codifligne request processed in #{elapsed_time_since start_time} seconds"
+
+          # Create or update Companies
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           operators.map       { |o| create_or_update_company(o) }
+          log_create_or_update "Companies", operators.count, stime
+
+          # Create or update Lines
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           lines.map           { |l| create_or_update_line(l) }
+          log_create_or_update "Lines", lines.count, stime
+
+          # Create or update Networks
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           networks.map        { |n| create_or_update_network(n) }
+          log_create_or_update "Networks", networks.count, stime
+
+          # Create or update Group of lines
+          stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
+          log_create_or_update "Group of lines", group_of_lines.count, stime
+
+          
+          # Delete deprecated Group of lines
+          deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
+          log_deleted "Group of lines", deleted_gr unless deleted_op == 0
+
+          # Delete deprecated Networks
+          deleted_ne = delete_deprecated(networks, Chouette::Network)
+          log_deleted "Networks", deleted_ne unless deleted_op == 0
           
-          total_deleted = delete_deprecated(operators, Chouette::Company)
-          total_deleted += delete_deprecated_lines(lines)
-          total_deleted += delete_deprecated(networks, Chouette::Network)
-          total_deleted += delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
+          # Delete deprecated Lines
+          deleted_li = delete_deprecated_lines(lines)
+          log_deleted "Lines", deleted_li unless deleted_op == 0
+          
+          # Delete deprecated Operators
+          deleted_op = delete_deprecated(operators, Chouette::Company)
+          log_deleted "Operators", deleted_op unless deleted_op == 0
 
           # Building log message
           total_codifligne_elements = operators.count + lines.count + networks.count + groups_of_lines.count
-          total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+          total_deleted = deleted_op + deleted_li + deleted_ne + deleted_gr
+          total_time = elapsed_time_since start_time
 
           LineReferential.first.line_referential_sync.record_status :ok, I18n.t('synchronization.message.success', time: total_time, imported: total_codifligne_elements, deleted: total_deleted)
         rescue Exception => e
-          Rails.logger.error "#{e} #{e.backtrace}"
+          total_time = elapsed_time_since start_time
 
-          total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+          Rails.logger.error "Codifligne:sync - Error: #{e}, ended after #{total_time} seconds"
           LineReferential.first.line_referential_sync.record_status :ko, I18n.t('synchronization.message.failure', time: total_time)
         end
       end
@@ -133,6 +163,20 @@ module Stif
         end
         object
       end
+
+      def elapsed_time_since start_time = 0
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) - start_time
+      end
+
+      def log_create_or_update name, count, start_time
+        time = elapsed_time_since start_time
+        Rails.logger.info "Codifligne:sync - #{count} #{name} retrieved"
+        Rails.logger.info "Codifligne:sync - Create or update #{name} done in #{time} seconds"
+      end
+
+      def log_deleted name, count
+        Rails.logger.info "Codifligne:sync - #{count} #{name} deleted"
+      end
     end
   end
 end
\ No newline at end of file
-- 
cgit v1.2.3


From d54645efac3c60cca069888e72d47d52646c6f20 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Fri, 2 Sep 2016 10:03:51 +0200
Subject: fix some bugs in codifli_line_synchronization

---
 lib/stif/codif_line_synchronization.rb | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 001572af6..6b0ba939d 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -38,20 +38,19 @@ module Stif
           # Create or update Group of lines
           stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
-          log_create_or_update "Group of lines", group_of_lines.count, stime
-
+          log_create_or_update "Group of lines", groups_of_lines.count, stime
           
           # Delete deprecated Group of lines
           deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
-          log_deleted "Group of lines", deleted_gr unless deleted_op == 0
+          log_deleted "Group of lines", deleted_gr unless deleted_gr == 0
 
           # Delete deprecated Networks
           deleted_ne = delete_deprecated(networks, Chouette::Network)
-          log_deleted "Networks", deleted_ne unless deleted_op == 0
+          log_deleted "Networks", deleted_ne unless deleted_ne == 0
           
           # Delete deprecated Lines
           deleted_li = delete_deprecated_lines(lines)
-          log_deleted "Lines", deleted_li unless deleted_op == 0
+          log_deleted "Lines", deleted_li unless deleted_li == 0
           
           # Delete deprecated Operators
           deleted_op = delete_deprecated(operators, Chouette::Company)
-- 
cgit v1.2.3


From 4ae538b87f180d69df8768d6ffdd067a3b1beab5 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Fri, 2 Sep 2016 11:45:19 +0200
Subject: Reflex store xml + purge deleted since last sync

---
 lib/stif/reflex_synchronization.rb | 52 +++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 3b43fd5df..580a41c66 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -11,8 +11,9 @@ module Stif
       end
 
       def synchronize
-        start  = Time.now
-        client = Reflex::API.new
+        start      = Time.now
+        client     = Reflex::API.new
+        processed  = []
 
         ['getOR', 'getOP'].each do |method|
           results = client.process method
@@ -25,7 +26,7 @@ module Stif
           stop_areas = results[:Quay].merge(results[:StopPlace])
           start = Time.now
           stop_areas.each do |id, entry|
-            self.create_or_update_stop_area entry
+            processed << self.create_or_update_stop_area(entry).objectid
           end
           Rails.logger.info "Reflex:sync - Create or update StopArea done in #{Time.now - start} seconds"
 
@@ -36,6 +37,19 @@ module Stif
           end
           Rails.logger.info "Reflex:sync - StopArea set parent done in #{Time.now - start} seconds"
         end
+        # Purge deleted stop_area
+        self.set_deleted_stop_area processed.uniq
+      end
+
+      def set_deleted_stop_area processed
+        Rails.logger.info "Reflex:sync - StopArea start deleted_stop_area"
+        start   = Time.now
+        deleted = Chouette::StopArea.pluck(:objectid).uniq - processed
+        deleted.each_slice(50) do |object_ids|
+          Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now)
+        end
+        Rails.logger.info "Reflex:sync - StopArea #{deleted.size} stop_area deleted since last sync"
+        Rails.logger.info "Reflex:sync - StopArea purge deleted in #{Time.now - start} seconds"
       end
 
       def stop_area_set_parent entry
@@ -60,12 +74,15 @@ module Stif
       def create_or_update_access_point entry, stop_area
         access = Chouette::AccessPoint.find_or_create_by(objectid: "dummy:AccessPoint:#{entry.id.tr(':', '')}")
         entry.version = entry.version.to_i + 1  if access.persisted?
-        access.name           = entry.name
-        access.stop_area      = stop_area
-        access.object_version = entry.version
-        access.zip_code       = entry.postal_code
-        access.city_name      = entry.city
-        access.access_type    = entry.area_type
+        access.stop_area = stop_area
+        {
+          :name           => :name,
+          :access_type    => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| access[k] = entry.try(v) end
         access.save if access.changed?
       end
 
@@ -73,16 +90,17 @@ module Stif
         stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
         # Hack, on save object_version will be incremented by 1
         entry.version = entry.version.to_i + 1  if stop.persisted?
-
         stop.stop_area_referential = self.defaut_referential
-        stop.name           = entry.name
-        stop.creation_time  = entry.created
-        stop.area_type      = entry.area_type
-        stop.object_version = entry.version
-        stop.zip_code       = entry.postal_code
-        stop.city_name      = entry.city
+        {
+          :name           => :name,
+          :creation_time  => :created,
+          :area_type      => :area_type,
+          :object_version => :version,
+          :zip_code       => :postal_code,
+          :city_name      => :city,
+          :import_xml     => :xml
+        }.each do |k, v| stop[k] = entry.try(v) end
         stop.save if stop.changed?
-
         # Create AccessPoint from StopPlaceEntrance
         if entry.try(:entrances)
           entry.entrances.each do |entrance|
-- 
cgit v1.2.3


From bc805a0873f8c6ae0df8a8b1fbec3f9a7c23dac2 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Fri, 2 Sep 2016 16:13:08 +0200
Subject: Fix reflex:sync stop_area_set_parent undefined parent

---
 lib/stif/reflex_synchronization.rb | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index 580a41c66..dea590eea 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -55,6 +55,7 @@ module Stif
       def stop_area_set_parent entry
         return false unless entry.try(:parent_site_ref) || entry.try(:quays)
         stop = self.find_by_object_id entry.id
+        return false unless stop
 
         if entry.try(:parent_site_ref)
           stop.parent = self.find_by_object_id entry.parent_site_ref
-- 
cgit v1.2.3


From fe540808f7a52f997ebc25441a5626a3903a777c Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Mon, 5 Sep 2016 12:17:22 +0200
Subject: fix codifligne synchronization for networks

---
 lib/stif/codif_line_synchronization.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 6b0ba939d..1c0a45a75 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -151,7 +151,7 @@ module Stif
       end
 
       def save_or_update(params, klass)
-        params[:line_referential] = LineReferential.first unless klass == Chouette::Network
+        params[:line_referential] = LineReferential.first
         object = klass.where(objectid: params[:objectid]).first
         if object
           object.assign_attributes(params)
-- 
cgit v1.2.3


From ce3640d67b28eb2193659a78af04cd7d52cfd320 Mon Sep 17 00:00:00 2001
From: Xinhui
Date: Wed, 7 Sep 2016 10:39:26 +0200
Subject: Reflex:sync default deleted_at to nil

---
 lib/stif/reflex_synchronization.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/stif/reflex_synchronization.rb b/lib/stif/reflex_synchronization.rb
index dea590eea..55fe3d404 100644
--- a/lib/stif/reflex_synchronization.rb
+++ b/lib/stif/reflex_synchronization.rb
@@ -44,7 +44,7 @@ module Stif
       def set_deleted_stop_area processed
         Rails.logger.info "Reflex:sync - StopArea start deleted_stop_area"
         start   = Time.now
-        deleted = Chouette::StopArea.pluck(:objectid).uniq - processed
+        deleted = Chouette::StopArea.where(deleted_at: nil).pluck(:objectid).uniq - processed
         deleted.each_slice(50) do |object_ids|
           Chouette::StopArea.where(objectid: object_ids).update_all(deleted_at: Time.now)
         end
@@ -91,6 +91,7 @@ module Stif
         stop = Chouette::StopArea.find_or_create_by(objectid: "dummy:StopArea:#{entry.id.tr(':', '')}")
         # Hack, on save object_version will be incremented by 1
         entry.version = entry.version.to_i + 1  if stop.persisted?
+        stop.deleted_at            = nil
         stop.stop_area_referential = self.defaut_referential
         {
           :name           => :name,
-- 
cgit v1.2.3


From c9bc203720a5591d6118e203818c7dd211ae3a51 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Wed, 7 Sep 2016 11:35:49 +0200
Subject: improve delete deprecated methods

---
 lib/stif/codif_line_synchronization.rb | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index 1c0a45a75..fa27e9a38 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -7,7 +7,7 @@ module Stif
         date = DateTime.now.to_date - LineReferential.first.sync_interval.days
         last_sync = LineReferential.first.line_referential_sync.line_sync_operations.where(status: :ok).last.try(:created_at)
         return if last_sync.present? && last_sync.to_date > date && !force_sync
-        
+
         start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
         # TODO Check exceptions and status messages
         begin
@@ -39,7 +39,7 @@ module Stif
           stime = Process.clock_gettime(Process::CLOCK_MONOTONIC, :second)
           groups_of_lines.map { |g| create_or_update_group_of_lines(g) }
           log_create_or_update "Group of lines", groups_of_lines.count, stime
-          
+
           # Delete deprecated Group of lines
           deleted_gr = delete_deprecated(groups_of_lines, Chouette::GroupOfLine)
           log_deleted "Group of lines", deleted_gr unless deleted_gr == 0
@@ -47,11 +47,11 @@ module Stif
           # Delete deprecated Networks
           deleted_ne = delete_deprecated(networks, Chouette::Network)
           log_deleted "Networks", deleted_ne unless deleted_ne == 0
-          
+
           # Delete deprecated Lines
           deleted_li = delete_deprecated_lines(lines)
           log_deleted "Lines", deleted_li unless deleted_li == 0
-          
+
           # Delete deprecated Operators
           deleted_op = delete_deprecated(operators, Chouette::Company)
           log_deleted "Operators", deleted_op unless deleted_op == 0
@@ -87,7 +87,7 @@ module Stif
           deactivated: (api_line.status == "inactive" ? true : false),
           import_xml: api_line.xml
         }
-        
+
         # Find Company
         # TODO Check behavior when operator_codes count is 0 or > 1
         if api_line.operator_codes.any?
@@ -137,17 +137,13 @@ module Stif
       def delete_deprecated(objects, klass)
         ids = objects.map{ |o| o.stif_id }.to_a
         deprecated = klass.where.not(objectid: ids)
-        count = deprecated.count
-        deprecated.destroy_all
-        count
+        deprecated.destroy_all.length
       end
 
       def delete_deprecated_lines(lines)
         ids = lines.map{ |l| l.stif_id }.to_a
         deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false)
-        count = deprecated.count
-        deprecated.map { |l| l.deactivated = true ; l.save }
-        count
+        deprecated.update_all desactivated: true
       end
 
       def save_or_update(params, klass)
-- 
cgit v1.2.3


From 0d9e4799add9219a32c396a5bb5731f6d525b378 Mon Sep 17 00:00:00 2001
From: Edouard Maffert
Date: Thu, 8 Sep 2016 18:01:58 +0200
Subject: fix typo in codifligne sync

---
 lib/stif/codif_line_synchronization.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/stif/codif_line_synchronization.rb b/lib/stif/codif_line_synchronization.rb
index fa27e9a38..01ad29afc 100644
--- a/lib/stif/codif_line_synchronization.rb
+++ b/lib/stif/codif_line_synchronization.rb
@@ -143,7 +143,7 @@ module Stif
       def delete_deprecated_lines(lines)
         ids = lines.map{ |l| l.stif_id }.to_a
         deprecated = Chouette::Line.where.not(objectid: ids).where(deactivated: false)
-        deprecated.update_all desactivated: true
+        deprecated.update_all deactivated: true
       end
 
       def save_or_update(params, klass)
-- 
cgit v1.2.3