Wip refactoring cleanup model

Refs #1933

9 files changed, 207 insertions, 124 deletions

diff --git a/app/controllers/clean_ups_controller.rb b/app/controllers/clean_ups_controller.rb
index 8718fc0dc..c6e4ecd44 100644
--- a/app/controllers/clean_ups_controller.rb
+++ b/app/controllers/clean_ups_controller.rb
@@ -1,23 +1,19 @@
 class CleanUpsController < ChouetteController
   respond_to :html, :only => [:create]
-
   belongs_to :referential
 
   def create
-    clean_up = CleanUp.new(params[:clean_up])
-
-    if clean_up.invalid?
-      flash[:alert] = clean_up.errors.full_messages.join("<br/>")
+    clean_up = CleanUp.new(clean_up_params)
+    clean_up.referential = @referential
+    if clean_up.valid?
+      clean_up.save
     else
-      begin
-        result = clean_up.clean
-        flash[:notice] = result.notice.join("<br/>")
-      rescue => e
-        Rails.logger.error "CleanUp failed : #{e} #{e.backtrace}"
-        flash[:alert] = t('clean_ups.failure', error_message: e.to_s)
-      end
+      flash[:alert] = clean_up.errors.full_messages.join("<br/>")
     end
     redirect_to referential_path(@referential)
   end
 
+  def clean_up_params
+    params.require(:clean_up).permit(:keep_lines, :keep_stops, :keep_companies, :keep_networks, :keep_group_of_lines, :expected_date)
+  end
 end
diff --git a/app/models/clean_up.rb b/app/models/clean_up.rb
index cc8a6e537..986fbdd24 100644
--- a/app/models/clean_up.rb
+++ b/app/models/clean_up.rb
@@ -1,138 +1,121 @@
-class CleanUp
-  include ActiveModel::Validations
-  include ActiveModel::Conversion
-  extend ActiveModel::Naming
+class CleanUp < ActiveRecord::Base
+  include AASM
+  belongs_to :referential
+  validates :expected_date, presence: true
+  after_commit :perform_cleanup, :on => :create
 
-  attr_accessor :expected_date, :keep_lines, :keep_stops , :keep_companies
-  attr_accessor :keep_networks, :keep_group_of_lines
+  def perform_cleanup
+    CleanUpWorker.perform_async(self.id)
+  end
+
+  aasm column: :status do
+    state :new, :initial => true
+    state :pending
+    state :successful
+    state :failed
+
+    event :run, after: :update_started_at do
+      transitions :from => [:new, :failed], :to => :pending
+    end
 
-  validates_presence_of :expected_date
+    event :successful, after: :update_ended_at do
+      transitions :from => [:pending, :failed], :to => :successful
+    end
 
-  def initialize(attributes = {})
-    attributes.each do |name, value|
-      send("#{name}=", value)
+    event :failed, after: :update_ended_at do
+      transitions :from => :pending, :to => :failed
     end
   end
 
-  def persisted?
-    false
+  def update_started_at
+    update_attribute(:started_at, Time.now)
   end
 
-  def physical_stop_areas
-    Chouette::StopArea.physical.includes(:stop_points).where(:stop_points => {id: nil})
+  def update_ended_at
+    update_attribute(:ended_at, Time.now)
   end
 
-  def clean_physical_stop_areas
-     Chouette::StopArea.where(id: self.physical_stop_areas.pluck(:id)).delete_all
+  def clean
+    # as foreign keys are presents , delete method can be used for faster performance
+    # find and remove time_tables
+    result = CleanUpResult.new
+    tms = Chouette::TimeTable.validity_out_from_on?(expected_date)
+    result.time_table_count = tms.size
+    tms.each.map(&:delete)
+
+    result.vehicle_journey_count = self.clean_vehicle_journeys
+    result.journey_pattern_count = self.clean_journey_patterns
+    result.route_count           = self.clean_routes
+    result.line_count            = self.clean_lines unless keep_lines
+
+    unless keep_stops
+      result.stop_count += self.clean_physical_stop_areas
+      result.stop_count += self.clean_commercial_stop_areas
+      result.stop_count += self.clean_stop_place_stop_areas
+      result.stop_count += self.clean_itl_stop_areas
+    end
+
+    # If asked remove companies without lines or vehicle journeys
+    result.company_count       = self.clean_companies unless keep_companies
+    # If asked remove networks without lines
+    result.network_count       = self.clean_networks unless keep_networks
+    # If asked remove group_of_lines without lines
+    result.group_of_line_count = self.clean_group_of_lines unless keep_group_of_lines
+    result
   end
 
-  def commercial_stop_areas
-    Chouette::StopArea.commercial
+  def clean_physical_stop_areas
+    ids = Chouette::StopArea.physical.includes(:stop_points).where(:stop_points => {id: nil}).pluck(:id)
+    Chouette::StopArea.physical.where(id: ids).delete_all
   end
 
-  def stop_place_stop_areas
-    Chouette::StopArea.stop_place
+  def clean_commercial_stop_areas
+    ids = Chouette::StopArea.commercial.where.not(parent_id: nil).pluck(:parent_id)
+    Chouette::StopArea.commercial.where.not(id: ids).delete_all
   end
 
-  def itl_stop_areas
-    Chouette::StopArea.itl
+  def clean_stop_place_stop_areas
+    ids = Chouette::StopArea.stop_place.includes(:stop_points).where(:stop_points => {id: nil}).pluck(:id)
+    Chouette::StopArea.stop_place.where(id: ids).delete_all
   end
 
-  def vehicle_journeys
-    Chouette::VehicleJourney.includes(:time_tables).where(:time_tables => {id: nil})
+  def clean_itl_stop_areas
+    ids = Chouette::StopArea.itl.includes(:stop_points).where(:stop_points => {id: nil}).pluck(:id)
+    Chouette::StopArea.itl.where(id: ids).delete_all
   end
+
   def clean_vehicle_journeys
-    Chouette::VehicleJourney.where(id: self.vehicle_journeys.pluck(:id)).delete_all
+    ids = Chouette::VehicleJourney.includes(:time_tables).where(:time_tables => {id: nil}).pluck(:id)
+    Chouette::VehicleJourney.where(id: ids).delete_all
   end
 
-  def lines
-    Chouette::Line.includes(:routes).where(:routes => {id: nil})
-  end
   def clean_lines
-    Chouette::Line.where(id: self.lines.pluck(:id)).delete_all
+    ids = Chouette::Line.includes(:routes).where(:routes => {id: nil}).pluck(:id)
+    Chouette::Line.where(id: ids).delete_all
   end
 
-  def routes
-    Chouette::Route.includes(:journey_patterns).where(:journey_patterns => {id: nil})
-  end
   def clean_routes
-    Chouette::Route.where(id: self.routes.pluck(:id)).delete_all
+    ids = Chouette::Route.includes(:journey_patterns).where(:journey_patterns => {id: nil}).pluck(:id)
+    Chouette::Route.where(id: ids).delete_all
   end
 
-  def journey_patterns
-    Chouette::JourneyPattern.includes(:vehicle_journeys).where(:vehicle_journeys => {id: nil})
-  end
   def clean_journey_patterns
-    Chouette::JourneyPattern.where(id: self.journey_patterns.pluck(:id)).delete_all
+    ids = Chouette::JourneyPattern.includes(:vehicle_journeys).where(:vehicle_journeys => {id: nil}).pluck(:id)
+    Chouette::JourneyPattern.where(id: ids).delete_all
   end
 
-  def clean
-    # as foreign keys are presents , delete method can be used for faster performance
-    result = CleanUpResult.new
-    # find and remove time_tables
-    tms = Chouette::TimeTable.validity_out_from_on?(Date.parse(expected_date))
-    result.time_table_count = tms.size
-    tms.each do |tm|
-      tm.delete
-    end
-
-    result.vehicle_journey_count = self.clean_vehicle_journeys
-    result.journey_pattern_count = self.clean_journey_patterns
-    result.route_count           = self.clean_routes
-
-    result.line_count = self.clean_lines if keep_lines == "0"
-    result.stop_count = self.clean_physical_stop_areas if keep_stops == "0"
-
-    if keep_stops == "0"
-      commercial_stop_areas.find_each do |csp|
-        if csp.children.size == 0
-          result.stop_count += 1
-          csp.delete
-        end
-      end
-      stop_place_stop_areas.find_each do |sp|
-        if sp.children.size == 0
-          result.stop_count += 1
-          sp.delete
-        end
-      end
-      itl_stop_areas.find_each do |itl|
-        if itl.routing_stops.size == 0
-          result.stop_count += 1
-          itl.delete
-        end
-      end
-    end
-    # if asked remove companies without lines or vehicle journeys
-    if keep_companies == "0"
-      Chouette::Company.find_each do |c|
-        if c.lines.size == 0
-          result.company_count += 1
-          c.delete
-        end
-      end
-    end
-
-    # if asked remove networks without lines
-    if keep_networks == "0"
-      Chouette::Network.find_each do |n|
-        if n.lines.size == 0
-          result.network_count += 1
-          n.delete
-        end
-      end
-    end
+  def clean_companies
+    ids = Chouette::Company.includes(:lines).where(:lines => {id: nil}).pluck(:id)
+    Chouette::Company.where(id: ids).delete_all
+  end
 
-    # if asked remove group_of_lines without lines
-    if keep_group_of_lines == "0"
-      Chouette::GroupOfLine.find_each do |n|
-        if n.lines.size == 0
-          result.group_of_line_count += 1
-          n.delete
-        end
-      end
-    end
-    result
+  def clean_networks
+    ids = Chouette::Network.includes(:lines).where(:lines => {id: nil}).pluck(:id)
+    Chouette::Network.where(id: ids).delete_all
   end
 
+  def clean_group_of_lines
+    ids = Chouette::GroupOfLine.includes(:lines).where(:lines => {id: nil}).pluck(:id)
+    Chouette::GroupOfLine.where(id: ids).delete_all
+  end
 end
diff --git a/app/workers/clean_up_worker.rb b/app/workers/clean_up_worker.rb
new file mode 100644
index 000000000..f758b900b
--- /dev/null
+++ b/app/workers/clean_up_worker.rb
@@ -0,0 +1,16 @@
+class CleanUpWorker
+  include Sidekiq::Worker
+
+  def perform(id)
+    cleaner = CleanUp.find id
+    cleaner.run!
+    begin
+      cleaner.referential.switch
+      result = cleaner.clean
+      cleaner.successful!
+    rescue Exception => e
+      Rails.logger.error "CleanUpWorker : #{e}"
+      cleaner.failed!
+    end
+  end
+end
diff --git a/config/initializers/apartment.rb b/config/initializers/apartment.rb
index 70bda1462..445e32b78 100644
--- a/config/initializers/apartment.rb
+++ b/config/initializers/apartment.rb
@@ -35,7 +35,8 @@ Apartment.configure do |config|
     "Chouette::Company",
     "Chouette::Network",
     "ReferentialCloning",
-    "Workbench"
+    "Workbench",
+    "CleanUp"
   ]
 
   # use postgres schemas?
diff --git a/db/migrate/20161114134518_create_clean_ups.rb b/db/migrate/20161114134518_create_clean_ups.rb
new file mode 100644
index 000000000..3fa60d484
--- /dev/null
+++ b/db/migrate/20161114134518_create_clean_ups.rb
@@ -0,0 +1,18 @@
+class CreateCleanUps < ActiveRecord::Migration
+  def change
+    create_table :clean_ups do |t|
+      t.string :status
+      t.datetime :started_at
+      t.datetime :ended_at
+      t.references :referential, index: true
+      t.boolean :keep_lines
+      t.boolean :keep_stops
+      t.boolean :keep_companies
+      t.boolean :keep_networks
+      t.boolean :keep_group_of_lines
+      t.datetime :expected_date
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index baf25daa5..2b682eb19 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20161109160857) do
+ActiveRecord::Schema.define(version: 20161114134518) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -77,6 +77,23 @@ ActiveRecord::Schema.define(version: 20161109160857) do
     t.datetime "updated_at"
   end
 
+  create_table "clean_ups", force: true do |t|
+    t.string   "status"
+    t.datetime "started_at"
+    t.datetime "ended_at"
+    t.integer  "referential_id"
+    t.boolean  "keep_lines"
+    t.boolean  "keep_stops"
+    t.boolean  "keep_companies"
+    t.boolean  "keep_networks"
+    t.boolean  "keep_group_of_lines"
+    t.datetime "expected_date"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "clean_ups", ["referential_id"], :name => "index_clean_ups_on_referential_id"
+
   create_table "companies", force: true do |t|
     t.string   "objectid",                  null: false
     t.integer  "object_version"
@@ -124,6 +141,22 @@ ActiveRecord::Schema.define(version: 20161109160857) do
 
   add_index "connection_links", ["objectid"], :name => "connection_links_objectid_key", :unique => true
 
+  create_table "delayed_jobs", force: true do |t|
+    t.integer  "priority",   default: 0
+    t.integer  "attempts",   default: 0
+    t.text     "handler"
+    t.text     "last_error"
+    t.datetime "run_at"
+    t.datetime "locked_at"
+    t.datetime "failed_at"
+    t.string   "locked_by"
+    t.string   "queue"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "delayed_jobs", ["priority", "run_at"], :name => "delayed_jobs_priority"
+
   create_table "exports", force: true do |t|
     t.integer  "referential_id",  limit: 8
     t.string   "status"
diff --git a/spec/factories/clean_ups.rb b/spec/factories/clean_ups.rb
new file mode 100644
index 000000000..41165ac16
--- /dev/null
+++ b/spec/factories/clean_ups.rb
@@ -0,0 +1,15 @@
+FactoryGirl.define do
+  factory :clean_up do
+    status "MyString"
+started_at "2016-11-14 14:45:18"
+ended_at "2016-11-14 14:45:18"
+referential nil
+keep_lines false
+keep_stops false
+keep_companies false
+keep_networks false
+keep_group_of_lines false
+expected_date "2016-11-14 14:45:18"
+  end
+
+end
diff --git a/spec/models/clean_up_spec.rb b/spec/models/clean_up_spec.rb
index d633443ba..e603b09b0 100644
--- a/spec/models/clean_up_spec.rb
+++ b/spec/models/clean_up_spec.rb
@@ -1,41 +1,58 @@
-require 'spec_helper'
+require 'rails_helper'
 
-describe CleanUp, :type => :model do
+RSpec.describe CleanUp, :type => :model do
   let(:cleaner) { CleanUp.new }
 
+  it { should validate_presence_of(:expected_date) }
+  it { should belong_to(:referential) }
+
+  it 'should convert initialize attribute to boolean' do
+    params  = { keep_lines: "0", keep_stops: "1" }
+    cleaner = CleanUp.new(params)
+    expect(cleaner.keep_lines).to eq false
+    expect(cleaner.keep_stops).to eq true
+  end
+
+  it 'should delete group of line without lines' do
+    create_list(:group_of_line, 2)
+    create_list(:line, 2, group_of_lines: [create(:group_of_line)])
+    expect(cleaner.clean_group_of_lines).to eq 2
+  end
+
+  it 'should delete company without lines' do
+    create_list(:company, 2)
+    create_list(:line, 2)
+    expect(cleaner.clean_companies).to eq 2
+  end
+
   it 'should delete physical stop_areas without stop_points' do
     create_list(:stop_area, 2)
     create_list(:stop_point, 2)
     Chouette::StopArea.update_all(area_type: "Quay")
-    expect(cleaner.physical_stop_areas.count).to eq 2
     expect(cleaner.clean_physical_stop_areas).to eq 2
   end
 
   it 'should delete vehiclejourneys without timetables' do
     create_list(:vehicle_journey, 2)
     create_list(:vehicle_journey, 2, time_tables:[create(:time_table)])
-    expect(cleaner.vehicle_journeys.count).to eq 2
     expect(cleaner.clean_vehicle_journeys).to eq 2
   end
 
   it 'should delete lines without routes' do
     create_list(:line, 2)
     create_list(:route, 2, line: create(:line))
-    expect(cleaner.lines.count).to eq 2
     expect(cleaner.clean_lines).to eq 2
   end
 
   it 'should delete routes without journeypatterns' do
     create_list(:route, 2)
     create_list(:journey_pattern, 2, route: create(:route))
-    expect(cleaner.routes.count).to eq 2
     expect(cleaner.clean_routes).to eq 2
   end
 
   it 'should delete journeypatterns without vehicle journeys' do
     create_list(:journey_pattern, 2)
     create_list(:vehicle_journey, 2, journey_pattern: create(:journey_pattern))
-    expect(cleaner.journey_patterns.count).to eq 2
     expect(cleaner.clean_journey_patterns).to eq 2
   end
 end
diff --git a/spec/workers/clean_up_worker_spec.rb b/spec/workers/clean_up_worker_spec.rb
new file mode 100644
index 000000000..e85768fa3
--- /dev/null
+++ b/spec/workers/clean_up_worker_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+RSpec.describe CleanUpWorker, type: :worker do
+    pending "add some examples to (or delete) #{__FILE__}"
+end