Refs #5797 @3h; Automatically clean referentials5372-clean-referential-after-cloning

Automatically trigger a CleanUp operation on referentials when the
metadatas change.
Also jupdate the CleanUp operation to use the referential metadatas
dates insterad of arbitrary one.

7 files changed, 470 insertions, 218 deletions

diff --git a/app/controllers/clean_ups_controller.rb b/app/controllers/clean_ups_controller.rb
index ec28aa0fc..266f4b8ce 100644
--- a/app/controllers/clean_ups_controller.rb
+++ b/app/controllers/clean_ups_controller.rb
@@ -15,6 +15,6 @@ class CleanUpsController < ChouetteController
   end
 
   def clean_up_params
-    params.require(:clean_up).permit(:date_type, :begin_date, :end_date)
+    params.require(:clean_up).permit(:date_type, :begin_date, :end_date).update({mode: :manual})
   end
 end
diff --git a/app/models/clean_up.rb b/app/models/clean_up.rb
index 7aab7f32e..cf665793b 100644
--- a/app/models/clean_up.rb
+++ b/app/models/clean_up.rb
@@ -5,13 +5,20 @@ class CleanUp < ActiveRecord::Base
   has_one :clean_up_result
 
   enumerize :date_type, in: %i(between before after)
+  enumerize :mode, in: %i(manual auto)
 
-  validates_presence_of :begin_date, message: :presence
-  validates_presence_of :end_date, message: :presence, if: Proc.new {|cu| cu.date_type == 'between'}
-  validates_presence_of :date_type, message: :presence
+  validates_presence_of :referential, message: :presence
+  validates_presence_of :mode, message: :presence
+  validates_presence_of :begin_date, message: :presence, if: :manual?
+  validates_presence_of :end_date, message: :presence, if: Proc.new {|cu| cu.mode == 'manual' && cu.date_type == 'between'}
+  validates_presence_of :date_type, message: :presence, if: :manual?
   validate :end_date_must_be_greater_that_begin_date
   after_commit :perform_cleanup, :on => :create
 
+  def manual?
+    mode == 'manual'
+  end
+
   def end_date_must_be_greater_that_begin_date
     if self.end_date && self.date_type == 'between' && self.begin_date >= self.end_date
       errors.add(:base, I18n.t('activerecord.errors.models.clean_up.invalid_period'))
@@ -22,6 +29,12 @@ class CleanUp < ActiveRecord::Base
     CleanUpWorker.perform_async(self.id)
   end
 
+  def date_type_with_mode
+    manual? ? date_type_without_mode : "based_on_referential"
+  end
+
+  alias_method_chain :date_type, :mode
+
   def clean
     {}.tap do |result|
       processed = send("destroy_time_tables_#{self.date_type}")
@@ -34,9 +47,20 @@ class CleanUp < ActiveRecord::Base
       self.overlapping_periods.each do |period|
         exclude_dates_in_overlapping_period(period)
       end
+      self.destroy_lines_related_objects_based_on_referential unless manual?
     end
   end
 
+  def destroy_lines_related_objects_based_on_referential
+    remaining_line_ids = referential.metadatas.pluck(:line_ids).flatten.uniq
+    Chouette::Route.where.not(line_id: remaining_line_ids).destroy_all
+  end
+
+  def destroy_time_tables_based_on_referential
+    time_tables = Chouette::TimeTable.none
+    self.destroy_time_tables(time_tables)
+  end
+
   def destroy_time_tables_between
     time_tables = Chouette::TimeTable.where('end_date < ? AND start_date > ?', self.end_date, self.begin_date)
     self.destroy_time_tables(time_tables)
@@ -52,6 +76,11 @@ class CleanUp < ActiveRecord::Base
     self.destroy_time_tables(time_tables)
   end
 
+  def destroy_time_tables_dates_based_on_referential
+    query = referential_metadata_periodes_query "(date < ? OR date > ?)", "AND"
+    Chouette::TimeTableDate.in_dates.where(*query).destroy_all
+  end
+
   def destroy_time_tables_dates_before
     Chouette::TimeTableDate.in_dates.where('date < ?', self.begin_date).destroy_all
   end
@@ -64,6 +93,11 @@ class CleanUp < ActiveRecord::Base
     Chouette::TimeTableDate.in_dates.where('date > ? AND date < ?', self.begin_date, self.end_date).destroy_all
   end
 
+  def destroy_time_tables_periods_based_on_referential
+    query = referential_metadata_periodes_query "(period_end < ? OR period_start > ?)", "AND"
+    Chouette::TimeTablePeriod.where(*query).destroy_all
+  end
+
   def destroy_time_tables_periods_before
     Chouette::TimeTablePeriod.where('period_end < ?', self.begin_date).destroy_all
   end
@@ -77,28 +111,48 @@ class CleanUp < ActiveRecord::Base
   end
 
   def overlapping_periods
-    self.end_date = self.begin_date if self.date_type != 'between'
-    Chouette::TimeTablePeriod.where('(period_start, period_end) OVERLAPS (?, ?)', self.begin_date, self.end_date)
+    if manual?
+      self.end_date = self.begin_date if self.date_type != 'between'
+      Chouette::TimeTablePeriod.where('(period_start, period_end) OVERLAPS (?, ?)', self.begin_date, self.end_date)
+    else
+      periods_query = [
+        (['(period_start, period_end) OVERLAPS (?, ?)'] * referential_metadata_periodes_boundaries.size).join(' OR '),
+        *referential_metadata_periodes_boundaries.flatten
+      ]
+
+      Chouette::TimeTablePeriod.where(*periods_query)
+    end
   end
 
   def exclude_dates_in_overlapping_period(period)
     days_in_period  = period.period_start..period.period_end
-    day_out         = period.time_table.dates.where(in_out: false).map(&:date)
-    # check if day is greater or less then cleanup date
-    if date_type != 'between'
-      operator = date_type == 'after' ? '>' : '<'
-      to_exclude_days = days_in_period.map do |day|
-        day if day.public_send(operator, self.begin_date)
+    day_out         = period.time_table.dates.where(in_out: false).pluck(:date)
+
+    if manual?
+      # check if day is greater or less then cleanup date
+      if date_type != 'between'
+        operator = date_type == 'after' ? '>' : '<'
+        to_exclude_days = days_in_period.map do |day|
+          day if day.public_send(operator, self.begin_date)
+        end
+      else
+        days_in_cleanup_periode = (self.begin_date..self.end_date)
+        to_exclude_days = days_in_period & days_in_cleanup_periode
       end
     else
-      days_in_cleanup_periode = (self.begin_date..self.end_date)
-      to_exclude_days = days_in_period & days_in_cleanup_periode
+      # Here the behaviour is the opposite:
+      # We want to KEEP dates overlapping the referential metadatas
+      to_keep_days = referential_metadata_periodes.map do |metadata_period|
+        (days_in_period & (metadata_period.begin..metadata_period.end)).to_a
+      end.flatten
+      to_exclude_days = days_in_period.to_a - to_keep_days
     end
 
-    to_exclude_days.to_a.compact.each do |day|
+
+    to_exclude_days.to_a.compact.uniq.each do |day|
       # we ensure day is not already an exclude date
       # and that day is not equal to the boundariy date of the clean up
-      if !day_out.include?(day) && day != self.begin_date && day != self.end_date
+      unless day_out.include?(day) || (manual? && (day == self.begin_date || day == self.end_date))
         self.add_exclude_date(period.time_table, day)
       end
     end
@@ -162,4 +216,23 @@ class CleanUp < ActiveRecord::Base
     update_attribute(:ended_at, Time.now)
     CleanUpResult.create(clean_up: self, message_key: :failed, message_attributes: message_attributes)
   end
+
+  private
+  def referential_metadata_periodes
+    referential.metadatas.map(&:periodes).flatten
+  end
+
+  def referential_metadata_periodes_boundaries
+    referential_metadata_periodes.map do |periode|
+      [periode.begin, periode.end]
+    end
+  end
+
+  def referential_metadata_periodes_query subquery, operator
+    boundaries = referential_metadata_periodes_boundaries
+    [
+      ([subquery] * boundaries.size).join(" #{operator} "),
+      *boundaries.flatten
+    ]
+  end
 end
diff --git a/app/models/referential.rb b/app/models/referential.rb
index eefd0b77d..e0c08f14d 100644
--- a/app/models/referential.rb
+++ b/app/models/referential.rb
@@ -415,9 +415,7 @@ class Referential < ActiveRecord::Base
   end
 
   def perform_cleanup
-    remaining_line_ids = self.metadatas.reload.pluck(:line_ids).flatten.uniq
-    switch
-    Chouette::Route.where.not(line_id: remaining_line_ids).destroy_all
+    CleanUp.new(referential: self, mode: :auto, date_type: :before).save!
   end
 
   private
diff --git a/db/migrate/20180105102012_add_mode_to_clean_ups.rb b/db/migrate/20180105102012_add_mode_to_clean_ups.rb
new file mode 100644
index 000000000..485594b19
--- /dev/null
+++ b/db/migrate/20180105102012_add_mode_to_clean_ups.rb
@@ -0,0 +1,5 @@
+class AddModeToCleanUps < ActiveRecord::Migration
+  def change
+    add_column :clean_ups, :mode, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e6fdd9d74..06875aed1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,13 +11,14 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171227113809) do
+ActiveRecord::Schema.define(version: 20180105102012) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
-  enable_extension "postgis"
   enable_extension "hstore"
+  enable_extension "postgis"
   enable_extension "unaccent"
+  enable_extension "objectid"
 
   create_table "access_links", id: :bigserial, force: :cascade do |t|
     t.integer  "access_point_id",                        limit: 8
@@ -115,6 +116,7 @@ ActiveRecord::Schema.define(version: 20171227113809) do
     t.datetime "updated_at"
     t.date     "end_date"
     t.string   "date_type"
+    t.string   "mode"
   end
 
   add_index "clean_ups", ["referential_id"], name: "index_clean_ups_on_referential_id", using: :btree
@@ -442,6 +444,7 @@ ActiveRecord::Schema.define(version: 20171227113809) do
     t.string   "checksum"
     t.text     "checksum_source"
     t.string   "data_source_ref"
+    t.json     "costs"
   end
 
   add_index "journey_patterns", ["objectid"], name: "journey_patterns_objectid_key", unique: true, using: :btree
diff --git a/spec/models/clean_up_spec.rb b/spec/models/clean_up_spec.rb
index 2753c8718..dbf300b52 100644
--- a/spec/models/clean_up_spec.rb
+++ b/spec/models/clean_up_spec.rb
@@ -1,261 +1,430 @@
 require 'rails_helper'
 
 RSpec.describe CleanUp, :type => :model do
+  subject{ build(:clean_up, mode: mode) }
+  let(:referential){ Referential.first }
 
-  it { should validate_presence_of(:date_type).with_message(:presence) }
-  it { should validate_presence_of(:begin_date).with_message(:presence) }
-  it { should belong_to(:referential) }
+  context "for automatic cleanups" do
+    let(:mode){ :auto }
+    let(:referential){ create(:referential, :with_metadatas) }
 
-  context 'Clean Up With Date Type : Between' do
-    subject(:cleaner) { create(:clean_up, date_type: :between) }
-    it { should validate_presence_of(:end_date).with_message(:presence) }
+    it { should validate_presence_of(:referential).with_message(:presence) }
+    it { should validate_presence_of(:mode).with_message(:presence) }
+    it { should belong_to(:referential) }
 
-    it 'should have a end date strictly greater than the begin date' do
-      expect(cleaner).to be_valid
+    let(:periode_1){ 1.day.from_now..10.days.from_now }
+    let(:periode_2){ 20.day.from_now..30.days.from_now }
+    let(:periode_3){ 40.day.from_now..50.days.from_now }
+    let(:periodes){ [periode_1, periode_2, periode_3] }
 
-      cleaner.end_date = cleaner.begin_date
-      expect(cleaner).not_to be_valid
-    end
-  end
-
-  context '#exclude_dates_in_overlapping_period with :before date_type' do
+    let(:cleaner) { create(:clean_up, mode: mode, referential: referential) }
     let(:time_table) { create(:time_table) }
     let(:period) { time_table.periods[0] }
-    let(:cleaner) { create(:clean_up, date_type: :before, begin_date: period.period_end) }
 
-    it 'should add exclude date into period for overlapping period' do
-      days_in_period = (period.period_start..period.period_end).count
-
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
-        time_table.dates.where(in_out: false).count
-      }.by(days_in_period - 1)
+    before(:each) do
+      metadata = referential.metadatas.first
+      metadata.periodes = periodes
     end
 
-    it 'should not add exclude date if no overlapping found' do
-      cleaner.begin_date = period.period_start
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
-        time_table.dates.where(in_out: false).count
-      }
-    end
-  end
+    context '#overlapping_periods' do
+      let(:periodes) { [periode_1] }
 
-  context '#exclude_dates_in_overlapping_period with :after date_type' do
-    let(:time_table) { create(:time_table) }
-    let(:period) { time_table.periods[0] }
-    let(:cleaner) { create(:clean_up, date_type: :after, begin_date: period.period_start + 1.day) }
+      context "with overlapping periods" do
+        let(:periode_1) { period.period_start..period.period_end }
 
-    it 'should add exclude date into period for overlapping period' do
-      days_in_period = (period.period_start..period.period_end).count
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
-        time_table.dates.where(in_out: false).count
-      }.by(days_in_period - 2)
-    end
+        it 'should detect overlapping periods' do
+          expect(cleaner.overlapping_periods).to include(time_table.periods[0])
+        end
+      end
 
-    it 'should not add exclude date if no overlapping found' do
-      cleaner.begin_date = period.period_end
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
-        time_table.dates.where(in_out: false).count
-      }
-    end
-  end
+      context "without overlapping periods" do
+        let(:periode_1) { period.period_end+1.day..period.period_end+2.days }
 
-   context '#exclude_dates_in_overlapping_period with :between date_type' do
-    let(:time_table) { create(:time_table) }
-    let(:period) { time_table.periods[0] }
-    let(:cleaner) { create(:clean_up, date_type: :between, begin_date: period.period_start + 3.day, end_date: period.period_end) }
-
-    it 'should add exclude date into period for overlapping period' do
+        it 'should not return non-overlapping periods' do
+          expect(cleaner.overlapping_periods).to_not include(time_table.periods[0])
+        end
+      end
+    end
 
-      expected_day_out = (cleaner.begin_date..cleaner.end_date).count
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
-        time_table.dates.where(in_out: false).count
-      }.by(expected_day_out - 2)
+    context '#exclude_dates_in_overlapping_period' do
+      context "with a fully overlapping period" do
+        let(:periode_1) { period.period_start..period.period_end }
+        it 'should exclude no date' do
+          expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+            time_table.dates.where(in_out: false).count
+          }.by(0)
+        end
+      end
+
+      context "with 2 partially overlapping periods" do
+        let(:periode_1) { period.period_start-10.days..period.period_start+1.day }
+        let(:periode_2) { period.period_end-1.day..period.period_end+10.days }
+        it 'should exclude only the no-overlapping dates' do
+          count = (period.period_start..period.period_end).count
+          count -= 4 # 4 overlapping days, 2 at the beginning, 2 at the end
+          expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+            time_table.dates.where(in_out: false).count
+          }.by(count)
+        end
+      end
+
+      context "without overlapping periods" do
+        let(:periode_1) { period.period_end+1.day..period.period_end+2.days }
+        let(:periodes) { [periode_1] }
+
+        it 'should exclude all the dates' do
+          expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+            time_table.dates.where(in_out: false).count
+          }.by((period.period_start..period.period_end).count)
+        end
+      end
     end
 
-    it 'should not add exclude date if no overlapping found' do
-      cleaner.begin_date = period.period_end  + 1.day
-      cleaner.end_date   = cleaner.begin_date + 1.day
+    context '#clean' do
+      let(:cleaner) { create(:clean_up, date_type: :before, mode: mode, referential: referential) }
 
-      expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
-        time_table.dates.where(in_out: false).count
-      }
+      it 'should call destroy_time_tables_before' do
+        expect(cleaner).to receive(:destroy_time_tables_based_on_referential)
+        expect(cleaner).to receive(:destroy_time_tables_dates_based_on_referential)
+        expect(cleaner).to receive(:destroy_time_tables_periods_based_on_referential)
+        expect(cleaner).to receive(:destroy_lines_related_objects_based_on_referential)
+        cleaner.clean
+      end
     end
-  end
-
-  context '#overlapping_periods' do
-    let(:time_table) { create(:time_table) }
-    let(:period) { time_table.periods[0] }
-    let(:cleaner) { create(:clean_up, date_type: :before, begin_date: period.period_start) }
 
-    it 'should detect overlapping periods' do
-      expect(cleaner.overlapping_periods).to include(time_table.periods[0])
+    context '#destroy_time_tables_dates_based_on_referential' do
+      let(:periodes) { [periode_1, periode_2, periode_3].compact }
+      let(:periode_3){ nil }
+
+      let(:time_table) do
+        time_table = create :time_table
+        time_table.periods.clear
+        time_table.save
+        time_table
+      end
+
+      context "for non-overlapping periods" do
+        let(:periode_1) { time_table.end_date+1.day..time_table.end_date+2.days}
+        let(:periode_2) { time_table.start_date-2.days..time_table.start_date-1.day}
+        it 'should destroy record' do
+          expect{ cleaner.destroy_time_tables_dates_based_on_referential }.to change {
+            Chouette::TimeTableDate.count
+          }.by(- time_table.dates.count)
+        end
+      end
+
+      context "for overlapping periods" do
+        let(:periode_1) { time_table.end_date-1.day..time_table.end_date+2.days}
+        let(:periode_2) { time_table.end_date+1000.days..time_table.end_date+1100.days}
+        let(:periode_2) { time_table.start_date-1100.days..time_table.start_date-1000.days}
+        it 'should destroy record' do
+          expect{ cleaner.destroy_time_tables_dates_based_on_referential }.to change {
+            Chouette::TimeTableDate.count
+          }.by(2 - time_table.dates.count)
+        end
+      end
+
+      context "for fully overlapping periods" do
+        let(:periode_1) { time_table.start_date..time_table.end_date}
+        let(:periode_2) { time_table.end_date+1000.days..time_table.end_date+1100.days}
+        it 'should destroy record' do
+          expect{ cleaner.destroy_time_tables_dates_based_on_referential }.to change {
+            Chouette::TimeTableDate.count
+          }.by(0)
+        end
+      end
     end
 
-    it 'should not return none overlapping periods' do
-      cleaner.begin_date = time_table.periods[0].period_start - 1.day
-      expect(cleaner.overlapping_periods).to_not include(time_table.periods[0])
+    context '#destroy_time_tables_periods_based_on_referential' do
+      let(:periodes) { [periode_1, periode_2, periode_3].compact }
+      let(:periode_2){ nil }
+      let(:periode_3){ nil }
+      let(:period){ create :time_table_period }
+      before do
+        Chouette::TimeTablePeriod.where.not(id: period.id).delete_all
+      end
+
+      context "with overlapping metadata period" do
+        let(:periode_1){ period.period_start-1.day..period.period_start }
+        it 'should not destroy record' do
+          expect{ cleaner.destroy_time_tables_periods_based_on_referential }.to change {
+            Chouette::TimeTablePeriod.count
+          }.by(0)
+        end
+      end
+
+      context "without overlapping metadata period" do
+        let(:periode_1){ period.period_start-10.days..period.period_start-1.day }
+        let(:periode_2){ period.period_end+1.day..period.period_end+10.days }
+        it 'should destroy record' do
+          expect{ cleaner.destroy_time_tables_periods_based_on_referential }.to change {
+            Chouette::TimeTablePeriod.count
+          }.by(-1)
+        end
+      end
     end
   end
 
-  context '#clean' do
-    let(:cleaner) { create(:clean_up, date_type: :before) }
+  context "for manual cleanups" do
+    let(:mode){ :manual }
 
-    it 'should call destroy_time_tables_before' do
-      cleaner.date_type = :before
-      expect(cleaner).to receive(:destroy_time_tables_before)
-      expect(cleaner).to receive(:destroy_time_tables_dates_before)
-      expect(cleaner).to receive(:destroy_time_tables_periods_before)
-      cleaner.clean
-    end
-
-    it 'should call destroy_time_tables_after' do
-      cleaner.date_type = :after
-      expect(cleaner).to receive(:destroy_time_tables_after)
-      expect(cleaner).to receive(:destroy_time_tables_dates_after)
-      expect(cleaner).to receive(:destroy_time_tables_periods_after)
-      cleaner.clean
-    end
+    it { should validate_presence_of(:date_type).with_message(:presence) }
+    it { should validate_presence_of(:begin_date).with_message(:presence) }
+    it { should validate_presence_of(:referential).with_message(:presence) }
+    it { should validate_presence_of(:mode).with_message(:presence) }
+    it { should belong_to(:referential) }
 
-    it 'should call destroy_time_tables_between' do
-      cleaner.date_type = :between
-      expect(cleaner).to receive(:destroy_time_tables_between)
-      expect(cleaner).to receive(:destroy_time_tables_dates_between)
-      expect(cleaner).to receive(:destroy_time_tables_periods_between)
-      cleaner.clean
-    end
-  end
+    context 'Clean Up With Date Type : Between' do
+      subject(:cleaner) { create(:clean_up, date_type: :between, mode: mode, referential: referential) }
+      it { should validate_presence_of(:end_date).with_message(:presence) }
 
-  context '#destroy_time_tables_dates_between' do
-    let!(:time_table) { create(:time_table) }
-    let(:cleaner) { create(:clean_up, date_type: :between) }
+      it 'should have a end date strictly greater than the begin date' do
+        expect(cleaner).to be_valid
 
-    before do
-      time_table.periods.clear
-      time_table.save
-      cleaner.begin_date = time_table.start_date
-      cleaner.end_date   = time_table.end_date
+        cleaner.end_date = cleaner.begin_date
+        expect(cleaner).not_to be_valid
+      end
     end
 
-    it 'should destroy record' do
-      expect{ cleaner.destroy_time_tables_dates_between }.to change {
-        Chouette::TimeTableDate.count
-      }.by(-time_table.dates.count + 2)
+    context '#exclude_dates_in_overlapping_period with :before date_type' do
+      let(:time_table) { create(:time_table) }
+      let(:period) { time_table.periods[0] }
+      let(:cleaner) { create(:clean_up, date_type: :before, begin_date: period.period_end, mode: mode, referential: referential) }
+
+      it 'should add exclude date into period for overlapping period' do
+        days_in_period = (period.period_start..period.period_end).count
+
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+          time_table.dates.where(in_out: false).count
+        }.by(days_in_period - 1)
+      end
+
+      it 'should not add exclude date if no overlapping found' do
+        cleaner.begin_date = period.period_start
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
+          time_table.dates.where(in_out: false).count
+        }
+      end
     end
 
-    it 'should not destroy record not in range' do
-      cleaner.begin_date = time_table.end_date + 1.day
-      cleaner.end_date   = cleaner.begin_date + 1.day
-
-      expect{ cleaner.destroy_time_tables_dates_between }.to_not change {
-        Chouette::TimeTableDate.count
-      }
+    context '#exclude_dates_in_overlapping_period with :after date_type' do
+      let(:time_table) { create(:time_table) }
+      let(:period) { time_table.periods[0] }
+      let(:cleaner) { create(:clean_up, date_type: :after, begin_date: period.period_start + 1.day, mode: mode, referential: referential) }
+
+      it 'should add exclude date into period for overlapping period' do
+        days_in_period = (period.period_start..period.period_end).count
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+          time_table.dates.where(in_out: false).count
+        }.by(days_in_period - 2)
+      end
+
+      it 'should not add exclude date if no overlapping found' do
+        cleaner.begin_date = period.period_end
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
+          time_table.dates.where(in_out: false).count
+        }
+      end
     end
-  end
 
-  context '#destroy_time_tables_dates_after' do
-    let!(:time_table_date) { create(:time_table_date, date: Date.yesterday, in_out: true) }
-    let(:cleaner) { create(:clean_up, date_type: :after, begin_date: time_table_date.date) }
+    context '#exclude_dates_in_overlapping_period with :between date_type' do
+      let(:time_table) { create(:time_table) }
+      let(:period) { time_table.periods[0] }
+      let(:cleaner) { create(:clean_up, date_type: :between, begin_date: period.period_start + 3.day, end_date: period.period_end, mode: mode, referential: referential) }
 
-    it 'should destroy record' do
-      count = Chouette::TimeTableDate.where('date > ?', cleaner.begin_date).count
-      expect{ cleaner.destroy_time_tables_dates_after }.to change {
-        Chouette::TimeTableDate.count
-      }.by(-count)
-    end
-  end
+      it 'should add exclude date into period for overlapping period' do
 
-  context '#destroy_time_tables_between' do
-    let!(:time_table) { create(:time_table ) }
-    let(:cleaner) { create(:clean_up, date_type: :between, begin_date: time_table.start_date - 1.day, end_date: time_table.end_date + 1.day) }
+        expected_day_out = (cleaner.begin_date..cleaner.end_date).count
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to change {
+          time_table.dates.where(in_out: false).count
+        }.by(expected_day_out - 2)
+      end
 
-    it 'should destroy time_tables with validity period in purge range' do
-      expect{ cleaner.destroy_time_tables_between }.to change {
-        Chouette::TimeTable.count
-      }.by(-1)
-    end
+      it 'should not add exclude date if no overlapping found' do
+        cleaner.begin_date = period.period_end  + 1.day
+        cleaner.end_date   = cleaner.begin_date + 1.day
 
-    it 'should not destroy time_tables if not totaly inside purge range' do
-      cleaner.begin_date = time_table.start_date + 1.day
-      expect{ cleaner.destroy_time_tables_between }.to_not change {
-        Chouette::TimeTable.count
-      }
+        expect { cleaner.exclude_dates_in_overlapping_period(period) }.to_not change {
+          time_table.dates.where(in_out: false).count
+        }
+      end
     end
-  end
 
-  context '#destroy_time_tables_after' do
-    let!(:time_table) { create(:time_table ) }
-    let(:cleaner) { create(:clean_up, date_type: :after, begin_date: time_table.start_date - 1.day) }
+    context '#overlapping_periods' do
+      let(:time_table) { create(:time_table) }
+      let(:period) { time_table.periods[0] }
+      let(:cleaner) { create(:clean_up, date_type: :before, begin_date: period.period_start, mode: mode, referential: referential) }
 
-    it 'should destroy time_tables with start_date > purge begin_date' do
-      expect{ cleaner.destroy_time_tables_after }.to change {
-        Chouette::TimeTable.count
-      }.by(-1)
-    end
+      it 'should detect overlapping periods' do
+        expect(cleaner.overlapping_periods).to include(time_table.periods[0])
+      end
 
-    it 'should not destroy time_tables with start_date < purge begin date' do
-      cleaner.begin_date = time_table.end_date
-      expect{ cleaner.destroy_time_tables_after }.to_not change {
-        Chouette::TimeTable.count
-      }
+      it 'should not return non-overlapping periods' do
+        cleaner.begin_date = time_table.periods[0].period_start - 1.day
+        expect(cleaner.overlapping_periods).to_not include(time_table.periods[0])
+      end
     end
-  end
 
-  context '#destroy_time_tables' do
-    let!(:time_table) { create(:time_table) }
-    let(:cleaner) { create(:clean_up) }
+    context '#clean' do
+      let(:cleaner) { create(:clean_up, date_type: :before, mode: mode, referential: referential) }
+
+      it 'should call destroy_time_tables_before' do
+        cleaner.date_type = :before
+        expect(cleaner).to receive(:destroy_time_tables_before)
+        expect(cleaner).to receive(:destroy_time_tables_dates_before)
+        expect(cleaner).to receive(:destroy_time_tables_periods_before)
+        cleaner.clean
+      end
+
+      it 'should call destroy_time_tables_after' do
+        cleaner.date_type = :after
+        expect(cleaner).to receive(:destroy_time_tables_after)
+        expect(cleaner).to receive(:destroy_time_tables_dates_after)
+        expect(cleaner).to receive(:destroy_time_tables_periods_after)
+        cleaner.clean
+      end
+
+      it 'should call destroy_time_tables_between' do
+        cleaner.date_type = :between
+        expect(cleaner).to receive(:destroy_time_tables_between)
+        expect(cleaner).to receive(:destroy_time_tables_dates_between)
+        expect(cleaner).to receive(:destroy_time_tables_periods_between)
+        cleaner.clean
+      end
+    end
 
-    it 'should destroy all time_tables' do
-      expect{cleaner.destroy_time_tables(Chouette::TimeTable.all)}.to change {
-        Chouette::TimeTable.count
-      }.by(-1)
+    context '#destroy_time_tables_dates_between' do
+      let!(:time_table) { create(:time_table) }
+      let(:cleaner) { create(:clean_up, date_type: :between, mode: mode, referential: referential) }
+
+      before do
+        time_table.periods.clear
+        time_table.save
+        cleaner.begin_date = time_table.start_date
+        cleaner.end_date   = time_table.end_date
+      end
+
+      it 'should destroy record' do
+        expect{ cleaner.destroy_time_tables_dates_between }.to change {
+          Chouette::TimeTableDate.count
+        }.by(-time_table.dates.count + 2)
+      end
+
+      it 'should not destroy record not in range' do
+        cleaner.begin_date = time_table.end_date + 1.day
+        cleaner.end_date   = cleaner.begin_date + 1.day
+
+        expect{ cleaner.destroy_time_tables_dates_between }.to_not change {
+          Chouette::TimeTableDate.count
+        }
+      end
     end
 
-    it 'should destroy time_table vehicle_journey association' do
-      vj = create(:vehicle_journey, time_tables: [time_table, create(:time_table)])
-      cleaner.destroy_time_tables(Chouette::TimeTable.where(id: time_table.id))
+    context '#destroy_time_tables_dates_after' do
+      let!(:time_table_date) { create(:time_table_date, date: Date.yesterday, in_out: true) }
+      let(:cleaner) { create(:clean_up, date_type: :after, begin_date: time_table_date.date, mode: mode, referential: referential) }
 
-      expect(vj.reload.time_tables.map(&:id)).to_not include(time_table.id)
+      it 'should destroy record' do
+        count = Chouette::TimeTableDate.where('date > ?', cleaner.begin_date).count
+        expect{ cleaner.destroy_time_tables_dates_after }.to change {
+          Chouette::TimeTableDate.count
+        }.by(-count)
+      end
     end
 
-    it 'should also destroy associated vehicle_journey if it belongs to any other time_table' do
-      vj = create(:vehicle_journey, time_tables: [time_table])
-      expect{cleaner.destroy_time_tables(Chouette::TimeTable.all)}.to change {
-        Chouette::VehicleJourney.count
-      }.by(-1)
+    context '#destroy_time_tables_between' do
+      let!(:time_table) { create(:time_table ) }
+      let(:cleaner) { create(:clean_up, date_type: :between, begin_date: time_table.start_date - 1.day, end_date: time_table.end_date + 1.day, mode: mode, referential: referential) }
+
+      it 'should destroy time_tables with validity period in purge range' do
+        expect{ cleaner.destroy_time_tables_between }.to change {
+          Chouette::TimeTable.count
+        }.by(-1)
+      end
+
+      it 'should not destroy time_tables if not totaly inside purge range' do
+        cleaner.begin_date = time_table.start_date + 1.day
+        expect{ cleaner.destroy_time_tables_between }.to_not change {
+          Chouette::TimeTable.count
+        }
+      end
     end
-  end
-
-  context '#destroy_vehicle_journey_without_time_table' do
-    let(:cleaner) { create(:clean_up) }
 
-    it 'should destroy vehicle_journey' do
-      vj = create(:vehicle_journey)
-      expect{cleaner.destroy_vehicle_journey_without_time_table
-      }.to change { Chouette::VehicleJourney.count }.by(-1)
+    context '#destroy_time_tables_after' do
+      let!(:time_table) { create(:time_table ) }
+      let(:cleaner) { create(:clean_up, date_type: :after, begin_date: time_table.start_date - 1.day, mode: mode, referential: referential) }
+
+      it 'should destroy time_tables with start_date > purge begin_date' do
+        expect{ cleaner.destroy_time_tables_after }.to change {
+          Chouette::TimeTable.count
+        }.by(-1)
+      end
+
+      it 'should not destroy time_tables with start_date < purge begin date' do
+        cleaner.begin_date = time_table.end_date
+        expect{ cleaner.destroy_time_tables_after }.to_not change {
+          Chouette::TimeTable.count
+        }
+      end
     end
 
-    it 'should not destroy vehicle_journey with time_table' do
-      create(:vehicle_journey, time_tables: [create(:time_table)])
-      expect{cleaner.destroy_vehicle_journey_without_time_table
-      }.to_not change { Chouette::VehicleJourney.count }
+    context '#destroy_time_tables' do
+      let!(:time_table) { create(:time_table) }
+      let(:cleaner) { create(:clean_up, mode: mode, referential: referential) }
+
+      it 'should destroy all time_tables' do
+        expect{cleaner.destroy_time_tables(Chouette::TimeTable.all)}.to change {
+          Chouette::TimeTable.count
+        }.by(-1)
+      end
+
+      it 'should destroy time_table vehicle_journey association' do
+        vj = create(:vehicle_journey, time_tables: [time_table, create(:time_table)])
+        cleaner.destroy_time_tables(Chouette::TimeTable.where(id: time_table.id))
+
+        expect(vj.reload.time_tables.map(&:id)).to_not include(time_table.id)
+      end
+
+      it 'should also destroy associated vehicle_journey if it belongs to any other time_table' do
+        vj = create(:vehicle_journey, time_tables: [time_table])
+        expect{cleaner.destroy_time_tables(Chouette::TimeTable.all)}.to change {
+          Chouette::VehicleJourney.count
+        }.by(-1)
+      end
     end
-  end
 
-  context '#destroy_time_tables_before' do
-    let!(:time_table) { create(:time_table ) }
-    let(:cleaner) { create(:clean_up, date_type: :before, begin_date: time_table.end_date + 1.day) }
+    context '#destroy_vehicle_journey_without_time_table' do
+      let(:cleaner) { create(:clean_up, mode: mode, referential: referential) }
+
+      it 'should destroy vehicle_journey' do
+        vj = create(:vehicle_journey)
+        expect{cleaner.destroy_vehicle_journey_without_time_table
+        }.to change { Chouette::VehicleJourney.count }.by(-1)
+      end
 
-    it 'should destroy time_tables with end_date < purge begin_date' do
-      expect{ cleaner.destroy_time_tables_before }.to change {
-        Chouette::TimeTable.count
-      }.by(-1)
+      it 'should not destroy vehicle_journey with time_table' do
+        create(:vehicle_journey, time_tables: [create(:time_table)])
+        expect{cleaner.destroy_vehicle_journey_without_time_table
+        }.to_not change { Chouette::VehicleJourney.count }
+      end
     end
 
-    it 'should not destroy time_tables with end_date > purge begin_date' do
-      cleaner.begin_date = Date.today
-      expect{ cleaner.destroy_time_tables_before }.to_not change {
-        Chouette::TimeTable.count
-      }
+    context '#destroy_time_tables_before' do
+      let!(:time_table) { create(:time_table ) }
+      let(:cleaner) { create(:clean_up, date_type: :before, begin_date: time_table.end_date + 1.day, mode: mode, referential: referential) }
+
+      it 'should destroy time_tables with end_date < purge begin_date' do
+        expect{ cleaner.destroy_time_tables_before }.to change {
+          Chouette::TimeTable.count
+        }.by(-1)
+      end
+
+      it 'should not destroy time_tables with end_date > purge begin_date' do
+        cleaner.begin_date = Date.today
+        expect{ cleaner.destroy_time_tables_before }.to_not change {
+          Chouette::TimeTable.count
+        }
+      end
     end
   end
 end
diff --git a/spec/models/referential_spec.rb b/spec/models/referential_spec.rb
index 549c5420c..22869f827 100644
--- a/spec/models/referential_spec.rb
+++ b/spec/models/referential_spec.rb
@@ -158,8 +158,12 @@ describe Referential, :type => :model do
     end
   end
 
-  context 'with data' do
+  context 'with data', skip_first_referential: true do
     before(:each) do
+      allow_any_instance_of(CleanUp).to receive(:save!) do |cleanup|
+        cleanup.referential.switch
+        cleanup.clean
+      end
       ref.switch
       Chouette::Line.all.each do |line|
         route = create(:route, line: line)